ansible kubernetes deployment
This commit is contained in:
17
ansible/playbooks/deploy_k8s.yml
Normal file
17
ansible/playbooks/deploy_k8s.yml
Normal file
@@ -0,0 +1,17 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: deploy_k8s.yml
|
||||
# ------------------------------------------------------------------------------
|
||||
- name: 1. Prepare all nodes for Kubernetes
|
||||
hosts: kube_cluster
|
||||
roles:
|
||||
- role: kubernetes
|
||||
|
||||
- name: 2. Initialize and configure the control plane
|
||||
hosts: fastpass_control_plane
|
||||
roles:
|
||||
- role: fastpass-control-plane
|
||||
|
||||
- name: 3. Join worker nodes to the cluster
|
||||
hosts: fastpass-workers
|
||||
roles:
|
||||
- role: fastpass-workers
|
||||
4
ansible/playbooks/roles/common/files/unbound.conf
Normal file
4
ansible/playbooks/roles/common/files/unbound.conf
Normal file
@@ -0,0 +1,4 @@
|
||||
[Resolve]
|
||||
DNS=127.0.0.1
|
||||
DNSSEC=yes
|
||||
DNSStubListener=no
|
||||
@@ -0,0 +1,59 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/control-plane/tasks/main.yml
|
||||
# ------------------------------------------------------------------------------
|
||||
- name: Check if cluster is already initialized
|
||||
ansible.builtin.stat:
|
||||
path: /etc/kubernetes/admin.conf
|
||||
register: kube_init_stat
|
||||
|
||||
- name: Initialize the Kubernetes cluster
|
||||
when: not kube_init_stat.stat.exists
|
||||
ansible.builtin.command: >
|
||||
kubeadm init
|
||||
--pod-network-cidr=10.244.0.0/16
|
||||
--control-plane-endpoint={{ inventory_hostname }}
|
||||
register: kubeadm_init
|
||||
|
||||
- name: Create .kube directory for the user
|
||||
become: false
|
||||
ansible.builtin.file:
|
||||
path: "{{ ansible_env.HOME }}/.kube"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
|
||||
- name: Copy admin.conf to user's .kube directory on controller
|
||||
when: not kube_init_stat.stat.exists
|
||||
ansible.builtin.fetch:
|
||||
src: /etc/kubernetes/admin.conf
|
||||
dest: "{{ ansible_env.HOME }}/.kube/config"
|
||||
flat: true
|
||||
|
||||
- name: Install Calico CNI
|
||||
become: false
|
||||
ansible.builtin.command: kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
when: not kube_init_stat.stat.exists
|
||||
|
||||
- name: Generate join command for control plane nodes
|
||||
when: not kube_init_stat.stat.exists
|
||||
ansible.builtin.command: kubeadm token create --print-join-command
|
||||
register: control_plane_join_command_raw
|
||||
|
||||
- name: Generate certificate key for control plane join
|
||||
when: not kube_init_stat.stat.exists
|
||||
ansible.builtin.command: kubeadm init phase upload-certs --upload-certs
|
||||
register: control_plane_cert_key
|
||||
|
||||
- name: Store control plane join command
|
||||
ansible.builtin.set_fact:
|
||||
control_plane_join_command: "{{ control_plane_join_command_raw.stdout }} --control-plane --certificate-key {{ control_plane_cert_key.stdout_lines[-1] }}"
|
||||
when: not kube_init_stat.stat.exists
|
||||
|
||||
- name: Join other control plane nodes to the cluster
|
||||
when:
|
||||
- inventory_hostname != groups['fastpass_control_plane'][0]
|
||||
- not kube_init_stat.stat.exists
|
||||
ansible.builtin.command: "{{ hostvars[groups['fastpass_control_plane'][0]].control_plane_join_command }}"
|
||||
35
ansible/playbooks/roles/fastpass-workers/tasks/main.yml
Normal file
35
ansible/playbooks/roles/fastpass-workers/tasks/main.yml
Normal file
@@ -0,0 +1,35 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/fastpass_workers/tasks/main.yml
|
||||
# ------------------------------------------------------------------------------
|
||||
- name: Generate join command for worker nodes
|
||||
ansible.builtin.command: kubeadm token create --print-join-command
|
||||
delegate_to: "{{ groups['fastpass_control_plane'][0] }}"
|
||||
run_once: true
|
||||
register: worker_join_command
|
||||
|
||||
- name: Join worker nodes to the cluster
|
||||
ansible.builtin.command: "{{ hostvars[groups['fastpass_control_plane'][0]].worker_join_command.stdout }}"
|
||||
|
||||
- name: Label and Taint on-stage worker nodes
|
||||
become: false
|
||||
ansible.builtin.command: "kubectl {{ item }}"
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
loop:
|
||||
- label node seven-dwarfs-mine-train zone=on-stage --overwrite
|
||||
- taint node seven-dwarfs-mine-train dedicated=external:NoSchedule --overwrite
|
||||
when: "'seven-dwarfs-mine-train' in inventory_hostname"
|
||||
|
||||
- name: Label backstage worker node haunted-mansion
|
||||
become: false
|
||||
ansible.builtin.command: "kubectl label node haunted-mansion zone=backstage --overwrite"
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
when: "'haunted-mansion' in inventory_hostname"
|
||||
|
||||
- name: Label backstage worker node peter-pans-flight
|
||||
become: false
|
||||
ansible.builtin.command: "kubectl label node peter-pans-flight zone=backstage --overwrite"
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
when: "'peter-pans-flight' in inventory_hostname"
|
||||
101
ansible/playbooks/roles/kubernetes/tasks/main.yml
Normal file
101
ansible/playbooks/roles/kubernetes/tasks/main.yml
Normal file
@@ -0,0 +1,101 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/common/tasks/main.yml
|
||||
# ------------------------------------------------------------------------------
|
||||
- name: Disable swap
|
||||
ansible.builtin.command: swapoff -a
|
||||
changed_when: false
|
||||
|
||||
- name: Persist swap off by commenting out swap in fstab
|
||||
ansible.builtin.replace:
|
||||
path: /etc/fstab
|
||||
regexp: '^(\s*)([^#\n]+\s+swap\s+.*)$'
|
||||
replace: '#\2'
|
||||
backup: true
|
||||
|
||||
- name: Load required kernel modules
|
||||
community.general.modprobe:
|
||||
name: "{{ item }}"
|
||||
state: present
|
||||
loop:
|
||||
- overlay
|
||||
- br_netfilter
|
||||
|
||||
- name: Persist kernel modules on boot
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/modules-load.d/k8s.conf
|
||||
content: |
|
||||
overlay
|
||||
br_netfilter
|
||||
|
||||
- name: Configure required sysctl params for Kubernetes
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/sysctl.d/k8s.conf
|
||||
content: |
|
||||
net.bridge.bridge-nf-call-iptables = 1
|
||||
net.bridge.bridge-nf-call-ip6tables = 1
|
||||
net.ipv4.ip_forward = 1
|
||||
|
||||
- name: Apply sysctl params without reboot
|
||||
ansible.builtin.command: sysctl --system
|
||||
changed_when: false
|
||||
|
||||
- name: Set SELinux to permissive mode
|
||||
selinux:
|
||||
policy: targeted
|
||||
state: permissive
|
||||
|
||||
- name: Install DNF plugins core for managing repositories
|
||||
ansible.builtin.dnf:
|
||||
name: dnf-plugins-core
|
||||
state: present
|
||||
|
||||
- name: Add Docker CE repository
|
||||
ansible.builtin.command: dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
|
||||
args:
|
||||
creates: /etc/yum.repos.d/docker-ce.repo
|
||||
|
||||
- name: Install containerd
|
||||
ansible.builtin.dnf:
|
||||
name: containerd.io
|
||||
state: present
|
||||
|
||||
- name: Create containerd config directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/containerd
|
||||
state: directory
|
||||
|
||||
- name: Generate default containerd config and enable SystemdCgroup
|
||||
ansible.builtin.shell: |
|
||||
containerd config default > /etc/containerd/config.toml
|
||||
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
|
||||
args:
|
||||
creates: /etc/containerd/config.toml
|
||||
|
||||
- name: Restart and enable containerd service
|
||||
ansible.builtin.systemd:
|
||||
name: containerd
|
||||
state: restarted
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
|
||||
- name: Add Kubernetes repository
|
||||
ansible.builtin.yum_repository:
|
||||
name: kubernetes
|
||||
description: Kubernetes
|
||||
baseurl: https://pkgs.k8s.io/core:/stable:/v1.33/rpm/
|
||||
gpgkey: https://pkgs.k8s.io/core:/stable:/v1.33/rpm/repodata/repomd.xml.key
|
||||
gpgcheck: true
|
||||
|
||||
- name: Install Kubernetes packages
|
||||
ansible.builtin.dnf:
|
||||
name:
|
||||
- kubelet
|
||||
- kubeadm
|
||||
- kubectl
|
||||
state: present
|
||||
disable_excludes: kubernetes
|
||||
|
||||
- name: Enable the kubelet service
|
||||
ansible.builtin.systemd:
|
||||
name: kubelet
|
||||
enabled: true
|
||||
11
ansible/playbooks/roles/observer/defaults/main.yml
Normal file
11
ansible/playbooks/roles/observer/defaults/main.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
---
|
||||
prometheus_version: v2.40.1
|
||||
grafana_version: "9.2.5"
|
||||
alertmanager_version: v0.24.0
|
||||
#alertmanager_smtp_password: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
64306663363562356132323065396635636630373031303739323666373262663961393132316333
|
||||
6135653763363566303331313639633030623530646239310a353236343035643132646230333466
|
||||
36336439376131333630346563323833313164353265313264643232373465633561663331396133
|
||||
3163303166373166390a396131303239356139653063616437363933333130393563646338663933
|
||||
3966
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: Pre-loaded local dashboards
|
||||
type: file
|
||||
options:
|
||||
foldersFromFilesStructure: true
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,814 @@
|
||||
{
|
||||
"__inputs": [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "prometheus",
|
||||
"description": "Prometheus as the datasource is obligatory",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus"
|
||||
}
|
||||
],
|
||||
"__requires": [
|
||||
{
|
||||
"type": "grafana",
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"version": "7.4.5"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "graph",
|
||||
"name": "Graph",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "table",
|
||||
"name": "Table",
|
||||
"version": ""
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": 14282,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"iteration": 1617715580880,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 8,
|
||||
"panels": [],
|
||||
"title": "CPU",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 15,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name) *100",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "CPU Usage",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "percent",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 11,
|
||||
"panels": [],
|
||||
"title": "Memory",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 9,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(container_memory_rss{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Memory Usage",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 14,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(container_memory_cache{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Memory Cached",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 17
|
||||
},
|
||||
"id": 2,
|
||||
"panels": [],
|
||||
"title": "Network",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 4,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"hideEmpty": false,
|
||||
"hideZero": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"sideWidth": null,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Received Network Traffic",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:674",
|
||||
"format": "Bps",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:675",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 6,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Sent Network Traffic",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:832",
|
||||
"format": "Bps",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:833",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "Prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 26
|
||||
},
|
||||
"id": 19,
|
||||
"panels": [],
|
||||
"title": "Misc",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": null,
|
||||
"filterable": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "id"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 260
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Running"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "d"
|
||||
},
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"id": "custom.displayMode",
|
||||
"value": "color-text"
|
||||
},
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "dark-green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
},
|
||||
"id": 17,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": []
|
||||
},
|
||||
"pluginVersion": "7.4.5",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(time() - container_start_time_seconds{instance=~\"$host\",name=~\"$container\",name=~\".+\"})/86400",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Containers Info",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": [
|
||||
"container_label_com_docker_compose_project",
|
||||
"container_label_com_docker_compose_project_working_dir",
|
||||
"image",
|
||||
"instance",
|
||||
"name",
|
||||
"Value",
|
||||
"container_label_com_docker_compose_service"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {},
|
||||
"indexByName": {},
|
||||
"renameByName": {
|
||||
"Value": "Running",
|
||||
"container_label_com_docker_compose_project": "Label",
|
||||
"container_label_com_docker_compose_project_working_dir": "Working dir",
|
||||
"container_label_com_docker_compose_service": "Service",
|
||||
"image": "Registry Image",
|
||||
"instance": "Instance",
|
||||
"name": "Name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 27,
|
||||
"style": "dark",
|
||||
"tags": ["cadvisor", "docker"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": "Prometheus",
|
||||
"definition": "label_values({__name__=~\"container.*\"},instance)",
|
||||
"description": null,
|
||||
"error": null,
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Host",
|
||||
"multi": false,
|
||||
"name": "host",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values({__name__=~\"container.*\"},instance)",
|
||||
"refId": "Prometheus-host-Variable-Query"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 5,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": "Prometheus",
|
||||
"definition": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
|
||||
"description": null,
|
||||
"error": null,
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Container",
|
||||
"multi": false,
|
||||
"name": "container",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
|
||||
"refId": "Prometheus-container-Variable-Query"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Cadvisor exporter",
|
||||
"uid": "pMEd7m0Mz",
|
||||
"version": 1,
|
||||
"description": "Simple exporter for cadvisor only"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://192.168.0.1:9090
|
||||
@@ -0,0 +1,11 @@
|
||||
groups:
|
||||
- name: AllInstances
|
||||
rules:
|
||||
- alert: InstanceDown
|
||||
expr: up == 0
|
||||
for: 1m
|
||||
annotations:
|
||||
title: 'Instance {{ $labels.instance }} down'
|
||||
description: '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minute.'
|
||||
labels:
|
||||
severity: 'critical'
|
||||
28
ansible/playbooks/roles/observer/files/prometheus_main.yml
Normal file
28
ansible/playbooks/roles/observer/files/prometheus_main.yml
Normal file
@@ -0,0 +1,28 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
|
||||
- job_name: node-exporter
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
["192.168.0.1:9100", "192.168.0.10:9100", "192.168.0.11:9100"]
|
||||
|
||||
- job_name: cadvisor
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets: ["192.168.0.1:9101", "192.168.0.11:9101"]
|
||||
|
||||
rule_files:
|
||||
- prometheus_alerts_rules.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- 192.168.0.1:9093
|
||||
85
ansible/playbooks/roles/observer/tasks/main.yml
Normal file
85
ansible/playbooks/roles/observer/tasks/main.yml
Normal file
@@ -0,0 +1,85 @@
|
||||
- name: Create Folder /srv/prometheus if not exist
|
||||
file:
|
||||
path: /srv/prometheus
|
||||
mode: 0755
|
||||
state: directory
|
||||
|
||||
- name: Create Folder /srv/grafana if not exist
|
||||
file:
|
||||
path: /srv/grafana
|
||||
mode: 0755
|
||||
state: directory
|
||||
|
||||
- name: Create Folder /srv/alertmanager if not exist
|
||||
file:
|
||||
path: /srv/alertmanager
|
||||
mode: 0755
|
||||
state: directory
|
||||
|
||||
- name: Create prometheus configuration file
|
||||
copy:
|
||||
dest: /srv/prometheus/prometheus.yml
|
||||
src: prometheus_main.yml
|
||||
mode: 0644
|
||||
|
||||
- name: Create prometheus alert configuration file
|
||||
copy:
|
||||
dest: /srv/prometheus/prometheus_alerts_rules.yml
|
||||
src: prometheus_alerts_rules.yml
|
||||
mode: 0644
|
||||
|
||||
- name: Create grafana configuration files
|
||||
copy:
|
||||
dest: /srv/
|
||||
src: grafana
|
||||
mode: 0644
|
||||
|
||||
- name: Create alertmanager configuration file
|
||||
template:
|
||||
dest: /srv/alertmanager/alertmanager.yml
|
||||
src: alertmanager/alertmanager.j2
|
||||
mode: 0644
|
||||
|
||||
- name: Create Prometheus container
|
||||
docker_container:
|
||||
name: prometheus
|
||||
restart_policy: always
|
||||
image: prom/prometheus:
|
||||
volumes:
|
||||
- /srv/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
- /srv/prometheus/prometheus_alerts_rules.yml:/etc/prometheus/prometheus_alerts_rules.yml
|
||||
- prometheus_main_data:/prometheus
|
||||
command: >
|
||||
--config.file=/etc/prometheus/prometheus.yml
|
||||
--storage.tsdb.path=/prometheus
|
||||
--web.console.libraries=/etc/prometheus/console_libraries
|
||||
--web.console.templates=/etc/prometheus/consoles
|
||||
--web.enable-lifecycle
|
||||
published_ports: "9090:9090"
|
||||
|
||||
- name: Create Grafana container
|
||||
docker_container:
|
||||
name: grafana
|
||||
restart_policy: always
|
||||
image: grafana/grafana:
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- /srv/grafana/provisioning:/etc/grafana/provisioning
|
||||
- /srv/grafana/dashboards:/var/lib/grafana/dashboards
|
||||
env:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
|
||||
published_ports: "3000:3000"
|
||||
|
||||
- name: Create Alertmanager container
|
||||
docker_container:
|
||||
name: alertmanager
|
||||
restart_policy: always
|
||||
image: prom/alertmanager:
|
||||
volumes:
|
||||
- alertmanager-data:/data
|
||||
- /srv/alertmanager:/config
|
||||
command: >
|
||||
--config.file=/config/alertmanager.yml
|
||||
--log.level=debug
|
||||
published_ports: "9093:9093"
|
||||
@@ -0,0 +1,13 @@
|
||||
route:
|
||||
receiver: "mail"
|
||||
repeat_interval: 4h
|
||||
group_by: [ alertname ]
|
||||
|
||||
receivers:
|
||||
- name: "mail"
|
||||
email_configs:
|
||||
- smarthost: "outlook.office365.com:587"
|
||||
auth_username: "test@padok.fr"
|
||||
auth_password: "{{ alertmanager_smtp_password }}"
|
||||
from: "test@padok.fr"
|
||||
to: "test@padok.fr"
|
||||
13
ansible/playbooks/roles/requirements.yml
Normal file
13
ansible/playbooks/roles/requirements.yml
Normal file
@@ -0,0 +1,13 @@
|
||||
# requirements.yml
|
||||
collections:
|
||||
- name: community.general
|
||||
version: 11.1.0
|
||||
|
||||
- name: nccurry.openshift
|
||||
version: 1.4.0
|
||||
|
||||
- name: somaz94.ansible_k8s_iac_tool
|
||||
version: 1.1.6
|
||||
|
||||
- name: prometheus.prometheus
|
||||
version: 0.27.0
|
||||
3
ansible/playbooks/roles/target/defaults/main.yml
Normal file
3
ansible/playbooks/roles/target/defaults/main.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
---
|
||||
node_exporter_version: v1.4.0
|
||||
cadvisor_version: v0.46.0
|
||||
29
ansible/playbooks/roles/target/tasks/main.yml
Normal file
29
ansible/playbooks/roles/target/tasks/main.yml
Normal file
@@ -0,0 +1,29 @@
|
||||
---
|
||||
- name: Create NodeExporter
|
||||
docker_container:
|
||||
name: node-exporter
|
||||
restart_policy: always
|
||||
image: prom/node-exporter:{{ node_exporter_version }}
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command: >
|
||||
--path.procfs=/host/proc
|
||||
--path.rootfs=/rootfs
|
||||
--path.sysfs=/host/sys
|
||||
--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)
|
||||
published_ports: "9100:9100"
|
||||
|
||||
- name: Create cAdvisor
|
||||
docker_container:
|
||||
name: cadvisor
|
||||
restart_policy: always
|
||||
image: gcr.io/cadvisor/cadvisor:{{ cadvisor_version }}
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
published_ports: "9101:8080"
|
||||
Reference in New Issue
Block a user