feat(observability): OneLab-only Promtail, provisioned OneLab logs dashboard
- Promtail: keep kubernetes-pods in namespace onelab; tag host file logs (host-logs) - Grafana: enable dashboard sidecar; ConfigMap onelab-logs.json - Dashboard: stats (total/error/warn heuristics), logs panel, component + regex filters Made-with: Cursor
This commit is contained in:
timotheereausanofi
205
gitops/observability/dashboards/onelab-logs.json
Normal file
205
gitops/observability/dashboards/onelab-logs.json
Normal file
@@ -0,0 +1,205 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {"type": "grafana", "uid": "-- Grafana --"},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{"color": "blue", "value": null}]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 5, "w": 8, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(count_over_time({namespace=\"onelab\", component=~\"$component\"} |~ \"$filter\" [$__range]))",
|
||||
"queryType": "instant",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Total lines (namespace onelab, matches line filter)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "orange", "value": 1}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 5, "w": 8, "x": 8, "y": 0},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(count_over_time({namespace=\"onelab\", component=~\"$component\"} |~ \"$filter\" |~ \"(?i)(\\\\[ERROR\\\\]|\\\\berror\\\\b|\\\\sERROR\\\\s)\" [$__range]))",
|
||||
"queryType": "instant",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "~ Error-like lines (heuristic)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {"mode": "thresholds"},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 1}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {"h": 5, "w": 8, "x": 16, "y": 0},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(count_over_time({namespace=\"onelab\", component=~\"$component\"} |~ \"$filter\" |~ \"(?i)(\\\\[WARN\\\\]|\\\\bwarn(ing)?\\\\b|\\\\sWARN\\\\s)\" [$__range]))",
|
||||
"queryType": "instant",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "~ Warning-like lines (heuristic)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"gridPos": {"h": 16, "w": 24, "x": 0, "y": 5},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": true,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"editorMode": "code",
|
||||
"expr": "{namespace=\"onelab\", component=~\"$component\"} |~ \"$filter\"",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "OneLab logs — use Component + Line filter (regex)",
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["onelab", "loki"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {"selected": true, "text": "All", "value": "$__all"},
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"definition": "label_values({namespace=\"onelab\"}, component)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Component",
|
||||
"multi": true,
|
||||
"name": "component",
|
||||
"options": [],
|
||||
"query": "label_values({namespace=\"onelab\"}, component)",
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"current": {"selected": true, "text": ".*", "value": ".*"},
|
||||
"hide": 0,
|
||||
"label": "Line filter (regex)",
|
||||
"name": "filter",
|
||||
"options": [
|
||||
{"selected": true, "text": ".*", "value": ".*"}
|
||||
],
|
||||
"query": ".*",
|
||||
"type": "textbox"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "OneLab logs",
|
||||
"uid": "onelab-logs",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{{- if .Values.grafana.sidecar.dashboards.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ printf "%s-dashboard-onelab-logs" .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
data:
|
||||
onelab-logs.json: |-
|
||||
{{ .Files.Get "dashboards/onelab-logs.json" | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -69,6 +69,12 @@ promtail:
|
||||
clients:
|
||||
- url: http://{{ .Release.Name }}-loki-gateway.{{ .Release.Namespace }}.svc.cluster.local/loki/api/v1/push
|
||||
snippets:
|
||||
# Only ingest pod logs from namespace onelab (Explore / Loki stay focused on OneLab).
|
||||
extraRelabelConfigs:
|
||||
- action: keep
|
||||
source_labels:
|
||||
- __meta_kubernetes_namespace
|
||||
regex: onelab
|
||||
extraScrapeConfigs: |
|
||||
- job_name: onelab-host-log-files
|
||||
static_configs:
|
||||
@@ -76,6 +82,8 @@ promtail:
|
||||
- localhost
|
||||
labels:
|
||||
job: onelab-files
|
||||
namespace: onelab
|
||||
component: host-logs
|
||||
__path__: /onelab-host-logs/**/*
|
||||
extraVolumes:
|
||||
- name: onelab-host-logs
|
||||
@@ -101,6 +109,17 @@ grafanaOnelabIngress:
|
||||
grafana:
|
||||
adminUser: admin
|
||||
adminPassword: changeme
|
||||
# Load dashboards from ConfigMaps labeled grafana_dashboard (see templates/configmap-dashboard-onelab-logs.yaml).
|
||||
sidecar:
|
||||
dashboards:
|
||||
enabled: true
|
||||
label: grafana_dashboard
|
||||
folder: /tmp/dashboards
|
||||
provider:
|
||||
foldersFromFilesStructure: false
|
||||
allowUiUpdates: true
|
||||
datasources:
|
||||
enabled: false
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 2Gi
|
||||
|
||||
Reference in New Issue
Block a user