Add observability stack

Signed-off-by: Ava Affine <ava@sunnypup.io>

templates/monitoring.yaml

@@ -0,0 +1,269 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: observability
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: observability
+  template:
+    metadata:
+      labels:
+        app: observability
+    spec:
+      containers:
+        - name: otel-lgtm
+          image: grafana/otel-lgtm
+          ports:
+            - name: web-interface
+              containerPort: 3000
+            - name: otel-http
+              containerPort: 4318
+            - name: otel-grpc
+              containerPort: 4317
+          env:
+            - name: ENABLE_LOGS_GRAFANA
+              value: "true"
+            - name: GF_AUTH_ANONYMOUS_ENABLED
+              value: "false"
+            - name: GF_AUTH_PASSWORDLESS_ENABLED
+              value: "false"
+            - name: GF_DATABASE_TYPE
+              value: "postgres"
+            - name: GF_DATABASE_USER
+              value: {{ .Values.pg.user }}
+            - name: GF_DATABASE_PASSWORD
+              value: {{ .Values.pg.pass }}
+            - name: GF_DATABASE_HOST
+              value: "postgres:5432"
+            - name: GF_DATABASE_INSTRUMENT_QUERIES
+              value: "true"
+          volumeMounts:
+            - name: tempo-data
+              mountPath: /data/tempo
+            - name: grafana-data
+              mountPath: /data/grafana
+            - name: loki-data
+              mountPath: /data/loki
+            - name: loki-storage
+              mountPath: /loki
+            - name: p8s-storage
+              mountPath: /data/prometheus
+            - name: pyroscope-storage
+              mountPath: /data/pyroscope
+      volumes:
+        - name: tempo-data
+          persistentVolumeClaim:
+            claimName: tempo-pvc
+        - name: loki-data
+          persistentVolumeClaim:
+            claimName: loki-data-pvc
+        - name: grafana-data
+          persistentVolumeClaim:
+            claimName: grafana-pvc
+        - name: loki-storage
+          persistentVolumeClaim:
+            claimName: loki-storage-pvc
+        - name: p8s-storage
+          persistentVolumeClaim:
+            claimName: p8s-pvc
+        - name: pyroscope-storage
+          persistentVolumeClaim:
+            claimName: pyroscope-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: observability
+spec:
+  selector:
+    app: observability
+  ports:
+    - name: grafana
+      protocol: TCP
+      port: 3000
+      targetPort: 3000
+    - name: otel-grpc
+      protocol: TCP
+      port: 4317
+      targetPort: 4317
+    - name: otel-http
+      protocol: TCP
+      port: 4318
+      targetPort: 4318
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+   name: tempo-pv
+   labels:
+     pvc_type: tempo-pv
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+   - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+   path: {{ .Values.observability.tempo }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: tempo-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  storageClassName: ""
+  volumeName: tempo-pv
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+   name: loki-data-pv
+   labels:
+     pvc_type: loki-data-pv
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+   - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+   path: {{ .Values.observability.loki.data_path }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: loki-data-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  storageClassName: ""
+  volumeName: loki-data-pv
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+   name: loki-storage-pv
+   labels:
+     pvc_type: loki-storage-pv
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+   - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+   path: {{ .Values.observability.loki.storage_path }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: loki-storage-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  storageClassName: ""
+  volumeName: loki-storage-pv
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+   name: grafana-pv
+   labels:
+     pvc_type: grafana-pv
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+   - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+   path: {{ .Values.observability.grafana }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  storageClassName: ""
+  volumeName: grafana-pv
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+   name: p8s-pv
+   labels:
+     pvc_type: p8s-pv
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+   - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+   path: {{ .Values.observability.p8s }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: p8s-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  storageClassName: ""
+  volumeName: p8s-pv
+  resources:
+    requests:
+      storage: 50Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+   name: pyroscope-pv
+   labels:
+     pvc_type: pyroscope-pv
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+   - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+   path: {{ .Values.observability.pyroscope }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pyroscope-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  storageClassName: ""
+  volumeName: pyroscope-pv
+  resources:
+    requests:
+      storage: 50Gi
+

templates/routing.yaml

@@ -1,6 +1,11 @@
-#dop_v1_8b514aa82e4930e58f8098a54088a36c01af2fb6020b792f7a7fe4be694ddc52
+kind: ConfigMap
-# BIG NOTE
+apiVersion: v1
-# CAMS AND FRIGATE REMAIN UNEXPOSED TO WEB
+metadata:
+  name: nginx-ingress
+data:
+  otel-exporter-endpoint: "observability:4317"
+  otel-trace-in-http: "true"
+---
 apiVersion: v1
 kind: Secret
 metadata:
@@ -41,6 +46,7 @@ spec:
   - echo.sunnypup.io
   - home.sunnypup.io
   - hephaestus.sunnypup.io
+  - mimir.sunnypup.io
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
@@ -56,6 +62,29 @@ spec:
 ---
 apiVersion: k8s.nginx.org/v1
 kind: VirtualServer
+metadata:
+  name: observability
+  labels:
+    acme.cert-manager.io/http01-solver: "true"
+spec:
+  host: mimir.sunnypup.io
+  tls:
+    cert-manager:
+      cluster-issuer: letsencrypt
+    secret: sunnypup-certs
+    redirect:
+      enable: true
+  upstreams:
+    - name: observability
+      service: observability
+      port: 3000
+  routes:
+    - path: /
+      action:
+        pass: observability
+---
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
 metadata:
   name: forge
   labels:
@@ -222,3 +251,4 @@ spec:
       port: 22222
   action:
     pass: forge
+

values.yaml

@@ -68,3 +68,12 @@ mqtt:
 forge:
   path: "/srv/sunnypup/forge"
   secret: ""
+
+observability:
+  tempo: "/srv/sunnypup/otel/tempo"
+  loki:
+    data_path: "/srv/sunnypup/otel/loki/data"
+    storage_path: "/srv/sunnypup/otel/loki/storage"
+  grafana: "/srv/sunnypup/otel/grafana"
+  p8s: "/srv/sunnypup/otel/p8s"
+  pyroscope: "/srv/sunnypup/otel/pyroscope"