Improve Loki (#305)

* Add names to docker projects

* Add build log to loki

* Fix collector

* Fix script

* Fix build log

* Rename grafana folder

* Fix grafana

* Fix script better

* Improve script

* Decrease loki logging

* Really decrease loki logs

* turn off grafana access logs

* fix build log collection

Author: mfosterw

…pose/production/grafana/datasources.yaml …se/production/telemetry/datasources.yamlcompose/production/grafana/datasources.yaml renamed to compose/production/telemetry/datasources.yaml compose/production/grafana/datasources.yaml renamed to compose/production/telemetry/datasources.yaml

@@ -2,6 +2,7 @@ auth_enabled: false
 
 server:
   http_listen_port: 3100
+  log_level: "warn"
 
 common:
   instance_addr: 127.0.0.1

compose/production/grafana/loki.yaml compose/production/telemetry/loki.yamlcompose/production/grafana/loki.yaml renamed to compose/production/telemetry/loki.yaml

@@ -4,7 +4,18 @@ receivers:
       grpc:
         endpoint: 0.0.0.0:4317
 
-  filelog:
+  filelog/host:
+    include: [/var/log/build.log]
+    poll_interval: 200ms
+    start_at: end
+    include_file_name: false
+    include_file_path: false
+    operators:
+      - type: add
+        field: resource["service.name"]
+        value: "host"
+
+  filelog/docker:
     include: [/var/lib/docker/containers/*/*-json.log]
     poll_interval: 200ms
     start_at: beginning
@@ -23,7 +34,7 @@ receivers:
         if: "attributes?.attrs?.tag != nil"
       - id: extract_service_from_docker_tag
         parse_from: attributes.name
-        regex: ^democrasite-(?<service>[^-]+)-(?<instance>\d+)$
+        regex: ^(democrasite|telemetry)-(?<service>[^-]+)-(?<instance>\d+)$
         type: regex_parser
         if: "attributes?.name != nil"
       - from: attributes.service
@@ -91,7 +102,7 @@ exporters:
 service:
   pipelines:
     logs:
-      receivers: [otlp, filelog]
+      receivers: [otlp, filelog/host, filelog/docker]
       processors: [batch]
       exporters: [otlp_http/loki]
 

…e/production/grafana/otelcol-config.yaml …production/telemetry/otelcol-config.yamlcompose/production/grafana/otelcol-config.yaml renamed to compose/production/telemetry/otelcol-config.yaml compose/production/grafana/otelcol-config.yaml renamed to compose/production/telemetry/otelcol-config.yaml

@@ -67,6 +67,8 @@ http:
       service: telemetry
       tls:
         certResolver: letsencrypt
+      observability:
+        accessLogs: false
 
   middlewares:
     csrf:

compose/production/grafana/prometheus.yml …pose/production/telemetry/prometheus.ymlcompose/production/grafana/prometheus.yml renamed to compose/production/telemetry/prometheus.yml compose/production/grafana/prometheus.yml renamed to compose/production/telemetry/prometheus.yml

@@ -1,3 +1,5 @@
+name: democrasite
+
 volumes:
   production_postgres_data:
   production_postgres_data_backups:

compose/production/grafana/tempo.yaml compose/production/telemetry/tempo.yamlcompose/production/grafana/tempo.yaml renamed to compose/production/telemetry/tempo.yaml

@@ -1,3 +1,5 @@
+name: telemetry
+
 volumes:
   production_prometheus_data:
   production_loki_data:
@@ -23,7 +25,7 @@ services:
     command:
       - --web.enable-otlp-receiver
     volumes:
-      - ./compose/production/grafana/prometheus.yml:/prometheus/prometheus.yml
+      - ./compose/production/telemetry/prometheus.yml:/prometheus/prometheus.yml
       - production_prometheus_data:/prometheus
     networks:
       - democrasite_metrics
@@ -32,17 +34,18 @@ services:
   loki:
     image: grafana/loki:3.6.4
     volumes:
-      - ./compose/production/grafana/loki.yaml:/etc/loki/loki.yaml
+      - ./compose/production/telemetry/loki.yaml:/etc/loki/loki.yaml
       - production_loki_data:/loki
     networks:
       - democrasite_metrics
+    command: ["--log.level=warn", "--config.file=/etc/loki/loki.yaml"]
     logging: *logging
 
   tempo:
     image: grafana/tempo:2.10.0
     command: -config.file=/etc/tempo/tempo.yaml
     volumes:
-      - ./compose/production/grafana/tempo.yaml:/etc/tempo/tempo.yaml
+      - ./compose/production/telemetry/tempo.yaml:/etc/tempo/tempo.yaml
       - production_tempo_data:/var/tempo
     networks:
       - democrasite_metrics
@@ -54,8 +57,9 @@ services:
     group_add:
       - "999" # to access docker socket
     volumes:
-      - ./compose/production/grafana/otelcol-config.yaml:/etc/otelcol-contrib/config.yaml
+      - ./compose/production/telemetry/otelcol-config.yaml:/etc/otelcol-contrib/config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log/build.log:/var/log/build.log
       - /var/run/docker.sock:/var/run/docker.sock
       - /:/hostfs:ro
     networks:
@@ -68,7 +72,7 @@ services:
   grafana:
     image: grafana/grafana:12.3.2
     volumes:
-      - ./compose/production/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - ./compose/production/telemetry/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
       - production_grafana_data:/var/lib/grafana
     env_file:
       - ./.envs/.production/.grafana

compose/production/traefik/traefik.yml

@@ -3,20 +3,20 @@
 # Scripts adapted from https://www.youtube.com/watch?v=jFrGhodqC08 by Tom Delande
 # Licensed under https://creativecommons.org/licenses/by/4.0/
 
-echo "$(date --utc +%FT%TZ): Fetching remote repository..."
 git fetch
 
 UPSTREAM=${1:-'@{u}'}
 LOCAL=$(git rev-parse @)
 REMOTE=$(git rev-parse "$UPSTREAM")
 BASE=$(git merge-base @ "$UPSTREAM")
 
-if [ $LOCAL = $REMOTE ]; then
-    echo "$(date --utc +%FT%TZ): No changes detected in git"
-elif [ $LOCAL = $BASE ]; then
+if [ "$LOCAL" = "$REMOTE" ]; then
+    true
+    # echo "$(date --utc +%FT%TZ): No changes detected in git"
+elif [ "$LOCAL" = "$BASE" ]; then
     git pull
     scripts/deploy.sh
-elif [ $REMOTE = $BASE ]; then
+elif [ "$REMOTE" = "$BASE" ]; then
      echo "$(date --utc +%FT%TZ): Local changes detected, you may need to stash"
 else
      echo "$(date --utc +%FT%TZ): Git is diverged, this is unexpected."

docker-compose.production.yml

@@ -4,4 +4,4 @@
 # Licensed under https://creativecommons.org/licenses/by/4.0/
 
 cd /home/matthew/democrasite
-flock -n ~/deploy.lock scripts/conditional-deploy.sh >> ~/build.log 2>&1
+flock -n ~/deploy.lock scripts/conditional-deploy.sh >> /var/log/build.log 2>&1