switching out promtail for alloy (#22)

* rework grafana log ingestion using alloy

* fix docker-compose.yml formatting

.gitignore

@@ -1,5 +1,4 @@
-license.mattermost
+license*
-license.txt
 
 ## Ignoring keycloak so it can be pulled already setup.
 volumes
@@ -13,3 +12,8 @@ files/postgres/replica/replica_1_postgres.conf
 
 files/postgres/replica/replica_2.conf
 files/postgres/replica/replica_2_postgres.conf
+
+**/.claude/settings.local.json
+
+*_backup.json
+*.log

CLAUDE.md

@@ -0,0 +1,92 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+This repository contains a CS Repro Mattermost environment - a reproduction of a standard Mattermost customer production environment with preconfigured SAML, LDAP, Advanced Logging, Prometheus, Grafana, Elasticsearch, and read replicas. It is designed for testing and reproducing customer environments.
+
+## Environment Setup
+
+### Prerequisites
+
+You need to have Docker and Docker Compose installed on your machine to run this environment.
+
+### Initial Setup
+
+1. Add an enterprise license file named `license.mattermost` to the root directory
+2. Run `make run` to start the core services
+
+## Common Commands
+
+### Environment Management
+
+- `make run` - Initialize environment and create containers
+- `make run-all` - Spin up all environment containers (with DB replicas and Mattermost HA)
+- `make run-db-replicas` - Add database replicas to the environment
+- `make run-mm-replicas` - Add Mattermost HA nodes to the environment
+- `make run-rtcd` - Launch RTCD service for Mattermost Calls
+- `make start` - Start existing containers
+- `make stop` - Stop all running containers
+- `make restart` - Restart all containers
+- `make restart-mattermost` - Restart only Mattermost containers (preferred for testing changes)
+- `make backup-keycloak` - Generate a backup of the current Keycloak setup
+- `make restore-keycloak` - Restore Keycloak data from an existing backup
+- `make reset` - Clean volumes directory and reinitialize the environment
+- `make delete-data` - Clear all data within volumes
+- `make nuke` - Erase all configurations and data
+- `make nuke-rmi` - Complete cleanup including removing Docker images
+
+### Upgrading and Downgrading
+
+To upgrade Mattermost:
+1. Modify the image tag in `docker-compose.yml`
+2. Run `make restart-mattermost`
+
+To downgrade Mattermost:
+1. Modify the image tag in `docker-compose.yml`
+2. Run `make downgrade`
+
+## Component Access
+
+### Mattermost
+
+- URL: http://localhost:8065
+- Default credentials available in the accounts section below
+
+### LDAP
+
+- Admin access: `cn=admin,dc=planetexpress,dc=com` / `GoodNewsEveryone`
+- Commands for LDAP operations are in the README.md
+
+### Grafana
+
+- URL: http://localhost:3000
+- Credentials: admin / admin
+
+### Elasticsearch
+
+- URL: http://localhost:9200
+
+### mitmproxy
+
+- URL: http://localhost:8181
+
+## Account Information
+
+| Username  | Password  | Role in Mattermost | Can use LDAP? | Can use SAML? |
+|-----------|-----------|-------------------|---------------|---------------|
+| admin     | admin     | n/a               | n/a           | n/a           |
+| professor | professor | Sys Admin         | Yes           | Yes           |
+| bender    | bender    | Member            | Yes           | Yes           |
+| hermes    | hermes    | Sys Admin         | Yes           | Yes           |
+| fry       | fry       | Member            | Yes           | Yes           |
+| leela     | leela     | Member            | Yes           | Yes           |
+| zoidberg  | zoidberg  | Member            | Yes           | Yes           |
+| amy       | amy       | Member            | Yes           | Yes           |
+
+## Database Information
+
+- Primary PostgreSQL: `postgresql://mmuser:mmuser_password@localhost:5432/mattermost`
+- Replica 1: `postgresql://mmuser:mmuser_password@localhost:5433/mattermost`
+- Replica 2: `postgresql://mmuser:mmuser_password@localhost:5434/mattermost`

Makefile

@@ -6,6 +6,12 @@ logs:
 	@echo "Done"
 
 setup-mattermost:
+	@cp ./files/mattermost/defaultConfig.json ./volumes/mattermost/config
+	@cp ./files/mattermost/replicaConfig.json ./volumes/mattermost/config
+	@cp ./files/mattermost/rtcdConfig.json ./volumes/mattermost/config
+	@cp ./files/mattermost/samlCert.crt ./volumes/mattermost/config
+	@cp ./license.mattermost ./volumes/mattermost/config/license.mattermost-enterprise
+
 	@./scripts/mattermost.sh setup
 
 check-mattermost:
@@ -22,7 +28,7 @@ echo-logins:
 
 run: 
 	@echo "Starting..."
-	#@make restore-keycloak
+	@make restore-keycloak
 	@make run-core
 	@make setup-mattermost
 	@make echo-logins
@@ -30,7 +36,7 @@ run:
 
 run-core:
 	@echo "Starting the core services... hang in there."
-	@docker-compose up -d postgres openldap prometheus grafana elasticsearch mattermost keycloak #mitmproxy
+	@docker-compose up -d postgres openldap prometheus grafana elasticsearch mattermost keycloak loki alloy
 
 run-db-replicas:
 	@echo "Starting with replicas. Hang in there..."
@@ -44,6 +50,11 @@ run-mm-replicas:
 	@echo "Starting Mattermost replicas. Hang in there..."
 	@docker exec -it cs-repro-mattermost mmctl config set ClusterSettings.Enable true --local
 	@docker-compose down mattermost
+	@cp ./files/mattermost/defaultConfig.json ./volumes/mattermost_2/config
+	@cp ./files/mattermost/replicaConfig.json ./volumes/mattermost_2/config
+	@cp ./files/mattermost/rtcdConfig.json ./volumes/mattermost_2/config
+	@cp ./files/mattermost/samlCert.crt ./volumes/mattermost_2/config
+	@cp ./license.mattermost ./volumes/mattermost/mattermost_2/license.mattermost-enterprise
 	@docker-compose up -d mattermost mattermost-2
 	@docker exec -it -u root cs-repro-mattermost-2 /bin/bash update-ca-certificates
 	@echo "Should be up and running. Go crazy."
@@ -82,6 +93,14 @@ restart-mattermost:
 	@docker start cs-repro-mattermost
 	@make check-mattermost
 
+restart-grafana:
+	@echo "Restarting Grafana container"
+	@docker stop cs-repro-grafana
+	@wait
+	@echo "Starting Grafana container"
+	@docker start cs-repro-grafana
+	@echo "Grafana restarted"
+
 reset:
 	@echo "Resetting..."
 	@make delete-data

README.md

@@ -71,7 +71,7 @@ Additionally, the keycloak container can take up to 5 minutes to spin up. If it'
 - **`make run`**: Initializes the environment and creates the containers
 - **`make run-all`**: Spins up all environment containers with the database replicas and Mattermost HA.
 - You must run `make run` before running the below:
-  - **`make run-db-replica`**: Launches the environment with replicas. Ideal for adding replicas to an existing setup or initializing with replicas from the get-go.
+  - **`make run-db-replicas`**: Launches the environment with replicas. Ideal for adding replicas to an existing setup or initializing with replicas from the get-go.
   - **`make run-mm-replica`**: Launches an additional Mattermost node and enables HA.
   - **`make run-rtcd`**: Launches the RTCD service for Mattermost Calls and updates the config to use it correctly.
 - **`make backup-keycloak`**: Generates a backup of the current Keycloak setup in the files directory. Useful for infrequent backups.
@@ -79,6 +79,7 @@ Additionally, the keycloak container can take up to 5 minutes to spin up. If it'
 - **`make stop`**: Halts all running containers.
 - **`make restart`**: Restarts all Docker containers in the environment.
 - **`make restart-mattermost`**: Specifically restarts only the Mattermost containers for quick testing.
+- **`make restart-grafana`**: Specifically restarts only the Grafana container.
 - **`make reset`**: Cleans the volumes directory and reinitializes the environment to default settings.
 - **`make delete-data`**: Clears all data within volumes, effectively stopping Mattermost.
 - **`make nuke`**: Erases all configurations and data, sparing your personal data.
@@ -90,10 +91,10 @@ Additionally, the keycloak container can take up to 5 minutes to spin up. If it'
 
 1. Modify the line in the `docker-compose.yml` file to be the version you want
 
-    You're just replacing the tag at the end, this one is `7.7` for example. It must be a version of Mattermost that exists on Docker.
+    You're just replacing the tag at the end, this one is `10.7` for example. It must be a version of Mattermost that exists on Docker.
 
     ```bash
-    mattermost/mattermost-enterprise-edition:release-7.7
+    mattermost/mattermost-enterprise-edition:10.7
     ```
 
 2. Run `make restart-mattermost`
@@ -106,10 +107,10 @@ Doing this will wipe anything you have in the database and any existing Mattermo
 
 1. Modify the line in the `docker-compose.yml` file to be the version you want
 
-    You're just replacing the tag at the end, this one is `7.7` for example. It must be a version of Mattermost that exists on Docker.
+    You're just replacing the tag at the end, this one is `10.7` for example. It must be a version of Mattermost that exists on Docker.
 
     ```bash
-    mattermost/mattermost-enterprise-edition:release-7.7
+    mattermost/mattermost-enterprise-edition:10.7
     ```
   
 2. Run `make downgrade`

docker-compose.yml

@@ -46,6 +46,40 @@ services:
     volumes:
       - ./files/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./volumes/prometheus:/prometheus
+  loki:
+    container_name: cs-repro-loki
+    image: grafana/loki:3.2.2
+    restart: unless-stopped
+    ports:
+      - 3100:3100
+    volumes:
+      - ./files/loki:/etc/loki
+      - ./volumes/loki-data:/tmp/loki
+    command: -config.file=/etc/loki/loki-config.yaml
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "wget --no-verbose --tries=1 --spider http://localhost:3100/loki/api/v1/labels || exit 1",
+        ]
+      interval: 20s
+      timeout: 3s
+      retries: 5
+      start_period: 40s
+  alloy:
+    container_name: cs-repro-alloy
+    image: grafana/alloy:latest
+    restart: unless-stopped
+    ports:
+      - 9080:9080
+      - 9999:9999
+    volumes:
+      - ./files/alloy:/etc/alloy:ro
+      - ./volumes/alloy/data:/var/lib/alloy/data:rw
+      - ./volumes/logs-node-1:/mattermost/logs:ro
+    command: run --server.http.listen-addr=0.0.0.0:9080 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+    depends_on:
+      - loki
   grafana:
     container_name: cs-repro-grafana
     image: grafana/grafana:10.0.4
@@ -66,9 +100,9 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-        #mitmproxy:
+      mitmproxy:
-        #  condition: service_started
+        condition: service_started
-    image: mattermost/mattermost-enterprise-edition:release-9.11
+    image: mattermost/mattermost-enterprise-edition:10.7
     restart: unless-stopped
     security_opt:
       - no-new-privileges:true
@@ -82,15 +116,10 @@ services:
     volumes:
       - ./volumes/mattermost/config:/mattermost/config:rw
       - ./volumes/mattermost/data:/mattermost/data:rw
-      - ./volumes/mattermost/logs:/mattermost/logs:rw
+      - ./volumes/logs-node-1:/mattermost/logs:rw
       - ./volumes/mattermost/plugins:/mattermost/plugins:rw
       - ./volumes/mattermost/client/plugins:/mattermost/client/plugins:rw
       - ./volumes/mattermost/bleve-indexes:/mattermost/bleve-indexes:rw
-      #- ./license.mattermost:/mattermost/config/license.mattermost-enterprise:ro
-      #- ./files/mattermost/samlCert.crt:/mattermost/config/saml-cert.crt
-      #- ./files/mattermost/defaultConfig.json:/mattermost/config/defaultConfig.json:ro
-      #- ./files/mattermost/replicaConfig.json:/mattermost/config/replicaConfig.json:ro
-      #- ./files/mattermost/rtcdConfig.json:/mattermost/config/rtcdConfig.json:ro
 
       ## Files are required for the mitmproxy on the box
       - ./files/mitmproxy/mitmproxy-ca.pem:/etc/ssl/certs/mitmproxy-ca.pem
@@ -104,6 +133,7 @@ services:
       - MM_ServiceSettings_EnableLocalMode=true
       - MM_ServiceSettings_LocalModeSocketLocation=/var/tmp/mattermost_local.socket
       - MM_ServiceSettings_LicenseFileLocation=/mattermost/config/license.mattermost-enterprise
+      - MM_LogSettings_Directory=
 
       ## mysql Settings
       # - MM_SqlSettings_DriverName=mysql
@@ -114,8 +144,7 @@ services:
   keycloak:
     container_name: cs-repro-keycloak
     restart: unless-stopped
-    platform: linux/amd64
+    image: quay.io/keycloak/keycloak:18.0
-    image: quay.io/keycloak/keycloak:latest
     volumes:
       - ./volumes/keycloak:/opt/keycloak/data:rw
     environment:
@@ -204,9 +233,9 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-          #mitmproxy:
+      mitmproxy:
-          #  condition: service_started
+        condition: service_started
-      image: mattermost/mattermost-enterprise-edition:release-9.11
+    image: mattermost/mattermost-enterprise-edition:10.7
     restart: unless-stopped
     security_opt:
       - no-new-privileges:true
@@ -220,14 +249,14 @@ services:
     volumes:
       - ./volumes/mattermost/config:/mattermost/config:rw
       - ./volumes/mattermost/data:/mattermost/data:rw
-        - ./volumes/mattermost_2/logs:/mattermost/logs:rw
+      - ./volumes/logs-node-2:/mattermost-1/logs:rw
       - ./volumes/mattermost_2/plugins:/mattermost/plugins:rw
       - ./volumes/mattermost_2/client/plugins:/mattermost/client/plugins:rw
       - ./volumes/mattermost_2/bleve-indexes:/mattermost/bleve-indexes:rw
-        - ./license.mattermost:/mattermost/config/license.mattermost-enterprise:ro
+      # - ./license.mattermost:/mattermost/config/license.mattermost-enterprise:ro
-        - ./files/mattermost/samlCert.crt:/mattermost/config/saml-cert.crt
+      # - ./files/mattermost/samlCert.crt:/mattermost/config/saml-cert.crt
-        - ./files/mattermost/defaultConfig.json:/mattermost/config/defaultConfig.json:ro
+      # - ./files/mattermost/defaultConfig.json:/mattermost/config/defaultConfig.json:ro
-        - ./files/mattermost/replicaConfig.json:/mattermost/config/replicaConfig.json:ro
+      # - ./files/mattermost/replicaConfig.json:/mattermost/config/replicaConfig.json:ro
 
       ## Files are required for the mitmproxy on the box
       - ./files/mitmproxy/mitmproxy-ca.pem:/etc/ssl/certs/mitmproxy-ca.pem
@@ -241,6 +270,7 @@ services:
       - MM_ServiceSettings_EnableLocalMode=true
       - MM_ServiceSettings_LocalModeSocketLocation=/var/tmp/mattermost_local.socket
       - MM_ServiceSettings_LicenseFileLocation=/mattermost/config/license.mattermost-enterprise
+      - MM_LogSettings_Directory=/mattermost/logs
   mattermost-rtcd:
     container_name: cs-repro-mattermost-rtcd
     platform: linux/amd64
@@ -258,16 +288,11 @@ services:
       - "8443:8443/udp"
       - "8443:8443/tcp"
       - "8045:8045"
-      #mitmproxy:
+  mitmproxy:
-      #  container_name: cs-repro-mitmproxy
+    container_name: cs-repro-mitmproxy
-      #  image: mitmproxy/mitmproxy
+    image: mitmproxy/mitmproxy
-      #  command: mitmweb --web-host 0.0.0.0 --set confdir=/certs --mode transparent
+    command: mitmweb --web-host 0.0.0.0 --set confdir=/certs --mode transparent
-      #  volumes:
+# Named volumes section is removed since we're using host path mounts
-      #    - ./files/mitmproxy:/certs
-      #  ports:
-      #    - "8180:8080"
-      #    - "8181:8081"
-      #  restart: unless-stopped
 # mysql:
 #   container_name: cs-repro-mysql
 #   image: mysql:8

files/alloy/README.md

@@ -0,0 +1,45 @@
+# Alloy Log Agent Configuration
+
+This directory contains the configuration for Grafana Alloy, which has replaced Promtail as the log agent in this environment.
+
+## Key Differences
+
+- Alloy uses a component-based configuration format with `.alloy` extension
+- The web UI is available at http://localhost:9080
+- Alloy can handle logs, metrics, and traces in one agent
+- Configuration is more flexible with the River language
+
+## Configuration Explanation
+
+The `config.alloy` file follows the component-based model where:
+
+1. `loki.source.file` components directly collect logs from Mattermost log files
+2. `loki.process` component parses and labels the JSON logs
+3. `loki.write` component sends the logs to Loki
+
+## Current Setup
+
+Our configuration:
+- Monitors Mattermost logs directly from mounted volumes
+- Labels all logs with `job="mattermost"` for Grafana dashboard compatibility
+- Extracts log level, message, and other metadata from JSON logs
+- Sends logs to Loki service
+
+## Converting Promtail Config to Alloy
+
+If you need to update the configuration, you can convert Promtail configs to Alloy format with:
+
+```bash
+# Example using the alloy CLI (if installed locally)
+alloy convert --source-format=promtail --output=config.alloy promtail-config.yaml
+```
+
+## Important Syntax Notes
+
+- The River language used by Alloy requires commas at the end of each key-value pair in objects
+- Trailing commas in lists and objects are supported and recommended
+
+## More Information
+
+- [Grafana Alloy Documentation](https://grafana.com/docs/alloy/latest/)
+- [Migrating from Promtail to Alloy](https://grafana.com/docs/loki/latest/send-data/alloy/migrate-from-promtail/)

files/alloy/config.alloy

@@ -0,0 +1,59 @@
+// File-based log sources
+loki.source.file "mattermost_logs" {
+  targets = [
+    { "__path__" = "/mattermost/logs/mattermost.log", "filename" = "/logs-node-1/mattermost.log", "job" = "mattermost" },
+  ]
+  forward_to = [loki.process.mm_logs_processor.receiver]
+}
+
+loki.source.file "mattermost_advanced_logs" {
+  targets = [
+    { "__path__" = "/mattermost/logs/advancedLogs.log", "filename" = "/logs-node-1/advancedLogs.log", "job" = "mattermost" },
+  ]
+  forward_to = [loki.process.mm_logs_processor.receiver]
+}
+
+loki.source.file "mattermost_ldap_logs" {
+  targets = [
+    { "__path__" = "/mattermost/logs/advancedLdapLogs.log", "filename" = "/logs-node-1/advancedLdapLogs.log", "job" = "mattermost" },
+  ]
+  forward_to = [loki.process.mm_logs_processor.receiver]
+}
+
+// Process Mattermost logs
+loki.process "mm_logs_processor" {
+  // First, try to parse as JSON
+  stage.json {
+    expressions = {
+      ts = "timestamp",
+      log_level = "level",
+      log_msg = "msg",
+      log_caller = "caller",
+    }
+  }
+
+  // Set labels based on extracted fields
+  stage.labels {
+    values = {
+      level = "log_level",
+    }
+  }
+
+  forward_to = [loki.write.loki.receiver]
+}
+
+// API endpoint for direct log pushing
+loki.source.api "push_api" {
+  http {
+    listen_address = "0.0.0.0"
+    listen_port = 9999
+  }
+  forward_to = [loki.write.loki.receiver]
+}
+
+// Send all logs to Loki
+loki.write "loki" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}

files/grafana/README.md

@@ -0,0 +1,63 @@
+# Grafana Configuration
+
+This directory contains the configuration for Grafana, which serves as the visualization and dashboarding platform for the CS Repro environment.
+
+## Overview
+
+Grafana is an open-source platform for monitoring and observability that allows you to query, visualize, alert on, and understand your metrics, logs, and traces.
+
+## Current Setup
+
+- Running as a single instance (version 10.0.4)
+- Available at http://localhost:3000
+- Default credentials: admin/admin
+
+## Directory Structure
+
+- **dashboards/**: Contains JSON dashboard definitions
+  - `enhanced_logs.json`: Dashboard for viewing and filtering Mattermost logs
+  - `kpiMetrics.json`: Dashboard for key performance indicators
+  - `metricsv2.json`: Dashboard for detailed application metrics
+  - `bonusMetrics.json`: Dashboard for additional metrics
+
+- **provisioning/**: Contains automatic provisioning configurations
+  - `datasources/`: Configures data source connections
+  - `dashboards/`: Sets up dashboard loading
+
+## Data Sources
+
+The environment is configured with the following data sources:
+
+1. **Prometheus** (Default): For metrics collection
+   - URL: http://prometheus:9090
+   - Used by most of the metric dashboards
+
+2. **Loki**: For log aggregation
+   - URL: http://loki:3100
+   - Used by the enhanced_logs dashboard
+   - Queries using the LogQL language
+
+## Log Dashboard
+
+The `enhanced_logs.json` dashboard is designed to work with logs collected via Alloy and stored in Loki. It provides:
+
+- Log filtering by level (error, warn, info, debug)
+- Visual metrics about log levels and counts
+- Error tracking and analysis
+- Time-series views of log patterns
+
+The dashboard queries use the label `job="mattermost"` to filter logs from the Mattermost application.
+
+## Best Practices
+
+When modifying dashboards:
+- Export/backup existing dashboards before making major changes
+- Test queries in the Explore interface before adding to dashboards
+- Use variables for consistent filtering across panels
+- Maintain consistent styling
+
+## More Information
+
+- [Grafana Documentation](https://grafana.com/docs/grafana/latest/)
+- [Dashboard JSON Model](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/dashboard-json-model/)
+- [LogQL in Grafana](https://grafana.com/docs/grafana/latest/datasources/loki/query-editor/)

files/grafana/dashboards/enhanced_logs.json

@@ -0,0 +1,827 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 7,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "builder",
+          "expr": "{job=\"mattermost\"} | json | level=~\"${log_level}\" | line_format \"{{.level}} [{{.caller}}] {{.msg}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Filtered Logs",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [
+            "sum"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({job=\"mattermost\"} | json | level=~\"(debug|info|warn|error)\" [$__interval])) by (level)",
+          "legendFormat": "{{level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Levels Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 0,
+        "y": 20
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({job=\"mattermost\"} | json | level=\"error\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Error Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 10
+              },
+              {
+                "color": "red",
+                "value": 20
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 4,
+        "y": 20
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({job=\"mattermost\"} | json | level=\"warn\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Warning Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 8,
+        "y": 20
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({job=\"mattermost\"} | json | level=\"info\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Info Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 5,
+        "x": 12,
+        "y": 20
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({job=\"mattermost\"} | json | level=\"debug\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Debug Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 20
+      },
+      "id": 6,
+      "options": {
+        "displayLabels": [
+          "name",
+          "percent"
+        ],
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "values": [
+            "value"
+          ]
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum by(level) (count_over_time({job=\"mattermost\"} | json [30m]))",
+          "legendFormat": "{{level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Level Distribution",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "count"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 80
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "component"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 200
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 28
+      },
+      "id": 5,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "topk(10, sum by(caller) (count_over_time({job=\"mattermost\"} | json | level=~\"warn|error\" [30m])))",
+          "legendFormat": "{{caller}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 10 Error Sources (30m)",
+      "transformations": [
+        {
+          "id": "reduce",
+          "options": {
+            "includeTimeField": false,
+            "mode": "reduceFields",
+            "reducers": [
+              "sum"
+            ]
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "Field": "component",
+              "Value": "count"
+            }
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "count"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "description": "Displays errors grouped by error message",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "filterable": true,
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Count"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 80
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Error Message"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 500
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 28
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Count"
+          }
+        ]
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "topk(10, sum by(error) (count_over_time({job=\"mattermost\"} | json | level=\"error\" [30m])))",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 10 Error Messages",
+      "transformations": [
+        {
+          "id": "reduce",
+          "options": {
+            "includeTimeField": false,
+            "mode": "reduceFields",
+            "reducers": [
+              "sum"
+            ]
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "Field": "Error Message",
+              "Value": "Count"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [
+    "mattermost",
+    "logs"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "loki",
+          "value": "loki"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_LOKI",
+        "options": [],
+        "query": "loki",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "info",
+            "warn",
+            "error",
+            "debug"
+          ],
+          "value": [
+            "info",
+            "warn",
+            "error",
+            "debug"
+          ]
+        },
+        "hide": 0,
+        "includeAll": true,
+        "label": "Log Level",
+        "multi": true,
+        "name": "log_level",
+        "options": [
+          {
+            "selected": false,
+            "text": "All",
+            "value": "$__all"
+          },
+          {
+            "selected": true,
+            "text": "debug",
+            "value": "debug"
+          },
+          {
+            "selected": true,
+            "text": "info",
+            "value": "info"
+          },
+          {
+            "selected": true,
+            "text": "warn",
+            "value": "warn"
+          },
+          {
+            "selected": true,
+            "text": "error",
+            "value": "error"
+          }
+        ],
+        "query": "debug,info,warn,error",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Mattermost Enhanced Logs",
+  "uid": "mattermost-logs",
+  "version": 2,
+  "weekStart": ""
+}

files/grafana/provisioning/datasources/automatic.yml

@@ -15,3 +15,17 @@ datasources:
         httpMethod: "POST"
      version: 1
      editable: true
+   - name: loki
+     type: loki
+     access: proxy
+     url: http://loki:3100
+     password:
+     user:
+     database:
+     isDefault: false
+     jsonData:
+        tlsAuth: false
+        tlsAuthWithCACert: false
+        httpMethod: "POST"
+     version: 1
+     editable: true

files/loki/README.md

@@ -0,0 +1,51 @@
+# Loki Configuration
+
+This directory contains the configuration for Grafana Loki, which serves as the log storage and aggregation system in this environment.
+
+## Overview
+
+Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system designed by Grafana Labs. It is optimized for efficiently storing and querying logs from Kubernetes and microservices deployments.
+
+## Current Setup
+
+- Running as a single instance in this CS Repro environment
+- Available at http://localhost:3100
+- Receives logs from Grafana Alloy (which replaced Promtail)
+
+## Configuration Details
+
+The `loki-config.yaml` file contains the core configuration for the Loki service:
+
+- **Storage**: Configured to use the local filesystem for simplicity
+- **Schema**: Uses the v11 schema with appropriate index/chunk configurations
+- **Compaction**: Enabled to optimize storage over time
+- **Limits**: Configured with reasonable defaults for this environment
+- **Frontend**: Configured for basic query handling
+
+## Querying Logs
+
+Logs can be queried in several ways:
+
+1. **LogQL via Grafana**: The primary and most user-friendly method
+2. **Direct Loki API**: Using the API endpoints at http://localhost:3100/loki/api/v1/
+3. **LogCLI**: If installed, you can use Grafana's logcli tool
+
+Example LogQL queries:
+```
+{job="mattermost"} | json | level=~"error|warn"
+{job="mattermost"} | json | level="error" | line_format "{{.msg}}"
+```
+
+## Labels
+
+The current configuration uses the following key labels:
+
+- `job`: The source application ("mattermost")
+- `level`: The log level (error, warn, info, debug, etc.)
+- `service_name`: Identifies the service
+- `filename`: The source log file
+
+## More Information
+
+- [Grafana Loki Documentation](https://grafana.com/docs/loki/latest/)
+- [LogQL Query Language](https://grafana.com/docs/loki/latest/logql/)

files/loki/loki-config.yaml

@@ -0,0 +1,50 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

files/promtail/promtail-config.yaml

@@ -0,0 +1,41 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: mmlogs-1
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-1
+      __path__: /logs-node-1/*.log
+- job_name: mmlogs-2
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-2
+      __path__: /logs-node-2/*.log
+- job_name: mmlogs-local-1
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-local-1
+      __path__: /logs-local-1/*.log
+- job_name: mmlogs-local-2
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-local-2
+      __path__: /logs-local-2/*.log
+
+

scripts/general.sh

@@ -9,6 +9,8 @@ logins () {
   echo "    - All Mattermost Grafana charts are setup."
   echo "    - For more info https://github.com/coltoneshaw/CS-Repro-Mattermost#use-grafana"
   echo "- Prometheus: http://localhost:9090"
+  echo "- Loki: http://localhost:3100/ready"
+  echo "- Alloy: http://localhost:9080"
   echo "- PostgreSQL"  "localhost:5432" with 'mmuser' / 'mmuser_password'
   echo
   echo ===========================================================