add claude file and update readme

* add promtail loki support

* updated images and removed arch directives in docker-compose

* fix grafana logs

* add restart-grafana make command

* clean up enhanced logs dashboard

* update both mattermost servers to use the same version

* update docker-compose and promtail config to use host path mounts for logs

* clean up dashboard for good

.gitignore

@@ -1,5 +1,4 @@
-license.mattermost
+license*
-license.txt
 
 ## Ignoring keycloak so it can be pulled already setup.
 volumes
@@ -13,3 +12,8 @@ files/postgres/replica/replica_1_postgres.conf
 
 files/postgres/replica/replica_2.conf
 files/postgres/replica/replica_2_postgres.conf
+
+**/.claude/settings.local.json
+
+*_backup.json
+*.log

CLAUDE.md

@@ -0,0 +1,92 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+This repository contains a CS Repro Mattermost environment - a reproduction of a standard Mattermost customer production environment with preconfigured SAML, LDAP, Advanced Logging, Prometheus, Grafana, Elasticsearch, and read replicas. It is designed for testing and reproducing customer environments.
+
+## Environment Setup
+
+### Prerequisites
+
+You need to have Docker and Docker Compose installed on your machine to run this environment.
+
+### Initial Setup
+
+1. Add an enterprise license file named `license.mattermost` to the root directory
+2. Run `make run` to start the core services
+
+## Common Commands
+
+### Environment Management
+
+- `make run` - Initialize environment and create containers
+- `make run-all` - Spin up all environment containers (with DB replicas and Mattermost HA)
+- `make run-db-replicas` - Add database replicas to the environment
+- `make run-mm-replicas` - Add Mattermost HA nodes to the environment
+- `make run-rtcd` - Launch RTCD service for Mattermost Calls
+- `make start` - Start existing containers
+- `make stop` - Stop all running containers
+- `make restart` - Restart all containers
+- `make restart-mattermost` - Restart only Mattermost containers (preferred for testing changes)
+- `make backup-keycloak` - Generate a backup of the current Keycloak setup
+- `make restore-keycloak` - Restore Keycloak data from an existing backup
+- `make reset` - Clean volumes directory and reinitialize the environment
+- `make delete-data` - Clear all data within volumes
+- `make nuke` - Erase all configurations and data
+- `make nuke-rmi` - Complete cleanup including removing Docker images
+
+### Upgrading and Downgrading
+
+To upgrade Mattermost:
+1. Modify the image tag in `docker-compose.yml`
+2. Run `make restart-mattermost`
+
+To downgrade Mattermost:
+1. Modify the image tag in `docker-compose.yml`
+2. Run `make downgrade`
+
+## Component Access
+
+### Mattermost
+
+- URL: http://localhost:8065
+- Default credentials available in the accounts section below
+
+### LDAP
+
+- Admin access: `cn=admin,dc=planetexpress,dc=com` / `GoodNewsEveryone`
+- Commands for LDAP operations are in the README.md
+
+### Grafana
+
+- URL: http://localhost:3000
+- Credentials: admin / admin
+
+### Elasticsearch
+
+- URL: http://localhost:9200
+
+### mitmproxy
+
+- URL: http://localhost:8181
+
+## Account Information
+
+| Username  | Password  | Role in Mattermost | Can use LDAP? | Can use SAML? |
+|-----------|-----------|-------------------|---------------|---------------|
+| admin     | admin     | n/a               | n/a           | n/a           |
+| professor | professor | Sys Admin         | Yes           | Yes           |
+| bender    | bender    | Member            | Yes           | Yes           |
+| hermes    | hermes    | Sys Admin         | Yes           | Yes           |
+| fry       | fry       | Member            | Yes           | Yes           |
+| leela     | leela     | Member            | Yes           | Yes           |
+| zoidberg  | zoidberg  | Member            | Yes           | Yes           |
+| amy       | amy       | Member            | Yes           | Yes           |
+
+## Database Information
+
+- Primary PostgreSQL: `postgresql://mmuser:mmuser_password@localhost:5432/mattermost`
+- Replica 1: `postgresql://mmuser:mmuser_password@localhost:5433/mattermost`
+- Replica 2: `postgresql://mmuser:mmuser_password@localhost:5434/mattermost`

Makefile

@@ -6,6 +6,12 @@ logs:
 	@echo "Done"
 
 setup-mattermost:
+	@cp ./files/mattermost/defaultConfig.json ./volumes/mattermost/config
+	@cp ./files/mattermost/replicaConfig.json ./volumes/mattermost/config
+	@cp ./files/mattermost/rtcdConfig.json ./volumes/mattermost/config
+	@cp ./files/mattermost/samlCert.crt ./volumes/mattermost/config
+	@cp ./license.mattermost ./volumes/mattermost/config/license.mattermost-enterprise
+
 	@./scripts/mattermost.sh setup
 
 check-mattermost:
@@ -30,7 +36,7 @@ run:
 
 run-core:
 	@echo "Starting the core services... hang in there."
-	@docker-compose up -d postgres openldap prometheus grafana elasticsearch mattermost keycloak mitmproxy
+	@docker-compose up -d postgres openldap prometheus grafana elasticsearch mattermost keycloak loki promtail
 
 run-db-replicas:
 	@echo "Starting with replicas. Hang in there..."
@@ -44,6 +50,11 @@ run-mm-replicas:
 	@echo "Starting Mattermost replicas. Hang in there..."
 	@docker exec -it cs-repro-mattermost mmctl config set ClusterSettings.Enable true --local
 	@docker-compose down mattermost
+	@cp ./files/mattermost/defaultConfig.json ./volumes/mattermost_2/config
+	@cp ./files/mattermost/replicaConfig.json ./volumes/mattermost_2/config
+	@cp ./files/mattermost/rtcdConfig.json ./volumes/mattermost_2/config
+	@cp ./files/mattermost/samlCert.crt ./volumes/mattermost_2/config
+	@cp ./license.mattermost ./volumes/mattermost/mattermost_2/license.mattermost-enterprise
 	@docker-compose up -d mattermost mattermost-2
 	@docker exec -it -u root cs-repro-mattermost-2 /bin/bash update-ca-certificates
 	@echo "Should be up and running. Go crazy."
@@ -82,6 +93,14 @@ restart-mattermost:
 	@docker start cs-repro-mattermost
 	@make check-mattermost
 
+restart-grafana:
+	@echo "Restarting Grafana container"
+	@docker stop cs-repro-grafana
+	@wait
+	@echo "Starting Grafana container"
+	@docker start cs-repro-grafana
+	@echo "Grafana restarted"
+
 reset:
 	@echo "Resetting..."
 	@make delete-data

README.md

@@ -71,7 +71,7 @@ Additionally, the keycloak container can take up to 5 minutes to spin up. If it'
 - **`make run`**: Initializes the environment and creates the containers
 - **`make run-all`**: Spins up all environment containers with the database replicas and Mattermost HA.
 - You must run `make run` before running the below:
-  - **`make run-db-replica`**: Launches the environment with replicas. Ideal for adding replicas to an existing setup or initializing with replicas from the get-go.
+  - **`make run-db-replicas`**: Launches the environment with replicas. Ideal for adding replicas to an existing setup or initializing with replicas from the get-go.
   - **`make run-mm-replica`**: Launches an additional Mattermost node and enables HA.
   - **`make run-rtcd`**: Launches the RTCD service for Mattermost Calls and updates the config to use it correctly.
 - **`make backup-keycloak`**: Generates a backup of the current Keycloak setup in the files directory. Useful for infrequent backups.
@@ -79,6 +79,7 @@ Additionally, the keycloak container can take up to 5 minutes to spin up. If it'
 - **`make stop`**: Halts all running containers.
 - **`make restart`**: Restarts all Docker containers in the environment.
 - **`make restart-mattermost`**: Specifically restarts only the Mattermost containers for quick testing.
+- **`make restart-grafana`**: Specifically restarts only the Grafana container.
 - **`make reset`**: Cleans the volumes directory and reinitializes the environment to default settings.
 - **`make delete-data`**: Clears all data within volumes, effectively stopping Mattermost.
 - **`make nuke`**: Erases all configurations and data, sparing your personal data.
@@ -90,10 +91,10 @@ Additionally, the keycloak container can take up to 5 minutes to spin up. If it'
 
 1. Modify the line in the `docker-compose.yml` file to be the version you want
 
-    You're just replacing the tag at the end, this one is `7.7` for example. It must be a version of Mattermost that exists on Docker.
+    You're just replacing the tag at the end, this one is `10.7` for example. It must be a version of Mattermost that exists on Docker.
 
     ```bash
-    mattermost/mattermost-enterprise-edition:release-7.7
+    mattermost/mattermost-enterprise-edition:10.7
     ```
 
 2. Run `make restart-mattermost`
@@ -106,10 +107,10 @@ Doing this will wipe anything you have in the database and any existing Mattermo
 
 1. Modify the line in the `docker-compose.yml` file to be the version you want
 
-    You're just replacing the tag at the end, this one is `7.7` for example. It must be a version of Mattermost that exists on Docker.
+    You're just replacing the tag at the end, this one is `10.7` for example. It must be a version of Mattermost that exists on Docker.
 
     ```bash
-    mattermost/mattermost-enterprise-edition:release-7.7
+    mattermost/mattermost-enterprise-edition:10.7
     ```
   
 2. Run `make downgrade`

docker-compose.yml

@@ -46,6 +46,30 @@ services:
     volumes:
       - ./files/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./volumes/prometheus:/prometheus
+  loki:
+    container_name: cs-repro-loki
+    image: grafana/loki:3.2.2
+    restart: unless-stopped
+    ports:
+      - 3100:3100
+    volumes:
+      - ./files/loki:/etc/loki
+      - ./volumes/logs-node-1:/logs/node-1
+      - ./volumes/logs-node-2:/logs/node-2
+    command: -config.file=/etc/loki/loki-config.yaml
+  promtail:
+    container_name: cs-repro-promtail
+    image: grafana/promtail:3.2.2
+    restart: unless-stopped
+    ports:
+      - 9080:9080
+    volumes:
+      - ./files/promtail:/etc/promtail
+      - ./volumes/logs-node-1:/logs-node-1
+      - ./volumes/logs-node-2:/logs-node-2
+    command: -config.file=/etc/promtail/promtail-config.yaml
+    depends_on:
+      - loki
   grafana:
     container_name: cs-repro-grafana
     image: grafana/grafana:10.0.4
@@ -68,7 +92,7 @@ services:
         condition: service_healthy
       mitmproxy:
         condition: service_started
-    image: mattermost/mattermost-enterprise-edition:release-9.7
+    image: mattermost/mattermost-enterprise-edition:10.7
     restart: unless-stopped
     security_opt:
       - no-new-privileges:true
@@ -82,28 +106,24 @@ services:
     volumes:
       - ./volumes/mattermost/config:/mattermost/config:rw
       - ./volumes/mattermost/data:/mattermost/data:rw
-      - ./volumes/mattermost/logs:/mattermost/logs:rw
+      - ./volumes/logs-node-1:/mattermost/logs:rw
       - ./volumes/mattermost/plugins:/mattermost/plugins:rw
       - ./volumes/mattermost/client/plugins:/mattermost/client/plugins:rw
       - ./volumes/mattermost/bleve-indexes:/mattermost/bleve-indexes:rw
-      - ./license.mattermost:/mattermost/config/license.mattermost-enterprise:ro
-      - ./files/mattermost/samlCert.crt:/mattermost/config/saml-cert.crt
-      - ./files/mattermost/defaultConfig.json:/mattermost/config/defaultConfig.json:ro
-      - ./files/mattermost/replicaConfig.json:/mattermost/config/replicaConfig.json:ro
-      - ./files/mattermost/rtcdConfig.json:/mattermost/config/rtcdConfig.json:ro
 
       ## Files are required for the mitmproxy on the box
       - ./files/mitmproxy/mitmproxy-ca.pem:/etc/ssl/certs/mitmproxy-ca.pem
       - ./files/mitmproxy/mitmproxy-dhparam.pem:/etc/ssl/certs/mitmproxy-dhparam.pem
     environment:
-      - HTTP_PROXY=http://cs-repro-mitmproxy:8080
+      # - HTTP_PROXY=http://cs-repro-mitmproxy:8080
-      - HTTPS_PROXY=http://cs-repro-mitmproxy:8080
+      # - HTTPS_PROXY=http://cs-repro-mitmproxy:8080
       - MM_SqlSettings_DriverName=postgres
       - MM_SqlSettings_DataSource=postgres://mmuser:mmuser_password@cs-repro-postgres:5432/mattermost?sslmode=disable&connect_timeout=10&binary_parameters=yes
       - MM_SAMLSETTINGS_IDPCERTIFICATEFILE=/mattermost/config/saml-cert.crt
       - MM_ServiceSettings_EnableLocalMode=true
       - MM_ServiceSettings_LocalModeSocketLocation=/var/tmp/mattermost_local.socket
       - MM_ServiceSettings_LicenseFileLocation=/mattermost/config/license.mattermost-enterprise
+      - MM_LogSettings_Directory=
 
       ## mysql Settings
       # - MM_SqlSettings_DriverName=mysql
@@ -114,8 +134,7 @@ services:
   keycloak:
     container_name: cs-repro-keycloak
     restart: unless-stopped
-    platform: linux/amd64
+    image: quay.io/keycloak/keycloak:18.0
-    image: quay.io/keycloak/keycloak:18.0.0
     volumes:
       - ./volumes/keycloak:/opt/keycloak/data:rw
     environment:
@@ -206,7 +225,7 @@ services:
           condition: service_healthy
         mitmproxy:
           condition: service_started
-      image: mattermost/mattermost-enterprise-edition:release-9.7
+      image: mattermost/mattermost-enterprise-edition:10.7
       restart: unless-stopped
       security_opt:
         - no-new-privileges:true
@@ -220,14 +239,14 @@ services:
       volumes:
         - ./volumes/mattermost/config:/mattermost/config:rw
         - ./volumes/mattermost/data:/mattermost/data:rw
-        - ./volumes/mattermost_2/logs:/mattermost/logs:rw
+        - ./volumes/logs-node-2:/mattermost-1/logs:rw
         - ./volumes/mattermost_2/plugins:/mattermost/plugins:rw
         - ./volumes/mattermost_2/client/plugins:/mattermost/client/plugins:rw
         - ./volumes/mattermost_2/bleve-indexes:/mattermost/bleve-indexes:rw
-        - ./license.mattermost:/mattermost/config/license.mattermost-enterprise:ro
+        # - ./license.mattermost:/mattermost/config/license.mattermost-enterprise:ro
-        - ./files/mattermost/samlCert.crt:/mattermost/config/saml-cert.crt
+        # - ./files/mattermost/samlCert.crt:/mattermost/config/saml-cert.crt
-        - ./files/mattermost/defaultConfig.json:/mattermost/config/defaultConfig.json:ro
+        # - ./files/mattermost/defaultConfig.json:/mattermost/config/defaultConfig.json:ro
-        - ./files/mattermost/replicaConfig.json:/mattermost/config/replicaConfig.json:ro
+        # - ./files/mattermost/replicaConfig.json:/mattermost/config/replicaConfig.json:ro
 
         ## Files are required for the mitmproxy on the box
         - ./files/mitmproxy/mitmproxy-ca.pem:/etc/ssl/certs/mitmproxy-ca.pem
@@ -241,6 +260,7 @@ services:
         - MM_ServiceSettings_EnableLocalMode=true
         - MM_ServiceSettings_LocalModeSocketLocation=/var/tmp/mattermost_local.socket
         - MM_ServiceSettings_LicenseFileLocation=/mattermost/config/license.mattermost-enterprise
+        - MM_LogSettings_Directory=/mattermost/logs
   mattermost-rtcd:
     container_name: cs-repro-mattermost-rtcd
     platform: linux/amd64
@@ -262,12 +282,7 @@ services:
     container_name: cs-repro-mitmproxy
     image: mitmproxy/mitmproxy
     command: mitmweb --web-host 0.0.0.0 --set confdir=/certs --mode transparent
-    volumes:
+# Named volumes section is removed since we're using host path mounts
-      - ./files/mitmproxy:/certs
-    ports:
-      - "8180:8080"
-      - "8181:8081"
-    restart: unless-stopped
 # mysql:
   #   container_name: cs-repro-mysql
   #   image: mysql:8
@@ -293,4 +308,4 @@ services:
   #     # Where our data will be persisted
   #   volumes:
   #     - ./volumes/db/mysql:/var/lib/mysql
-   
+   

files/grafana/dashboards/enhanced_logs.json

@@ -0,0 +1,827 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 7,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "builder",
+          "expr": "{filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=~\"${log_level}\" | line_format \"{{.level}} [{{.caller}}] {{.msg}}\"",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Filtered Logs",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [
+            "sum"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=~\"(debug|info|warn|error)\" [$__interval])) by (level)",
+          "legendFormat": "{{level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Levels Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 0,
+        "y": 20
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=\"error\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Error Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 10
+              },
+              {
+                "color": "red",
+                "value": 20
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 4,
+        "y": 20
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=\"warn\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Warning Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 8,
+        "y": 20
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=\"info\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Info Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "blue",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 5,
+        "x": 12,
+        "y": 20
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=\"debug\" [$__range]))",
+          "queryType": "instant",
+          "refId": "A"
+        }
+      ],
+      "title": "Debug Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 20
+      },
+      "id": 6,
+      "options": {
+        "displayLabels": [
+          "name",
+          "percent"
+        ],
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "values": [
+            "value"
+          ]
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "sum"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "sum by(level) (count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json [30m]))",
+          "legendFormat": "{{level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Level Distribution",
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "count"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 80
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "component"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 200
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 28
+      },
+      "id": 5,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "topk(10, sum by(caller) (count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=~\"warn|error\" [30m])))",
+          "legendFormat": "{{caller}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 10 Error Sources (30m)",
+      "transformations": [
+        {
+          "id": "reduce",
+          "options": {
+            "includeTimeField": false,
+            "mode": "reduceFields",
+            "reducers": [
+              "sum"
+            ]
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "Field": "component",
+              "Value": "count"
+            }
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "count"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${DS_LOKI}"
+      },
+      "description": "Displays errors grouped by error message",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "filterable": true,
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Count"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 80
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Error Message"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 500
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 28
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Count"
+          }
+        ]
+      },
+      "pluginVersion": "10.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${DS_LOKI}"
+          },
+          "editorMode": "code",
+          "expr": "topk(10, sum by(error) (count_over_time({filename=~\"/logs-node-.*/(mattermost|advancedLogs).log\"} | json | level=\"error\" [30m])))",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 10 Error Messages",
+      "transformations": [
+        {
+          "id": "reduce",
+          "options": {
+            "includeTimeField": false,
+            "mode": "reduceFields",
+            "reducers": [
+              "sum"
+            ]
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "Field": "Error Message",
+              "Value": "Count"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [
+    "mattermost",
+    "logs"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "loki",
+          "value": "loki"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_LOKI",
+        "options": [],
+        "query": "loki",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "info",
+            "warn",
+            "error",
+            "debug"
+          ],
+          "value": [
+            "info",
+            "warn",
+            "error",
+            "debug"
+          ]
+        },
+        "hide": 0,
+        "includeAll": true,
+        "label": "Log Level",
+        "multi": true,
+        "name": "log_level",
+        "options": [
+          {
+            "selected": false,
+            "text": "All",
+            "value": "$__all"
+          },
+          {
+            "selected": true,
+            "text": "debug",
+            "value": "debug"
+          },
+          {
+            "selected": true,
+            "text": "info",
+            "value": "info"
+          },
+          {
+            "selected": true,
+            "text": "warn",
+            "value": "warn"
+          },
+          {
+            "selected": true,
+            "text": "error",
+            "value": "error"
+          }
+        ],
+        "query": "debug,info,warn,error",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Mattermost Enhanced Logs",
+  "uid": "mattermost-logs",
+  "version": 2,
+  "weekStart": ""
+}

files/grafana/provisioning/datasources/automatic.yml

@@ -1,17 +1,31 @@
 apiVersion: 1
 
 datasources:
-- name: mattermost
+   - name: mattermost
-  type: prometheus
+     type: prometheus
-  access: proxy
+     access: proxy
-  url: http://prometheus:9090
+     url: http://prometheus:9090
-  password:
+     password:
-  user:
+     user:
-  database: 
+     database:
-  isDefault: true
+     isDefault: true
-  jsonData:
+     jsonData:
-     tlsAuth: false
+        tlsAuth: false
-     tlsAuthWithCACert: false
+        tlsAuthWithCACert: false
-     httpMethod: "POST"
+        httpMethod: "POST"
-  version: 1
+     version: 1
-  editable: true
+     editable: true
+   - name: loki
+     type: loki
+     access: proxy
+     url: http://loki:3100
+     password:
+     user:
+     database:
+     isDefault: false
+     jsonData:
+        tlsAuth: false
+        tlsAuthWithCACert: false
+        httpMethod: "POST"
+     version: 1
+     editable: true

files/loki/loki-config.yaml

@@ -0,0 +1,50 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

files/promtail/promtail-config.yaml

@@ -0,0 +1,41 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: mmlogs-1
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-1
+      __path__: /logs-node-1/*.log
+- job_name: mmlogs-2
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-2
+      __path__: /logs-node-2/*.log
+- job_name: mmlogs-local-1
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-local-1
+      __path__: /logs-local-1/*.log
+- job_name: mmlogs-local-2
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: mmlogs-local-2
+      __path__: /logs-local-2/*.log
+
+

scripts/general.sh

@@ -9,6 +9,8 @@ logins () {
   echo "    - All Mattermost Grafana charts are setup."
   echo "    - For more info https://github.com/coltoneshaw/CS-Repro-Mattermost#use-grafana"
   echo "- Prometheus: http://localhost:9090"
+  echo "- Loki: http://localhost:3100/ready"
+  echo "- Promtail: http://localhost:9080"
   echo "- PostgreSQL"  "localhost:5432" with 'mmuser' / 'mmuser_password'
   echo
   echo ===========================================================