added initial setup of replicas

.gitignore

@@ -6,3 +6,4 @@ volumes
 
 **/.DS_Store
 .DS_Store
+**/baseConfig.conf

Makefile

@@ -30,6 +30,10 @@ run:
 start:
 	@make run
 
+start-replicas:
+	@echo "Starting the replicas... hold on a moment..."
+	@docker-compose -f docker-compose.yml -f docker-compose-read-replicas.yml up -d
+
 stop:
 	@echo "Stopping..."
 	@docker-compose stop

README.md

@@ -11,6 +11,7 @@ This is a basic reproduction that includes various components preconfigured like
   - [How to Downgrade](#how-to-downgrade)
   - [Migrating to Config in DB](#using-config-in-db)
   - [MMCTL](#mmctl)
+  - [Adding Postgres Read Replicas](#adding-postgres-read-replicas)
 
 ## Making Changes
 
@@ -168,6 +169,16 @@ To use `mmctl` it's already setup for local, just run the below docker command.
 docker exec -it cs-repro-mattermost mmctl user list --local
 ```
 
+### Adding Postgres Read Replicas
+
+The basic structure for you to add two read replicas has been included in the repo already. To utilize it you need to start your files with the below command. This should work on a running install of this repo also.
+
+```bash
+make start-replicas
+```
+
+This will take 2-5 minutes to get the replication setup, based on how much data you have in the database right now.
+
 ## Use Grafana
 
 All the Mattermost grafana charts are already installed and linked, you just have to access them. 

docker-compose-read-replicas.yml

@@ -0,0 +1,59 @@
+services:
+  postgres-replica-1:
+    container_name: cs-repro-postgres-replica-1
+    environment:
+      - POSTGRES_USER=mmuser
+      - POSTGRES_PASSWORD=mmuser_password
+      - LISTEN_ADDRESS="*"
+    image: postgres:14-alpine
+    restart: unless-stopped
+    ports:
+      - "5433:5432"
+    security_opt:
+      - no-new-privileges:true
+    pids_limit: 100
+    read_only: false
+    tmpfs:
+      - /tmp
+      - /var/run/postgresql
+    volumes:
+      - ./files/postgres/replica/replica_1/init.sh:/docker-entrypoint-initdb.d/init.sh
+      - ./volumes/db/replica_1/data:/var/lib/postgresql/data
+      - ./files/postgres/replica:/files/postgres/replica
+    healthcheck:
+      test: pg_isready -U mmuser -d mattermost -h cs-repro-postgres
+      interval: 10s
+      timeout: 3s
+      retries: 3
+    depends_on:
+      - mattermost
+      - postgres
+  postgres-replica-2:
+    container_name: cs-repro-postgres-replica-2
+    environment:
+      - POSTGRES_USER=mmuser
+      - POSTGRES_PASSWORD=mmuser_password
+      - LISTEN_ADDRESS="*"
+    image: postgres:14-alpine
+    restart: unless-stopped
+    ports:
+      - "5434:5432"
+    security_opt:
+      - no-new-privileges:true
+    pids_limit: 100
+    read_only: false
+    tmpfs:
+      - /tmp
+      - /var/run/postgresql
+    volumes:
+      - ./files/postgres/replica/replica_2/init.sh:/docker-entrypoint-initdb.d/init.sh
+      - ./files/postgres/replica:/files/postgres/replica
+      - ./volumes/db/replica_2/data:/var/lib/postgresql/data
+    healthcheck:
+      test: pg_isready -U mmuser -d mattermost -h cs-repro-postgres
+      interval: 10s
+      timeout: 3s
+      retries: 3
+    depends_on:
+      - mattermost
+      - postgres

docker-compose.yml

@@ -13,7 +13,7 @@ services:
       - /tmp
       - /var/run/postgresql
     volumes:
-      - ./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data
+      - ./volumes/db/primary/data:/var/lib/postgresql/data
       - ./files/postgres/primary/init.sh:/docker-entrypoint-initdb.d/init.sh
       - ./files/postgres/primary/primary_config.conf:/files/postgres/primary/primary_config.conf
     healthcheck:

files/postgres/primary/init.sh

@@ -1,3 +1,10 @@
 #!/bin/bash
 
 echo "include '/files/postgres/primary/primary_config.conf'" >> /var/lib/postgresql/data/postgresql.conf
+
+psql -U mmuser -d mattermost -c "create role replicauser with replication password 'replicauser_password' login"
+psql -U mmuser -d mattermost -c "select pg_create_physical_replication_slot('replica_2_slot');"
+psql -U mmuser -d mattermost -c "select pg_create_physical_replication_slot('replica_1_slot');"
+
+
+echo "host    replication     replicauser    0.0.0.0/0   md5" >> /var/lib/postgresql/data/pg_hba.conf

files/postgres/primary/primary_config.conf

@@ -1,9 +1,14 @@
-# This value should match the "SqlSettings.MaxConnections" value within your config.json for Mattermost
+# If the instance is lower capacity than r5.xlarge, then set it to a lower number.
-# This is a suggestion and can be set lower / higher based on the size of your server.
+# Also tune the "MaxOpenConns" setting under the "SqlSettings" of the Mattermost app accordingly.
-max_connections = 1020
+# Note that "MaxOpenConns" on Mattermost is per data source name.
-tcp_keepalives_idle = 5
+max_connections = 1024
-tcp_keepalives_interval = 1
+
-tcp_keepalives_count = 5
+# Set it to 1.1, unless the DB is using spinning disks.
+random_page_cost = 1.1
+
+# This should be 32MB if using read replicas, or 16MB if using a single PostgreSQL instance.
+# If the instance is of a lower capacity than r5.xlarge, then set it to a lower number.
+work_mem = 16MB
 
 # Set both of the below settings to 65% of total memory. For a 32 GB instance, it should be 21 GB.
 # If on a smaller server, set this to 20% or less total RAM.
@@ -11,15 +16,27 @@ tcp_keepalives_count = 5
 shared_buffers = 512MB
 effective_cache_size = 512MB
 
-# Set it to 16 MB for readers and 32 MB for writers. If it's a single instance, 16 MB should be sufficient. If the instance is of a lower capacity than r5.xlarge, then set it to a lower number.
+# If you are using pgbouncer, or any similar connection pooling proxy,
-work_mem = 16MB
+# in front of your DB, then apply the keepalive settings to the proxy instead,
+# and revert the keepalive settings for the DB back to defaults.
+tcp_keepalives_idle = 5
+tcp_keepalives_interval = 1
+tcp_keepalives_count = 5
 
 # 1GB (reduce this to 512MB if your server has less than 32GB of RAM)
-autovacuum_work_mem = 512MB
+maintenance_work_mem = 512MB
+
 autovacuum_max_workers = 4
 autovacuum_vacuum_cost_limit = 500
 
-#Set it to 1.1 unless the DB is using spinning disks.
+## Intentionally disabled here because running in docker this may cause performance problems.
-random_page_cost = 1.1
 
-restart_after_crash = on
+# If you have more than 32 CPUs on your database server, please set the following options to utilize more CPU for your server:
+# max_worker_processes = 12
+# max_parallel_workers_per_gather = 4
+# max_parallel_workers = 12
+# max_parallel_maintenance_workers = 4
+
+log_connections = on
+log_disconnections = on
+log_replication_commands = on

files/postgres/replica/init.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf
+cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/baseconfig.conf
+
+
+rm -rf /var/lib/postgresql/data/*
+export PGPASSWORD='replicauser_password'
+pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R
+
+rm -rf /var/lib/postgresql/data/postgresql.conf
+
+cp /files/postgres/replica/baseconfig.conf /var/lib/postgresql/data/postgresql.conf
+
+exec pg_ctl -D /var/lib/postgresql/data start

files/postgres/replica/replica_1/init.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf
+echo "include '/files/postgres/replica/replica_1/replica_1_config.conf'" >> /var/lib/postgresql/data/postgresql.conf
+
+cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/replica_1/baseconfig.conf
+
+
+rm -rf /var/lib/postgresql/data/*
+export PGPASSWORD='replicauser_password'
+pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R
+
+rm -rf /var/lib/postgresql/data/postgresql.conf
+
+cp /files/postgres/replica/replica_1/baseconfig.conf /var/lib/postgresql/data/postgresql.conf
+
+exec pg_ctl -D /var/lib/postgresql/data start

files/postgres/replica/replica_1/replica_1_config.conf

@@ -0,0 +1,3 @@
+# replication slot on sending server
+# This needs to be configured using the docs below for "Keeping the replica and primary in sync."
+primary_slot_name = 'replica_1_slot'

files/postgres/replica/replica_2/init.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf
+echo "include '/files/postgres/replica/replica_2/replica_2_config.conf'" >> /var/lib/postgresql/data/postgresql.conf
+
+cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/replica_2/baseconfig.conf
+
+
+rm -rf /var/lib/postgresql/data/*
+export PGPASSWORD='replicauser_password'
+pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R
+
+rm -rf /var/lib/postgresql/data/postgresql.conf
+
+cp /files/postgres/replica/replica_2/baseconfig.conf /var/lib/postgresql/data/postgresql.conf
+
+exec pg_ctl -D /var/lib/postgresql/data start

files/postgres/replica/replica_2/replica_2_config.conf

@@ -0,0 +1,3 @@
+# replication slot on sending server
+# This needs to be configured using the docs below for "Keeping the replica and primary in sync."
+primary_slot_name = 'replica_2_slot'

files/postgres/replica/replica_config.conf

@@ -0,0 +1,64 @@
+# If the instance is lower capacity than r5.xlarge, then set it to a lower number.
+# Also tune the "MaxOpenConns" setting under the "SqlSettings" of the Mattermost app accordingly.
+# Note that "MaxOpenConns" on Mattermost is per data source name.
+max_connections = 1024
+
+# Set it to 1.1, unless the DB is using spinning disks.
+random_page_cost = 1.1
+
+# This should be 32MB if using read replicas, or 16MB if using a single PostgreSQL instance.
+# If the instance is of a lower capacity than r5.xlarge, then set it to a lower number.
+work_mem = 16MB
+
+# Set both of the below settings to 65% of total memory. For a 32 GB instance, it should be 21 GB.
+# If on a smaller server, set this to 20% or less total RAM.
+# ex: 512MB would work for a 4GB RAM server
+shared_buffers = 512MB
+effective_cache_size = 512MB
+
+# If you are using pgbouncer, or any similar connection pooling proxy,
+# in front of your DB, then apply the keepalive settings to the proxy instead,
+# and revert the keepalive settings for the DB back to defaults.
+tcp_keepalives_idle = 5
+tcp_keepalives_interval = 1
+tcp_keepalives_count = 5
+
+# 1GB (reduce this to 512MB if your server has less than 32GB of RAM)
+maintenance_work_mem = 512MB
+
+autovacuum_max_workers = 4
+autovacuum_vacuum_cost_limit = 500
+
+## Intentionally disabled here because running in docker this may cause performance problems.
+
+# If you have more than 32 CPUs on your database server, please set the following options to utilize more CPU for your server:
+# max_worker_processes = 12
+# max_parallel_workers_per_gather = 4
+# max_parallel_workers = 12
+# max_parallel_maintenance_workers = 4
+
+
+# If the instance is lower capacity than r5.xlarge, then set it to a lower number.
+# Also tune the "MaxOpenConns" setting under the "SqlSettings" of the Mattermost app accordingly.
+# Note that "MaxOpenConns" on Mattermost is per data source name.
+max_connections = 1024
+
+# This setting should be 16MB on read nodes, and 32MB on writer nodes
+work_mem = 16MB
+
+# The below settings allow the reader to return query results even when the primary has a write process running, a query conflict.
+# This is set to on because of the high volume of write traffic that can prevent the reader from returning query results within the timeout.
+# https://www.postgresql.org/docs/current/hot-standby.html#HOT-STANDBY-CONFLICT
+hot_standby = on
+hot_standby_feedback = on
+
+# connection string to sending server
+# This was created when you added a replication role to the primary database.
+# username - replace "test" with the role you made
+# password - replace "testpassword" with the role password
+# host - replace "x.x.x.x" with your IP or URL string.
+primary_conninfo = 'host=cs-repro-postgres port=5432 user=replicauser password=replicauser_password'
+
+log_connections = on
+log_disconnections = on
+log_replication_commands = on