diff --git a/.gitignore b/.gitignore index eaa6fe5..2a2aad7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ license.txt volumes **/.DS_Store -.DS_Store \ No newline at end of file +.DS_Store +**/baseConfig.conf \ No newline at end of file diff --git a/Makefile b/Makefile index 28d26e0..407dbab 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,10 @@ run: start: @make run +start-replicas: + @echo "Starting the replicas... hold on a moment..." + @docker-compose -f docker-compose.yml -f docker-compose-read-replicas.yml up -d + stop: @echo "Stopping..." @docker-compose stop diff --git a/README.md b/README.md index fd786ab..8acce9c 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ This is a basic reproduction that includes various components preconfigured like - [How to Downgrade](#how-to-downgrade) - [Migrating to Config in DB](#using-config-in-db) - [MMCTL](#mmctl) + - [Adding Postgres Read Replicas](#adding-postgres-read-replicas) ## Making Changes @@ -168,6 +169,16 @@ To use `mmctl` it's already setup for local, just run the below docker command. docker exec -it cs-repro-mattermost mmctl user list --local ``` +### Adding Postgres Read Replicas + +The basic structure for you to add two read replicas has been included in the repo already. To utilize it you need to start your files with the below command. This should work on a running install of this repo also. + +```bash +make start-replicas +``` + +This will take 2-5 minutes to get the replication setup, based on how much data you have in the database right now. + ## Use Grafana All the Mattermost grafana charts are already installed and linked, you just have to access them. diff --git a/docker-compose-read-replicas.yml b/docker-compose-read-replicas.yml new file mode 100644 index 0000000..e5607ff --- /dev/null +++ b/docker-compose-read-replicas.yml @@ -0,0 +1,59 @@ +services: + postgres-replica-1: + container_name: cs-repro-postgres-replica-1 + environment: + - POSTGRES_USER=mmuser + - POSTGRES_PASSWORD=mmuser_password + - LISTEN_ADDRESS="*" + image: postgres:14-alpine + restart: unless-stopped + ports: + - "5433:5432" + security_opt: + - no-new-privileges:true + pids_limit: 100 + read_only: false + tmpfs: + - /tmp + - /var/run/postgresql + volumes: + - ./files/postgres/replica/replica_1/init.sh:/docker-entrypoint-initdb.d/init.sh + - ./volumes/db/replica_1/data:/var/lib/postgresql/data + - ./files/postgres/replica:/files/postgres/replica + healthcheck: + test: pg_isready -U mmuser -d mattermost -h cs-repro-postgres + interval: 10s + timeout: 3s + retries: 3 + depends_on: + - mattermost + - postgres + postgres-replica-2: + container_name: cs-repro-postgres-replica-2 + environment: + - POSTGRES_USER=mmuser + - POSTGRES_PASSWORD=mmuser_password + - LISTEN_ADDRESS="*" + image: postgres:14-alpine + restart: unless-stopped + ports: + - "5434:5432" + security_opt: + - no-new-privileges:true + pids_limit: 100 + read_only: false + tmpfs: + - /tmp + - /var/run/postgresql + volumes: + - ./files/postgres/replica/replica_2/init.sh:/docker-entrypoint-initdb.d/init.sh + - ./files/postgres/replica:/files/postgres/replica + - ./volumes/db/replica_2/data:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U mmuser -d mattermost -h cs-repro-postgres + interval: 10s + timeout: 3s + retries: 3 + depends_on: + - mattermost + - postgres diff --git a/docker-compose.yml b/docker-compose.yml index a6862c3..1110823 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: - /tmp - /var/run/postgresql volumes: - - ./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data + - ./volumes/db/primary/data:/var/lib/postgresql/data - ./files/postgres/primary/init.sh:/docker-entrypoint-initdb.d/init.sh - ./files/postgres/primary/primary_config.conf:/files/postgres/primary/primary_config.conf healthcheck: diff --git a/files/postgres/primary/init.sh b/files/postgres/primary/init.sh index e4906df..aad9fc0 100755 --- a/files/postgres/primary/init.sh +++ b/files/postgres/primary/init.sh @@ -1,3 +1,10 @@ #!/bin/bash -echo "include '/files/postgres/primary/primary_config.conf'" >> /var/lib/postgresql/data/postgresql.conf \ No newline at end of file +echo "include '/files/postgres/primary/primary_config.conf'" >> /var/lib/postgresql/data/postgresql.conf + +psql -U mmuser -d mattermost -c "create role replicauser with replication password 'replicauser_password' login" +psql -U mmuser -d mattermost -c "select pg_create_physical_replication_slot('replica_2_slot');" +psql -U mmuser -d mattermost -c "select pg_create_physical_replication_slot('replica_1_slot');" + + +echo "host replication replicauser 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf diff --git a/files/postgres/primary/primary_config.conf b/files/postgres/primary/primary_config.conf index 157ed14..e5d132d 100644 --- a/files/postgres/primary/primary_config.conf +++ b/files/postgres/primary/primary_config.conf @@ -1,9 +1,14 @@ -# This value should match the "SqlSettings.MaxConnections" value within your config.json for Mattermost -# This is a suggestion and can be set lower / higher based on the size of your server. -max_connections = 1020 -tcp_keepalives_idle = 5 -tcp_keepalives_interval = 1 -tcp_keepalives_count = 5 +# If the instance is lower capacity than r5.xlarge, then set it to a lower number. +# Also tune the "MaxOpenConns" setting under the "SqlSettings" of the Mattermost app accordingly. +# Note that "MaxOpenConns" on Mattermost is per data source name. +max_connections = 1024 + +# Set it to 1.1, unless the DB is using spinning disks. +random_page_cost = 1.1 + +# This should be 32MB if using read replicas, or 16MB if using a single PostgreSQL instance. +# If the instance is of a lower capacity than r5.xlarge, then set it to a lower number. +work_mem = 16MB # Set both of the below settings to 65% of total memory. For a 32 GB instance, it should be 21 GB. # If on a smaller server, set this to 20% or less total RAM. @@ -11,15 +16,27 @@ tcp_keepalives_count = 5 shared_buffers = 512MB effective_cache_size = 512MB -# Set it to 16 MB for readers and 32 MB for writers. If it's a single instance, 16 MB should be sufficient. If the instance is of a lower capacity than r5.xlarge, then set it to a lower number. -work_mem = 16MB +# If you are using pgbouncer, or any similar connection pooling proxy, +# in front of your DB, then apply the keepalive settings to the proxy instead, +# and revert the keepalive settings for the DB back to defaults. +tcp_keepalives_idle = 5 +tcp_keepalives_interval = 1 +tcp_keepalives_count = 5 # 1GB (reduce this to 512MB if your server has less than 32GB of RAM) -autovacuum_work_mem = 512MB +maintenance_work_mem = 512MB + autovacuum_max_workers = 4 autovacuum_vacuum_cost_limit = 500 -#Set it to 1.1 unless the DB is using spinning disks. -random_page_cost = 1.1 +## Intentionally disabled here because running in docker this may cause performance problems. -restart_after_crash = on \ No newline at end of file +# If you have more than 32 CPUs on your database server, please set the following options to utilize more CPU for your server: +# max_worker_processes = 12 +# max_parallel_workers_per_gather = 4 +# max_parallel_workers = 12 +# max_parallel_maintenance_workers = 4 + +log_connections = on +log_disconnections = on +log_replication_commands = on diff --git a/files/postgres/replica/init.sh b/files/postgres/replica/init.sh new file mode 100755 index 0000000..46c1b2c --- /dev/null +++ b/files/postgres/replica/init.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf +cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/baseconfig.conf + + +rm -rf /var/lib/postgresql/data/* +export PGPASSWORD='replicauser_password' +pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R + +rm -rf /var/lib/postgresql/data/postgresql.conf + +cp /files/postgres/replica/baseconfig.conf /var/lib/postgresql/data/postgresql.conf + +exec pg_ctl -D /var/lib/postgresql/data start \ No newline at end of file diff --git a/files/postgres/replica/replica_1/init.sh b/files/postgres/replica/replica_1/init.sh new file mode 100755 index 0000000..f48f04f --- /dev/null +++ b/files/postgres/replica/replica_1/init.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf +echo "include '/files/postgres/replica/replica_1/replica_1_config.conf'" >> /var/lib/postgresql/data/postgresql.conf + +cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/replica_1/baseconfig.conf + + +rm -rf /var/lib/postgresql/data/* +export PGPASSWORD='replicauser_password' +pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R + +rm -rf /var/lib/postgresql/data/postgresql.conf + +cp /files/postgres/replica/replica_1/baseconfig.conf /var/lib/postgresql/data/postgresql.conf + +exec pg_ctl -D /var/lib/postgresql/data start \ No newline at end of file diff --git a/files/postgres/replica/replica_1/replica_1_config.conf b/files/postgres/replica/replica_1/replica_1_config.conf new file mode 100644 index 0000000..5bb1fc2 --- /dev/null +++ b/files/postgres/replica/replica_1/replica_1_config.conf @@ -0,0 +1,3 @@ +# replication slot on sending server +# This needs to be configured using the docs below for "Keeping the replica and primary in sync." +primary_slot_name = 'replica_1_slot' \ No newline at end of file diff --git a/files/postgres/replica/replica_2/init.sh b/files/postgres/replica/replica_2/init.sh new file mode 100755 index 0000000..31adaef --- /dev/null +++ b/files/postgres/replica/replica_2/init.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf +echo "include '/files/postgres/replica/replica_2/replica_2_config.conf'" >> /var/lib/postgresql/data/postgresql.conf + +cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/replica_2/baseconfig.conf + + +rm -rf /var/lib/postgresql/data/* +export PGPASSWORD='replicauser_password' +pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R + +rm -rf /var/lib/postgresql/data/postgresql.conf + +cp /files/postgres/replica/replica_2/baseconfig.conf /var/lib/postgresql/data/postgresql.conf + +exec pg_ctl -D /var/lib/postgresql/data start \ No newline at end of file diff --git a/files/postgres/replica/replica_2/replica_2_config.conf b/files/postgres/replica/replica_2/replica_2_config.conf new file mode 100644 index 0000000..e0fc88f --- /dev/null +++ b/files/postgres/replica/replica_2/replica_2_config.conf @@ -0,0 +1,3 @@ +# replication slot on sending server +# This needs to be configured using the docs below for "Keeping the replica and primary in sync." +primary_slot_name = 'replica_2_slot' \ No newline at end of file diff --git a/files/postgres/replica/replica_config.conf b/files/postgres/replica/replica_config.conf new file mode 100644 index 0000000..6977f01 --- /dev/null +++ b/files/postgres/replica/replica_config.conf @@ -0,0 +1,64 @@ +# If the instance is lower capacity than r5.xlarge, then set it to a lower number. +# Also tune the "MaxOpenConns" setting under the "SqlSettings" of the Mattermost app accordingly. +# Note that "MaxOpenConns" on Mattermost is per data source name. +max_connections = 1024 + +# Set it to 1.1, unless the DB is using spinning disks. +random_page_cost = 1.1 + +# This should be 32MB if using read replicas, or 16MB if using a single PostgreSQL instance. +# If the instance is of a lower capacity than r5.xlarge, then set it to a lower number. +work_mem = 16MB + +# Set both of the below settings to 65% of total memory. For a 32 GB instance, it should be 21 GB. +# If on a smaller server, set this to 20% or less total RAM. +# ex: 512MB would work for a 4GB RAM server +shared_buffers = 512MB +effective_cache_size = 512MB + +# If you are using pgbouncer, or any similar connection pooling proxy, +# in front of your DB, then apply the keepalive settings to the proxy instead, +# and revert the keepalive settings for the DB back to defaults. +tcp_keepalives_idle = 5 +tcp_keepalives_interval = 1 +tcp_keepalives_count = 5 + +# 1GB (reduce this to 512MB if your server has less than 32GB of RAM) +maintenance_work_mem = 512MB + +autovacuum_max_workers = 4 +autovacuum_vacuum_cost_limit = 500 + +## Intentionally disabled here because running in docker this may cause performance problems. + +# If you have more than 32 CPUs on your database server, please set the following options to utilize more CPU for your server: +# max_worker_processes = 12 +# max_parallel_workers_per_gather = 4 +# max_parallel_workers = 12 +# max_parallel_maintenance_workers = 4 + + +# If the instance is lower capacity than r5.xlarge, then set it to a lower number. +# Also tune the "MaxOpenConns" setting under the "SqlSettings" of the Mattermost app accordingly. +# Note that "MaxOpenConns" on Mattermost is per data source name. +max_connections = 1024 + +# This setting should be 16MB on read nodes, and 32MB on writer nodes +work_mem = 16MB + +# The below settings allow the reader to return query results even when the primary has a write process running, a query conflict. +# This is set to on because of the high volume of write traffic that can prevent the reader from returning query results within the timeout. +# https://www.postgresql.org/docs/current/hot-standby.html#HOT-STANDBY-CONFLICT +hot_standby = on +hot_standby_feedback = on + +# connection string to sending server +# This was created when you added a replication role to the primary database. +# username - replace "test" with the role you made +# password - replace "testpassword" with the role password +# host - replace "x.x.x.x" with your IP or URL string. +primary_conninfo = 'host=cs-repro-postgres port=5432 user=replicauser password=replicauser_password' + +log_connections = on +log_disconnections = on +log_replication_commands = on