diff --git a/.gitignore b/.gitignore index 2a2aad7..c9c6911 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ license.txt ## Ignoring keycloak so it can be pulled already setup. volumes +## We hate DS_Store files. **/.DS_Store .DS_Store -**/baseConfig.conf \ No newline at end of file + +files/postgres/replica/replica_1.conf +files/postgres/replica/replica_1_postgres.conf + +files/postgres/replica/replica_2.conf +files/postgres/replica/replica_2_postgres.conf diff --git a/Makefile b/Makefile index 407dbab..f5a7790 100644 --- a/Makefile +++ b/Makefile @@ -31,8 +31,14 @@ start: @make run start-replicas: + @make run @echo "Starting the replicas... hold on a moment..." @docker-compose -f docker-compose.yml -f docker-compose-read-replicas.yml up -d + @docker exec -it cs-repro-mattermost mmctl config patch /mattermost/config/replicaConfig.json --local + @echo "Sleeping for 2 minutes while the replication is established. Be back in a moment..." + @sleep 120 + @make restart-mattermost + @echo "Should be up and running. Go crazy." stop: @echo "Stopping..." @@ -69,6 +75,7 @@ delete-dockerfiles: @echo "Deleting data..." @docker-compose rm @rm -rf ./volumes + @rm -rf ./files/postgres/replica/replica_* @echo "Done" delete-data: stop delete-dockerfiles diff --git a/README.md b/README.md index 8acce9c..4e4c6f9 100644 --- a/README.md +++ b/README.md @@ -171,13 +171,29 @@ docker exec -it cs-repro-mattermost mmctl user list --local ### Adding Postgres Read Replicas -The basic structure for you to add two read replicas has been included in the repo already. To utilize it you need to start your files with the below command. This should work on a running install of this repo also. +The basic structure for you to add two read replicas has been included in the repo already. This will take 2-5 minutes to get the replication setup, based on how much data you have in the database right now. + +If you are starting from fresh run: ```bash make start-replicas ``` -This will take 2-5 minutes to get the replication setup, based on how much data you have in the database right now. +If you want to add replicas to an existing cs repro: + +```bash +make start-replicas +``` + +#### Replication Config and access + +All replica config files can be found in `./files/postgres/replica`. You can edit the `replica_x.conf` file to edit the specific configuration for a replica. You will need to restart the replicas once down, easiest way is `make stop && make start` + +You can access each replica with the same username / password. Just need to change the port. Here's the output when using `postgresql`. Note if you use this in the mattermost config you need to replace `postgresql` to `postgres`. + +- primary - `postgresql://mmuser:mmuser_password@localhost:5432/mattermost` +- replica_1 - `postgresql://mmuser:mmuser_password@localhost:5433/mattermost` +- replica_2 - `postgresql://mmuser:mmuser_password@localhost:5434/mattermost` ## Use Grafana diff --git a/docker-compose-read-replicas.yml b/docker-compose-read-replicas.yml index e5607ff..617e147 100644 --- a/docker-compose-read-replicas.yml +++ b/docker-compose-read-replicas.yml @@ -5,6 +5,7 @@ services: - POSTGRES_USER=mmuser - POSTGRES_PASSWORD=mmuser_password - LISTEN_ADDRESS="*" + - REPLICA_NAME=replica_1 image: postgres:14-alpine restart: unless-stopped ports: @@ -17,14 +18,9 @@ services: - /tmp - /var/run/postgresql volumes: - - ./files/postgres/replica/replica_1/init.sh:/docker-entrypoint-initdb.d/init.sh + - ./files/postgres/replica/init.sh:/docker-entrypoint-initdb.d/init.sh - ./volumes/db/replica_1/data:/var/lib/postgresql/data - ./files/postgres/replica:/files/postgres/replica - healthcheck: - test: pg_isready -U mmuser -d mattermost -h cs-repro-postgres - interval: 10s - timeout: 3s - retries: 3 depends_on: - mattermost - postgres @@ -34,6 +30,7 @@ services: - POSTGRES_USER=mmuser - POSTGRES_PASSWORD=mmuser_password - LISTEN_ADDRESS="*" + - REPLICA_NAME=replica_2 image: postgres:14-alpine restart: unless-stopped ports: @@ -46,14 +43,9 @@ services: - /tmp - /var/run/postgresql volumes: - - ./files/postgres/replica/replica_2/init.sh:/docker-entrypoint-initdb.d/init.sh + - ./files/postgres/replica/init.sh:/docker-entrypoint-initdb.d/init.sh - ./files/postgres/replica:/files/postgres/replica - ./volumes/db/replica_2/data:/var/lib/postgresql/data - healthcheck: - test: pg_isready -U mmuser -d mattermost -h cs-repro-postgres - interval: 10s - timeout: 3s - retries: 3 depends_on: - mattermost - postgres diff --git a/docker-compose.yml b/docker-compose.yml index 1110823..8c4f9e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -88,6 +88,7 @@ services: - ./files/mattermost/samlCert.crt:/mattermost/config/saml-cert.crt - ./files/mattermost/advancedLogging.json:/mattermost/config/advancedLogging.json:ro - ./files/mattermost/defaultConfig.json:/mattermost/config/defaultConfig.json:ro + - ./files/mattermost/replicaConfig.json:/mattermost/config/replicaConfig.json:ro environment: - MM_SqlSettings_DriverName=postgres - MM_SqlSettings_DataSource=postgres://mmuser:mmuser_password@cs-repro-postgres:5432/mattermost?sslmode=disable&connect_timeout=10&binary_parameters=yes diff --git a/files/mattermost/replicaConfig.json b/files/mattermost/replicaConfig.json new file mode 100644 index 0000000..22a4d42 --- /dev/null +++ b/files/mattermost/replicaConfig.json @@ -0,0 +1,8 @@ +{ + "SqlSettings": { + "DataSourceReplicas": [ + "postgres://mmuser:mmuser_password@cs-repro-postgres-replica-1:5432/mattermost?sslmode=disable&connect_timeout=10&binary_parameters=yes", + "postgres://mmuser:mmuser_password@cs-repro-postgres-replica-2:5432/mattermost?sslmode=disable&connect_timeout=10&binary_parameters=yes" + ] + } +} \ No newline at end of file diff --git a/files/postgres/replica/replica_config.conf b/files/postgres/replica/base_replica_config.conf similarity index 94% rename from files/postgres/replica/replica_config.conf rename to files/postgres/replica/base_replica_config.conf index 6977f01..0738418 100644 --- a/files/postgres/replica/replica_config.conf +++ b/files/postgres/replica/base_replica_config.conf @@ -59,6 +59,10 @@ hot_standby_feedback = on # host - replace "x.x.x.x" with your IP or URL string. primary_conninfo = 'host=cs-repro-postgres port=5432 user=replicauser password=replicauser_password' +# replication slot on sending server +# This needs to be configured using the docs below for "Keeping the replica and primary in sync." +primary_slot_name = '' + log_connections = on log_disconnections = on log_replication_commands = on diff --git a/files/postgres/replica/init.sh b/files/postgres/replica/init.sh index 46c1b2c..9c514d4 100755 --- a/files/postgres/replica/init.sh +++ b/files/postgres/replica/init.sh @@ -2,16 +2,23 @@ set -e -echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf -cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/baseconfig.conf +## copying the base replica configuration and updating the primary slot name +cp /files/postgres/replica/base_replica_config.conf /files/postgres/replica/${REPLICA_NAME}.conf +sed -i "s/primary_slot_name = ''/primary_slot_name = '${REPLICA_NAME}_slot'/" /files/postgres/replica/${REPLICA_NAME}.conf +## copying the default postgres config and adding the replica specific configuration +echo "include '/files/postgres/replica/${REPLICA_NAME}.conf'" >> /var/lib/postgresql/data/postgresql.conf +cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/${REPLICA_NAME}_postgres.conf +## Removing the pg data information to make way for the backup rm -rf /var/lib/postgresql/data/* export PGPASSWORD='replicauser_password' pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R +## Backup brings over the primary config, so we do not need it anymore. rm -rf /var/lib/postgresql/data/postgresql.conf -cp /files/postgres/replica/baseconfig.conf /var/lib/postgresql/data/postgresql.conf +## Bringing back in the replica specific config. +cp /files/postgres/replica/${REPLICA_NAME}_postgres.conf /var/lib/postgresql/data/postgresql.conf exec pg_ctl -D /var/lib/postgresql/data start \ No newline at end of file diff --git a/files/postgres/replica/replica_1/init.sh b/files/postgres/replica/replica_1/init.sh deleted file mode 100755 index f48f04f..0000000 --- a/files/postgres/replica/replica_1/init.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf -echo "include '/files/postgres/replica/replica_1/replica_1_config.conf'" >> /var/lib/postgresql/data/postgresql.conf - -cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/replica_1/baseconfig.conf - - -rm -rf /var/lib/postgresql/data/* -export PGPASSWORD='replicauser_password' -pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R - -rm -rf /var/lib/postgresql/data/postgresql.conf - -cp /files/postgres/replica/replica_1/baseconfig.conf /var/lib/postgresql/data/postgresql.conf - -exec pg_ctl -D /var/lib/postgresql/data start \ No newline at end of file diff --git a/files/postgres/replica/replica_1/replica_1_config.conf b/files/postgres/replica/replica_1/replica_1_config.conf deleted file mode 100644 index 5bb1fc2..0000000 --- a/files/postgres/replica/replica_1/replica_1_config.conf +++ /dev/null @@ -1,3 +0,0 @@ -# replication slot on sending server -# This needs to be configured using the docs below for "Keeping the replica and primary in sync." -primary_slot_name = 'replica_1_slot' \ No newline at end of file diff --git a/files/postgres/replica/replica_2/init.sh b/files/postgres/replica/replica_2/init.sh deleted file mode 100755 index 31adaef..0000000 --- a/files/postgres/replica/replica_2/init.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -echo "include '/files/postgres/replica/replica_config.conf'" >> /var/lib/postgresql/data/postgresql.conf -echo "include '/files/postgres/replica/replica_2/replica_2_config.conf'" >> /var/lib/postgresql/data/postgresql.conf - -cp /var/lib/postgresql/data/postgresql.conf /files/postgres/replica/replica_2/baseconfig.conf - - -rm -rf /var/lib/postgresql/data/* -export PGPASSWORD='replicauser_password' -pg_basebackup -h cs-repro-postgres -p 5432 -U replicauser -D /var/lib/postgresql/data -Fp -Xs -R - -rm -rf /var/lib/postgresql/data/postgresql.conf - -cp /files/postgres/replica/replica_2/baseconfig.conf /var/lib/postgresql/data/postgresql.conf - -exec pg_ctl -D /var/lib/postgresql/data start \ No newline at end of file diff --git a/files/postgres/replica/replica_2/replica_2_config.conf b/files/postgres/replica/replica_2/replica_2_config.conf deleted file mode 100644 index e0fc88f..0000000 --- a/files/postgres/replica/replica_2/replica_2_config.conf +++ /dev/null @@ -1,3 +0,0 @@ -# replication slot on sending server -# This needs to be configured using the docs below for "Keeping the replica and primary in sync." -primary_slot_name = 'replica_2_slot' \ No newline at end of file