From 7e58663cb6fc14b203c857dad736c98d405377c3 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Mon, 28 Jun 2021 16:18:59 +0200 Subject: [PATCH 01/13] Modify stolen upgrade script from here: https://github.com/mattermost/mattermost-docker/issues/489#issuecomment-790277661 --- scripts/upgrade-from-mattermost-docker.sh | 155 ++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100755 scripts/upgrade-from-mattermost-docker.sh diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh new file mode 100755 index 0000000..90d73c0 --- /dev/null +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +## +## Instructions +## + +# 1. Edit the variables below to match your environment. This uses default variables and assumes you're on 5.31.0. +# If you're wanting to use another version of Postgres/Mattermost , update the variables as desired. + +# 2. run 'sudo bash upgrade-from-mattermost-docker.sh' replace upgrade.sh with what you've named the file. +# This may take some time to complete as it's migrating the database to Postgres 13.6 from 9.4 + + +## +## Environment Variables +## +PATH_TO_MATTERMOST_DOCKER=$PATH_TO_MATTERMOST_DOCKER # i.e. $PWD + +# Below is default values in the mattermost-docker container. If you've edited these you will need +# to modify them before running the script or this will generate a new database. +POSTGRES_USER=$POSTGRES_USER # i.e. mmuser +POSTGRES_PASSWORD=$POSTGRES_PASSWORD # i.e. mmuser_password +POSTGRES_DB=$POSTGRES_DB # i.e. mattermost + +# You should be on Postgres 9.4. To get your version run +# 'sudo cat volumes/db/var/lib/postgresql/data/PG_VERSION' to confirm this. +POSTGRES_OLD_VERSION=$POSTGRES_OLD_VERSION # i.e. 9.4 +POSTGRES_NEW_VERSION=$POSTGRES_NEW_VERSION # i.e. 13 + +# This tag is found here - https://hub.docker.com/_/postgres' +# This tag needs to be an alpine release to include python3-dev +POSTGRES_DOCKER_TAG=$POSTGRES_DOCKER_TAG # i.e. '13.2-alpine' +POSTGRES_OLD_DOCKERFILE=$POSTGRES_OLD_DOCKERFILE # i.e. `sudo cat $PATH_TO_MATTERMOST_DOCKER/db/Dockerfile | grep 'FROM postgres'` +POSTGRES_NEW_DOCKERFILE=$POSTGRES_NEW_VERSION # i.e. 'FROM postgres:'$POSTGRES_DOCKER_TAG + + +# This is found here - https://github.com/tianon/docker-postgres-upgrade +# The string here needs to match a folder on that repo. It should read 'old-to-new'. +UPGRADE_LINE=$UPGRADE_LINE # i.e. '9.4-to-13' + +# Mattermost Versions +MM_OLD_VERSION=$MM_OLD_VERSION # i.e. "5.31.0" +MM_NEW_VERSION=$MM_NEW_VERSION # i.e. "5.32.1" + +if [[ $PATH_TO_MATTERMOST_DOCKER == "" ]]; then + echo "Please set environment variable PATH_TO_MATTERMOST_DOCKER in the script. " + exit 1 +fi +if [[ $POSTGRES_USER == "" ]]; then + echo "Please set environment variable POSTGRES_USER in the script. " + exit 1 +fi +if [[ $POSTGRES_PASSWORD == "" ]]; then + echo "Please set environment variable POSTGRES_PASSWORD in the script. " + exit 1 +fi +if [[ $POSTGRES_DB == "" ]]; then + echo "Please set environment variable POSTGRES_DB in the script. " + exit 1 +fi +if [[ $POSTGRES_OLD_VERSION == "" ]]; then + echo "Please set environment variable POSTGRES_OLD_VERSION in the script. " + exit 1 +fi +if [[ $POSTGRES_NEW_VERSION == "" ]]; then + echo "Please set environment variable POSTGRES_NEW_VERSION in the script. " + exit 1 +fi +if [[ $POSTGRES_DOCKER_TAG == "" ]]; then + echo "Please set environment variable POSTGRES_DOCKER_TAG in the script. " + exit 1 +fi +if [[ $POSTGRES_OLD_DOCKERFILE == "" ]]; then + echo "Please set environment variable POSTGRES_OLD_DOCKERFILE in the script. " + exit 1 +fi +if [[ $POSTGRES_NEW_DOCKERFILE == "" ]]; then + echo "Please set environment variable POSTGRES_NEW_DOCKERFILE in the script. " + exit 1 +fi +if [[ $UPGRADE_LINE == "" ]]; then + echo "Please set environment variable UPGRADE_LINE in the script. " + exit 1 +fi +if [[ $MM_OLD_VERSION == "" ]]; then + echo "Please set environment variable MM_OLD_VERSION in the script. " + exit 1 +fi +if [[ $MM_NEW_VERSION == "" ]]; then + echo "Please set environment variable MM_NEW_VERSION in the script. " + exit 1 +fi +echo "Path to mattermost-docker: $PATH_TO_MATTERMOST_DOCKER" +echo "Postgres user: $POSTGRES_USER" +echo "Postgres password: $POSTGRES_PASSWORD" +echo "Postgres password: $POSTGRES_DB" +echo "Postgres database name: $POSTGRES_DB" +echo "Postgres old version: $POSTGRES_OLD_VERSION" +echo "Postgres new version: $POSTGRES_NEW_VERSION" +echo "Postgres alpine docker tag including python3-dev: $POSTGRES_DOCKER_TAG" +echo "Postgres old Dockerfile: $POSTGRES_OLD_DOCKERFILE" +echo "Postgres new Dockerfile: $POSTGRES_NEW_DOCKERFILE" +echo "Postgres upgrade-line: $UPGRADE_LINE" +echo "Mattermost old version: $MM_OLD_VERSION" +echo "Mattermost new version: $MM_NEW_VERSION" +df -h +read -p "Please make sure you have enough disk space left on your devices. Try to backup and upgrade now? (y/n)" choice +if [[ "$choice" != "y" && "$choice" != "Y" && "$choice" && "yes" ]]; then + exit 0; +fi + +## +## Script Start +## +cd $PATH_TO_MATTERMOST_DOCKER +docker-compose stop + +# Creating a backup folder and backing up the mattermost / database. +mkdir $PATH_TO_MATTERMOST_DOCKER/backups +cp -ra $PATH_TO_MATTERMOST_DOCKER/volumes/app/mattermost/ $PATH_TO_MATTERMOST_DOCKER/backups/mattermost-backup-$(date +'%F-%H-%M')/ +cp -ra $PATH_TO_MATTERMOST_DOCKER/volumes/db/ $PATH_TO_MATTERMOST_DOCKER/backups/database-backup-$(date +'%F-%H-%M')/ + +mkdir $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_OLD_VERSION +mv $PATH_TO_MATTERMOST_DOCKER/volumes/db/var/lib/postgresql/data/ $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_OLD_VERSION +rm -rf $PATH_TO_MATTERMOST_DOCKER/volumes/db/var +mkdir -p $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_NEW_VERSION/data + + +sed -i "s/$POSTGRES_OLD_DOCKERFILE/$POSTGRES_NEW_DOCKERFILE/" $PATH_TO_MATTERMOST_DOCKER/db/Dockerfile +sed -i "s/python-dev/python3-dev/" $PATH_TO_MATTERMOST_DOCKER/db/Dockerfile +sed -i "s/$MM_OLD_VERSION/$MM_NEW_VERSION/" $PATH_TO_MATTERMOST_DOCKER/app/Dockerfile + + +# replacing the old postgres path with a new path +sed -i "s#./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data#./volumes/db/$POSTGRES_NEW_VERSION/data:/var/lib/postgresql/data#" $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml + +# migrate the database to the new postgres version +docker run --rm \ + -e PGUSER="$POSTGRES_USER" \ + -e POSTGRES_INITDB_ARGS=" -U $POSTGRES_USER" \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + -e POSTGRES_DB="$POSTGRES_DB" \ + -v $PATH_TO_MATTERMOST_DOCKER/volumes/db:/var/lib/postgresql \ + tianon/postgres-upgrade:$UPGRADE_LINE \ + --link + +cp -p $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_OLD_VERSION/data/pg_hba.conf $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_NEW_VERSION/data/ + +# rebuild the containers +docker-compose build +docker-compose up -d + +# reindex the database +echo "REINDEX SCHEMA CONCURRENTLY public;" | docker exec mattermost-docker_db_1 psql -U $POSTGRES_USER $POSTGRES_DB +cd - From d3e1c199243aa9c71dfa5b9b02887babab5877f5 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Mon, 28 Jun 2021 16:20:13 +0200 Subject: [PATCH 02/13] Fix typo. --- scripts/upgrade-from-mattermost-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh index 90d73c0..1ca593b 100755 --- a/scripts/upgrade-from-mattermost-docker.sh +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -16,7 +16,7 @@ ## PATH_TO_MATTERMOST_DOCKER=$PATH_TO_MATTERMOST_DOCKER # i.e. $PWD -# Below is default values in the mattermost-docker container. If you've edited these you will need +# Below are default values in the mattermost-docker container. If you've edited these you will need # to modify them before running the script or this will generate a new database. POSTGRES_USER=$POSTGRES_USER # i.e. mmuser POSTGRES_PASSWORD=$POSTGRES_PASSWORD # i.e. mmuser_password From 35b7f6fe1ad696b19bcb0eb2eef3a2bddbaedcd1 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Mon, 28 Jun 2021 16:43:52 +0200 Subject: [PATCH 03/13] Add -e --- scripts/upgrade-from-mattermost-docker.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh index 1ca593b..e94c143 100755 --- a/scripts/upgrade-from-mattermost-docker.sh +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + ## ## Instructions ## From 8bc3e6c8ba33047a330a36fa23b687b4fd74c5ab Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Mon, 28 Jun 2021 17:02:29 +0200 Subject: [PATCH 04/13] Consistency --- scripts/upgrade-from-mattermost-docker.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh index e94c143..e21500d 100755 --- a/scripts/upgrade-from-mattermost-docker.sh +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -38,7 +38,7 @@ POSTGRES_NEW_DOCKERFILE=$POSTGRES_NEW_VERSION # i.e. 'FROM postgres:'$POSTGRES_D # This is found here - https://github.com/tianon/docker-postgres-upgrade # The string here needs to match a folder on that repo. It should read 'old-to-new'. -UPGRADE_LINE=$UPGRADE_LINE # i.e. '9.4-to-13' +POSTGRES_UPGRADE_LINE=$POSTGRES_UPGRADE_LINE # i.e. '9.4-to-13' # Mattermost Versions MM_OLD_VERSION=$MM_OLD_VERSION # i.e. "5.31.0" @@ -80,8 +80,8 @@ if [[ $POSTGRES_NEW_DOCKERFILE == "" ]]; then echo "Please set environment variable POSTGRES_NEW_DOCKERFILE in the script. " exit 1 fi -if [[ $UPGRADE_LINE == "" ]]; then - echo "Please set environment variable UPGRADE_LINE in the script. " +if [[ $POSTGRES_UPGRADE_LINE == "" ]]; then + echo "Please set environment variable POSTGRES_UPGRADE_LINE in the script. " exit 1 fi if [[ $MM_OLD_VERSION == "" ]]; then @@ -102,7 +102,7 @@ echo "Postgres new version: $POSTGRES_NEW_VERSION" echo "Postgres alpine docker tag including python3-dev: $POSTGRES_DOCKER_TAG" echo "Postgres old Dockerfile: $POSTGRES_OLD_DOCKERFILE" echo "Postgres new Dockerfile: $POSTGRES_NEW_DOCKERFILE" -echo "Postgres upgrade-line: $UPGRADE_LINE" +echo "Postgres upgrade-line: $POSTGRES_UPGRADE_LINE" echo "Mattermost old version: $MM_OLD_VERSION" echo "Mattermost new version: $MM_NEW_VERSION" df -h @@ -143,7 +143,7 @@ docker run --rm \ -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ -e POSTGRES_DB="$POSTGRES_DB" \ -v $PATH_TO_MATTERMOST_DOCKER/volumes/db:/var/lib/postgresql \ - tianon/postgres-upgrade:$UPGRADE_LINE \ + tianon/postgres-upgrade:$POSTGRES_UPGRADE_LINE \ --link cp -p $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_OLD_VERSION/data/pg_hba.conf $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_NEW_VERSION/data/ From dceb5bb493c585630505d6c4d41497d7174f7d46 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Tue, 13 Jul 2021 11:55:51 +0200 Subject: [PATCH 05/13] Improvements in usability. --- scripts/README.md | 31 ++++ scripts/upgrade-from-mattermost-docker.sh | 199 +++++++++++++--------- 2 files changed, 150 insertions(+), 80 deletions(-) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..17c6830 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,31 @@ +## Upgrading from mattermost-docker + +``` +$ export PATH_TO_MATTERMOST_DOCKER=path/to/mattermost-docker +$ ./scripts/upgrade-from-mattermost-docker.sh +``` + +Environment variables for upgrading: +`ttf` means, the script 'tries to find' the environment variables. + +| Name | Description | Type | Default | Required | +|------|-------------|------|:---------:|:--------:| +| PATH_TO_MATTERMOST_DOCKER | absolute path to your mattermost-docker folder | `string` | n/a | yes | +| POSTGRES_USER | postgres user to connect to the mattermost database | `string` | ttf | yes | +| POSTGRES_PASSWORD | postgres password for the POSTGRES_USER to connect to the mattermost database | `string` | ttf | yes | +| POSTGRES_DB | postgres database name for the mattermost database | `string` | ttf | yes | +| POSTGRES_OLD_VERSION | postgres database old version which should be upgraded from | `semver` | ttf | yes | +| POSTGRES_NEW_VERSION | postgres database new version which should be upgraded to | `semver` | 13 | yes | +| POSTGRES_DOCKER_TAG | postgres docker tag found [here](https://hub.docker.com/_/postgres) including python3-dev | `string` | 13.2-alpine | yes | +| POSTGRES_OLD_DOCKER_FROM | FROM declaration in the postgres Dockerfile to be replaced | `string` | ttf | yes | +| POSTGRES_NEW_DOCKER_FROM | FROM declaration in the postgres Dockerfile replacing POSTGRES_OLD_DOCKER_FROM | `string` | ttf | yes | +| POSTGRES_UPGRADE_LINE | folder name required to upgrade postgres (Needs to match a folder [here](https://github.com/tianon/docker-postgres-upgrade)) | `string` | ttf | yes | +| MM_OLD_VERSION | mattermost old version which should be upgraded from | `semver` | ttf | yes | +| MM_NEW_VERSION | mattermost new version which should be upgraded to | `semver` | 5.32.1 | yes | + +You can overwrite any of these variables before running this script with: +``` +$ export VAR_NAME_FROM_ABOVE=yourValue +$ export PATH_TO_MATTERMOST_DOCKER=path/to/mattermost-docker +$ ./scripts/upgrade-from-mattermost-docker.sh +``` diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh index e21500d..53985ff 100755 --- a/scripts/upgrade-from-mattermost-docker.sh +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -5,6 +5,7 @@ set -e ## ## Instructions ## +# Dockerfile stolen from contributions in this issue: # 1. Edit the variables below to match your environment. This uses default variables and assumes you're on 5.31.0. # If you're wanting to use another version of Postgres/Mattermost , update the variables as desired. @@ -13,128 +14,166 @@ set -e # This may take some time to complete as it's migrating the database to Postgres 13.6 from 9.4 +PATH_TO_MATTERMOST_DOCKER=$PATH_TO_MATTERMOST_DOCKER # i.e. $PWD +if [[ $PATH_TO_MATTERMOST_DOCKER == "" ]]; then + echo "Please export environment variable PATH_TO_MATTERMOST_DOCKER with '$ export PATH_TO_MATTERMOST_DOCKER=/path/to/mattermost-docker' before running this script. " + exit 1 +fi + ## ## Environment Variables ## -PATH_TO_MATTERMOST_DOCKER=$PATH_TO_MATTERMOST_DOCKER # i.e. $PWD - -# Below are default values in the mattermost-docker container. If you've edited these you will need -# to modify them before running the script or this will generate a new database. -POSTGRES_USER=$POSTGRES_USER # i.e. mmuser -POSTGRES_PASSWORD=$POSTGRES_PASSWORD # i.e. mmuser_password -POSTGRES_DB=$POSTGRES_DB # i.e. mattermost - -# You should be on Postgres 9.4. To get your version run -# 'sudo cat volumes/db/var/lib/postgresql/data/PG_VERSION' to confirm this. -POSTGRES_OLD_VERSION=$POSTGRES_OLD_VERSION # i.e. 9.4 -POSTGRES_NEW_VERSION=$POSTGRES_NEW_VERSION # i.e. 13 - -# This tag is found here - https://hub.docker.com/_/postgres' -# This tag needs to be an alpine release to include python3-dev -POSTGRES_DOCKER_TAG=$POSTGRES_DOCKER_TAG # i.e. '13.2-alpine' -POSTGRES_OLD_DOCKERFILE=$POSTGRES_OLD_DOCKERFILE # i.e. `sudo cat $PATH_TO_MATTERMOST_DOCKER/db/Dockerfile | grep 'FROM postgres'` -POSTGRES_NEW_DOCKERFILE=$POSTGRES_NEW_VERSION # i.e. 'FROM postgres:'$POSTGRES_DOCKER_TAG - - -# This is found here - https://github.com/tianon/docker-postgres-upgrade -# The string here needs to match a folder on that repo. It should read 'old-to-new'. -POSTGRES_UPGRADE_LINE=$POSTGRES_UPGRADE_LINE # i.e. '9.4-to-13' - -# Mattermost Versions -MM_OLD_VERSION=$MM_OLD_VERSION # i.e. "5.31.0" -MM_NEW_VERSION=$MM_NEW_VERSION # i.e. "5.32.1" - -if [[ $PATH_TO_MATTERMOST_DOCKER == "" ]]; then - echo "Please set environment variable PATH_TO_MATTERMOST_DOCKER in the script. " - exit 1 -fi +# Below are default values in the mattermost-docker container. +# The script is trying to fetch those variables first. Should fetching fail, please export the variables before running the script. if [[ $POSTGRES_USER == "" ]]; then - echo "Please set environment variable POSTGRES_USER in the script. " - exit 1 + echo "trying to fetch POSTGRES_USER from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + POSTGRES_USER=$(echo "$(grep "^.*-.*POSTGRES_USER=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~^.*-.*POSTGRES_USER=~~g) + if [[ $POSTGRES_USER == "" ]]; then + echo "could not find POSTGRES_USER set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + echo "please run 'export POSTGRES_USER=yourPostgresUser' before running this script" + exit 1 + fi + echo "found POSTGRES_USER=redacted" fi + if [[ $POSTGRES_PASSWORD == "" ]]; then - echo "Please set environment variable POSTGRES_PASSWORD in the script. " - exit 1 + echo "trying to fetch POSTGRES_PASSWORD from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + POSTGRES_PASSWORD=$(echo "$(grep "^.*-.*POSTGRES_PASSWORD=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~^.*-.*POSTGRES_PASSWORD=~~g) + if [[ $POSTGRES_PASSWORD == "" ]]; then + echo "could not find POSTGRES_PASSWORD set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + echo "please run 'export POSTGRES_PASSWORD=yourPostgresPassword' before running this script" + exit 1 + fi + echo "found POSTGRES_PASSWORD=redacted" fi + if [[ $POSTGRES_DB == "" ]]; then - echo "Please set environment variable POSTGRES_DB in the script. " - exit 1 + echo "trying to fetch POSTGRES_DB from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + POSTGRES_DB=$(echo "$(grep "^.*-.*POSTGRES_DB=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~^.*-.*POSTGRES_DB=~~g) + if [[ $POSTGRES_DB == "" ]]; then + echo "could not find POSTGRES_DB set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + echo "please run 'export POSTGRES_DB=yourPostgresDatabase' before running this script" + exit 1 + fi + echo "found POSTGRES_DB=$POSTGRES_DB" fi + +printf "\n" if [[ $POSTGRES_OLD_VERSION == "" ]]; then - echo "Please set environment variable POSTGRES_OLD_VERSION in the script. " - exit 1 + echo "trying to fetch POSTGRES_OLD_VERSION by connecting to database container and echoing the environment variable PG_VERSION" + POSTGRES_OLD_VERSION=$(docker exec mattermost-docker_db_1 bash -c 'echo $PG_VERSION') # i.e. 9.4 + if [[ $POSTGRES_OLD_VERSION == "" ]]; then + echo "could not connect to database container to get PG_VERSION" + echo "please run 'export POSTGRES_OLD_VERSION=i.e. 9.4' before running this script" + echo "check by i.e. running 'sudo cat $PATH_TO_MATTERMOST_DOCKER/volumes/db/var/lib/postgresql/data/PG_VERSION'" + exit 1 + fi + echo "found POSTGRES_OLD_VERSION=$POSTGRES_OLD_VERSION" fi + if [[ $POSTGRES_NEW_VERSION == "" ]]; then - echo "Please set environment variable POSTGRES_NEW_VERSION in the script. " - exit 1 + echo "no exported POSTGRES_NEW_VERSION environment variable found" + echo "setting POSTGRES_NEW_VERSION environment variable to default 13" + POSTGRES_NEW_VERSION=13 # i.e. 13 + echo "set POSTGRES_NEW_VERSION=$POSTGRES_NEW_VERSION" fi + + if [[ $POSTGRES_DOCKER_TAG == "" ]]; then - echo "Please set environment variable POSTGRES_DOCKER_TAG in the script. " - exit 1 + echo "no exported POSTGRES_DOCKER_TAG environment variable found" + echo "setting POSTGRES_DOCKER_TAG environment variable to default 13.2-alpine" + echo "tag needs to be an alpine release to include python3-dev found here - https://hub.docker.com/_/postgres" + POSTGRES_DOCKER_TAG=13.2-alpine # i.e. '13.2-alpine' + echo "set POSTGRES_DOCKER_TAG=$POSTGRES_DOCKER_TAG" fi -if [[ $POSTGRES_OLD_DOCKERFILE == "" ]]; then - echo "Please set environment variable POSTGRES_OLD_DOCKERFILE in the script. " - exit 1 + +if [[ $POSTGRES_OLD_DOCKER_FROM == "" ]]; then + echo "no exported POSTGRES_OLD_DOCKER_FROM environment variable found" + echo "setting POSTGRES_OLD_DOCKER_FROM to default '$(grep 'FROM postgres' "$PATH_TO_MATTERMOST_DOCKER"/db/Dockerfile)'" + POSTGRES_OLD_DOCKER_FROM=$(grep 'FROM postgres' "$PATH_TO_MATTERMOST_DOCKER/db/Dockerfile") + echo "set POSTGRES_OLD_DOCKER_FROM=$POSTGRES_OLD_DOCKER_FROM" fi -if [[ $POSTGRES_NEW_DOCKERFILE == "" ]]; then - echo "Please set environment variable POSTGRES_NEW_DOCKERFILE in the script. " - exit 1 + +if [[ $POSTGRES_NEW_DOCKER_FROM == "" ]]; then + echo "no exported POSTGRES_NEW_DOCKER_FROM environment variable found" + echo "setting POSTGRES_NEW_DOCKER_FROM to default 'FROM postgres:$POSTGRES_DOCKER_TAG'" + POSTGRES_NEW_DOCKER_FROM="FROM postgres:$POSTGRES_DOCKER_TAG" + echo "set POSTGRES_NEW_DOCKER_FROM=$POSTGRES_NEW_DOCKER_FROM" fi + if [[ $POSTGRES_UPGRADE_LINE == "" ]]; then - echo "Please set environment variable POSTGRES_UPGRADE_LINE in the script. " - exit 1 + echo "no exported POSTGRES_UPGRADE_LINE environment variable found" + echo "setting POSTGRES_UPGRADE_LINE to default $POSTGRES_OLD_VERSION-to-$POSTGRES_POSTGRES_NEW_VERSION" + echo "the POSTGRES_UPGRADE_LINE needs to match a folder found here - https://github.com/tianon/docker-postgres-upgrade" + echo "it should read 'old-to-new'" + POSTGRES_UPGRADE_LINE=$POSTGRES_OLD_VERSION-to-$POSTGRES_NEW_VERSION # i.e. '9.4-to-13' + echo "set POSTGRES_UPGRADE_LINE=$POSTGRES_UPGRADE_LINE" fi + +printf "\n" if [[ $MM_OLD_VERSION == "" ]]; then - echo "Please set environment variable MM_OLD_VERSION in the script. " - exit 1 + echo "trying to fetch MM_OLD_VERSION from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + MM_OLD_VERSION=$(echo "$(grep ".*-.*MM_VERSION=.*" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~.*-.*MM_VERSION=~~g) + if [[ $MM_OLD_VERSION == "" ]]; then + echo "could not find MM_OLD_VERSION set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" + echo "please run 'export MM_OLD_VERSION=yourMMVersion' before running this script" + exit 1 + fi + echo "found MM_OLD_VERSION=$MM_OLD_VERSION" fi + if [[ $MM_NEW_VERSION == "" ]]; then - echo "Please set environment variable MM_NEW_VERSION in the script. " - exit 1 + echo "no exported MM_NEW_VERSION environment variable found" + echo "setting MM_NEW_VERSION to default 5.32.1" + MM_NEW_VERSION=5.32.1 + echo "found MM_NEW_VERSION=$MM_NEW_VERSION" fi + +printf "\n" echo "Path to mattermost-docker: $PATH_TO_MATTERMOST_DOCKER" -echo "Postgres user: $POSTGRES_USER" -echo "Postgres password: $POSTGRES_PASSWORD" -echo "Postgres password: $POSTGRES_DB" +echo "Postgres user: redacted" +echo "Postgres password: redacted" echo "Postgres database name: $POSTGRES_DB" echo "Postgres old version: $POSTGRES_OLD_VERSION" echo "Postgres new version: $POSTGRES_NEW_VERSION" echo "Postgres alpine docker tag including python3-dev: $POSTGRES_DOCKER_TAG" -echo "Postgres old Dockerfile: $POSTGRES_OLD_DOCKERFILE" -echo "Postgres new Dockerfile: $POSTGRES_NEW_DOCKERFILE" -echo "Postgres upgrade-line: $POSTGRES_UPGRADE_LINE" +echo "Postgres old Dockerfile: $POSTGRES_OLD_DOCKER_FROM" +echo "Postgres new Dockerfile: $POSTGRES_NEW_DOCKER_FROM" +echo "Postgres upgrade-line matches a folder here - https://github.com/tianon/docker-postgres-upgrade: $POSTGRES_UPGRADE_LINE" echo "Mattermost old version: $MM_OLD_VERSION" echo "Mattermost new version: $MM_NEW_VERSION" +printf "\n" df -h -read -p "Please make sure you have enough disk space left on your devices. Try to backup and upgrade now? (y/n)" choice -if [[ "$choice" != "y" && "$choice" != "Y" && "$choice" && "yes" ]]; then +read -rp "Please make sure you have enough disk space left on your devices. Try to backup and upgrade now? (y/n)" choice +if [[ "$choice" != "y" && "$choice" != "Y" && "$choice" != "yes" ]]; then exit 0; fi ## ## Script Start ## -cd $PATH_TO_MATTERMOST_DOCKER +cd "$PATH_TO_MATTERMOST_DOCKER" docker-compose stop # Creating a backup folder and backing up the mattermost / database. -mkdir $PATH_TO_MATTERMOST_DOCKER/backups -cp -ra $PATH_TO_MATTERMOST_DOCKER/volumes/app/mattermost/ $PATH_TO_MATTERMOST_DOCKER/backups/mattermost-backup-$(date +'%F-%H-%M')/ -cp -ra $PATH_TO_MATTERMOST_DOCKER/volumes/db/ $PATH_TO_MATTERMOST_DOCKER/backups/database-backup-$(date +'%F-%H-%M')/ +mkdir "$PATH_TO_MATTERMOST_DOCKER"/backups +DATE=$(date +'%F-%H-%M') +cp -ra "$PATH_TO_MATTERMOST_DOCKER"/volumes/app/mattermost/ "$PATH_TO_MATTERMOST_DOCKER"/backups/mattermost-backup-"$DATE"/ +cp -ra "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/ "$PATH_TO_MATTERMOST_DOCKER"/backups/database-backup-"$DATE"/ -mkdir $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_OLD_VERSION -mv $PATH_TO_MATTERMOST_DOCKER/volumes/db/var/lib/postgresql/data/ $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_OLD_VERSION -rm -rf $PATH_TO_MATTERMOST_DOCKER/volumes/db/var -mkdir -p $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_NEW_VERSION/data +mkdir "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/"$POSTGRES_OLD_VERSION" +mv "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/var/lib/postgresql/data/ "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/"$POSTGRES_OLD_VERSION" +rm -rf "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/var +mkdir -p "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/$POSTGRES_NEW_VERSION/data -sed -i "s/$POSTGRES_OLD_DOCKERFILE/$POSTGRES_NEW_DOCKERFILE/" $PATH_TO_MATTERMOST_DOCKER/db/Dockerfile -sed -i "s/python-dev/python3-dev/" $PATH_TO_MATTERMOST_DOCKER/db/Dockerfile -sed -i "s/$MM_OLD_VERSION/$MM_NEW_VERSION/" $PATH_TO_MATTERMOST_DOCKER/app/Dockerfile +sed -i "s/$POSTGRES_OLD_DOCKER_FROM/$POSTGRES_NEW_DOCKER_FROM/" "$PATH_TO_MATTERMOST_DOCKER"/db/Dockerfile +sed -i "s/python-dev/python3-dev/" "$PATH_TO_MATTERMOST_DOCKER"/db/Dockerfile +sed -i "s/$MM_OLD_VERSION/$MM_NEW_VERSION/" "$PATH_TO_MATTERMOST_DOCKER"/app/Dockerfile # replacing the old postgres path with a new path -sed -i "s#./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data#./volumes/db/$POSTGRES_NEW_VERSION/data:/var/lib/postgresql/data#" $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml +sed -i "s#./volumes/db/var/lib/postgresql/data:/var/lib/postgresql/data#./volumes/db/$POSTGRES_NEW_VERSION/data:/var/lib/postgresql/data#" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml # migrate the database to the new postgres version docker run --rm \ @@ -142,16 +181,16 @@ docker run --rm \ -e POSTGRES_INITDB_ARGS=" -U $POSTGRES_USER" \ -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ -e POSTGRES_DB="$POSTGRES_DB" \ - -v $PATH_TO_MATTERMOST_DOCKER/volumes/db:/var/lib/postgresql \ - tianon/postgres-upgrade:$POSTGRES_UPGRADE_LINE \ + -v "$PATH_TO_MATTERMOST_DOCKER"/volumes/db:/var/lib/postgresql \ + tianon/postgres-upgrade:"$POSTGRES_UPGRADE_LINE" \ --link -cp -p $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_OLD_VERSION/data/pg_hba.conf $PATH_TO_MATTERMOST_DOCKER/volumes/db/$POSTGRES_NEW_VERSION/data/ +cp -p "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/"$POSTGRES_OLD_VERSION"/data/pg_hba.conf "$PATH_TO_MATTERMOST_DOCKER"/volumes/db/$POSTGRES_NEW_VERSION/data/ # rebuild the containers docker-compose build docker-compose up -d # reindex the database -echo "REINDEX SCHEMA CONCURRENTLY public;" | docker exec mattermost-docker_db_1 psql -U $POSTGRES_USER $POSTGRES_DB +echo "REINDEX SCHEMA CONCURRENTLY public;" | docker exec mattermost-docker_db_1 psql -U "$POSTGRES_USER" "$POSTGRES_DB" cd - From 82758a28c9b8c58ace96df87558807ceef1e6cee Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Tue, 13 Jul 2021 11:56:25 +0200 Subject: [PATCH 06/13] Improvements in usability. --- scripts/upgrade-from-mattermost-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh index 53985ff..19fdd2c 100755 --- a/scripts/upgrade-from-mattermost-docker.sh +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -5,7 +5,7 @@ set -e ## ## Instructions ## -# Dockerfile stolen from contributions in this issue: +# Dockerfile stolen from contributions in this issue: https://github.com/mattermost/mattermost-docker/issues/489#issuecomment-790277661 # 1. Edit the variables below to match your environment. This uses default variables and assumes you're on 5.31.0. # If you're wanting to use another version of Postgres/Mattermost , update the variables as desired. From 38265a9f2172fee945f687d45a37519530354327 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Fri, 16 Jul 2021 11:11:16 +0200 Subject: [PATCH 07/13] Add circleci --- .circleci/config.yml | 8 ++++++++ scripts/upgrade-from-mattermost-docker.sh | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..6c51b3c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,8 @@ +version: 2.1 +orbs: + shellcheck: circleci/shellcheck@2.2.4 +workflows: + check-scripts: + jobs: + - shellcheck/check: + dir: ./scripts diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh index 19fdd2c..30c5587 100755 --- a/scripts/upgrade-from-mattermost-docker.sh +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash -set -e +set -o errexit ## ## Instructions From a962d3d8b667e8699eec05102f9a9efce13da5f4 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Fri, 16 Jul 2021 11:12:33 +0200 Subject: [PATCH 08/13] Add CI. Add exclude rule. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c51b3c..d9bcb72 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,3 +6,4 @@ workflows: jobs: - shellcheck/check: dir: ./scripts + exclude: SC2001 From c303819d5a17335775de54151ea5e3c7dc17320d Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Fri, 16 Jul 2021 11:32:50 +0200 Subject: [PATCH 09/13] Address changes requested. --- scripts/issue-certificate.sh | 12 ++++++------ scripts/upgrade-from-mattermost-docker.sh | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/issue-certificate.sh b/scripts/issue-certificate.sh index 8b35621..aa82e26 100755 --- a/scripts/issue-certificate.sh +++ b/scripts/issue-certificate.sh @@ -16,12 +16,12 @@ issue_cert_standalone() { docker run -it --rm --name certbot -p 80:80 \ -v "${1}/etc/letsencrypt:/etc/letsencrypt" \ -v "${1}/lib/letsencrypt:/var/lib/letsencrypt" \ - certbot/certbot certonly --standalone -d ${2} + certbot/certbot certonly --standalone -d "${2}" } authenticator_to_webroot() { - sed -i 's/standalone/webroot/' ${1}/etc/letsencrypt/renewal/${2}.conf - tee -a ${1}/etc/letsencrypt/renewal/${2}.conf >/dev/null </dev/null </dev/null); then +if ! which docker 1>/dev/null; then echo "Can't find Docker command" >&2 exit 64 fi -issue_cert_standalone ${output} ${domain} -authenticator_to_webroot ${output} ${domain} +issue_cert_standalone "${output}" "${domain}" +authenticator_to_webroot "${output}" "${domain}" diff --git a/scripts/upgrade-from-mattermost-docker.sh b/scripts/upgrade-from-mattermost-docker.sh index 30c5587..2d43c84 100755 --- a/scripts/upgrade-from-mattermost-docker.sh +++ b/scripts/upgrade-from-mattermost-docker.sh @@ -14,9 +14,9 @@ set -o errexit # This may take some time to complete as it's migrating the database to Postgres 13.6 from 9.4 -PATH_TO_MATTERMOST_DOCKER=$PATH_TO_MATTERMOST_DOCKER # i.e. $PWD if [[ $PATH_TO_MATTERMOST_DOCKER == "" ]]; then - echo "Please export environment variable PATH_TO_MATTERMOST_DOCKER with '$ export PATH_TO_MATTERMOST_DOCKER=/path/to/mattermost-docker' before running this script. " + # shellcheck disable=SC2016 + echo 'Please export environment variable PATH_TO_MATTERMOST_DOCKER with "$ export PATH_TO_MATTERMOST_DOCKER=/path/to/mattermost-docker", i.e. $PWD before running this script. ' exit 1 fi @@ -27,7 +27,7 @@ fi # The script is trying to fetch those variables first. Should fetching fail, please export the variables before running the script. if [[ $POSTGRES_USER == "" ]]; then echo "trying to fetch POSTGRES_USER from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" - POSTGRES_USER=$(echo "$(grep "^.*-.*POSTGRES_USER=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~^.*-.*POSTGRES_USER=~~g) + POSTGRES_USER=$(grep "^.*-.*POSTGRES_USER=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml | sed s~^.*-.*POSTGRES_USER=~~g) if [[ $POSTGRES_USER == "" ]]; then echo "could not find POSTGRES_USER set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" echo "please run 'export POSTGRES_USER=yourPostgresUser' before running this script" @@ -38,7 +38,7 @@ fi if [[ $POSTGRES_PASSWORD == "" ]]; then echo "trying to fetch POSTGRES_PASSWORD from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" - POSTGRES_PASSWORD=$(echo "$(grep "^.*-.*POSTGRES_PASSWORD=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~^.*-.*POSTGRES_PASSWORD=~~g) + POSTGRES_PASSWORD=$(grep "^.*-.*POSTGRES_PASSWORD=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml | sed s~^.*-.*POSTGRES_PASSWORD=~~g) if [[ $POSTGRES_PASSWORD == "" ]]; then echo "could not find POSTGRES_PASSWORD set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" echo "please run 'export POSTGRES_PASSWORD=yourPostgresPassword' before running this script" @@ -49,7 +49,7 @@ fi if [[ $POSTGRES_DB == "" ]]; then echo "trying to fetch POSTGRES_DB from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" - POSTGRES_DB=$(echo "$(grep "^.*-.*POSTGRES_DB=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~^.*-.*POSTGRES_DB=~~g) + POSTGRES_DB=$(grep "^.*-.*POSTGRES_DB=.*$" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml | sed s~^.*-.*POSTGRES_DB=~~g) if [[ $POSTGRES_DB == "" ]]; then echo "could not find POSTGRES_DB set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" echo "please run 'export POSTGRES_DB=yourPostgresDatabase' before running this script" @@ -113,7 +113,7 @@ fi printf "\n" if [[ $MM_OLD_VERSION == "" ]]; then echo "trying to fetch MM_OLD_VERSION from $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" - MM_OLD_VERSION=$(echo "$(grep ".*-.*MM_VERSION=.*" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml)" | sed s~.*-.*MM_VERSION=~~g) + MM_OLD_VERSION=$(grep ".*-.*MM_VERSION=.*" "$PATH_TO_MATTERMOST_DOCKER"/docker-compose.yml | sed s~.*-.*MM_VERSION=~~g) if [[ $MM_OLD_VERSION == "" ]]; then echo "could not find MM_OLD_VERSION set in $PATH_TO_MATTERMOST_DOCKER/docker-compose.yml" echo "please run 'export MM_OLD_VERSION=yourMMVersion' before running this script" From 85a50db457f96605762fc94ada6b7a5017a2baff Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Fri, 16 Jul 2021 11:33:45 +0200 Subject: [PATCH 10/13] Address changes requested. --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9bcb72..6c51b3c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,4 +6,3 @@ workflows: jobs: - shellcheck/check: dir: ./scripts - exclude: SC2001 From 2c74a0936af01713f632ddd1ddaed5e158cc104b Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Fri, 16 Jul 2021 11:35:27 +0200 Subject: [PATCH 11/13] Trigger CI From 6753d0264be2725110de318a6d5f7ece249cb1ff Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Mon, 19 Jul 2021 13:17:27 +0200 Subject: [PATCH 12/13] Update README.md Warn about disk space problems --- scripts/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/README.md b/scripts/README.md index 17c6830..74bea74 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,3 +1,7 @@ +# IMPORTANT: Please make sure you have enough disk space available for the backups! +Because it is more complicated to check the available disk space for various disk formatting options provided by different linux distributions, the script does currently not check for if there is enough disk space. +Please check manually before executing this script! + ## Upgrading from mattermost-docker ``` From df620ea4ee6db076fb2b8fd062fe3313af2478a0 Mon Sep 17 00:00:00 2001 From: Elisabeth Kulzer Date: Mon, 19 Jul 2021 13:19:03 +0200 Subject: [PATCH 13/13] Rename README.md to UPGRADE.md --- scripts/{README.md => UPGRADE.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{README.md => UPGRADE.md} (100%) diff --git a/scripts/README.md b/scripts/UPGRADE.md similarity index 100% rename from scripts/README.md rename to scripts/UPGRADE.md