Unverified Commit 4ac25e1e authored by Alexander Kukushkin's avatar Alexander Kukushkin Committed by GitHub
Browse files

Execute pg_upgrade after custom bootstrap if major version doesn't match (#282)

Before doing the actual call of `pg_upgrade`, the `maybe_pg_upgrade.py` script starts the old postgres binary, waits until `pg_is_in_recovery()` starts returning `False` and does a clean shutdown.
parent 4517148c
Showing with 168 additions and 12 deletions
+168 -12
......@@ -177,7 +177,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
&& if [ "$version" = "9.6" ] || [ "$version" = "10" ] || [ "$version" = "11" ] ; then \
cd timescaledb \
&& rm -fr build \
&& ./bootstrap \
&& ./bootstrap -DAPACHE_ONLY=1 \
&& make -C build install \
&& strip /usr/lib/postgresql/$version/lib/timescaledb*.so \
&& cd ..; \
......@@ -373,9 +373,9 @@ RUN export DEBIAN_FRONTEND=noninteractive \
/usr/lib/postgresql/*/bin/pltcl_*
# Install patroni, wal-e and wal-g
ENV PATRONIVERSION=1.5.3
ENV PATRONIVERSION=1.5.4
ENV WALE_VERSION=1.1.0
ENV WALG_VERSION=v0.2.2
ENV WALG_VERSION=v0.2.3
RUN export DEBIAN_FRONTEND=noninteractive \
&& set -ex \
&& BUILD_PACKAGES="python3-pip python3-wheel python3-dev git patchutils binutils" \
......@@ -429,10 +429,6 @@ RUN export DEBIAN_FRONTEND=noninteractive \
fi \
&& pip3 install "patroni[kubernetes$EXTRAS]==$PATRONIVERSION" \
# https://github.com/zalando/patroni/pull/896, temporary patch Patroni until the new version is not released
&& cd /usr/local/lib/python3.6/dist-packages \
&& curl -sL https://github.com/zalando/patroni/pull/896.diff | patch -p1 \
&& for d in /usr/local/lib/python3.6 /usr/lib/python3; do \
cd $d/dist-packages \
&& find . -type d -name tests | xargs rm -fr \
......
......@@ -3,6 +3,9 @@
import argparse
import logging
import subprocess
import sys
from maybe_pg_upgrade import call_maybe_pg_upgrade
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)
......@@ -55,11 +58,12 @@ def run_basebackup(options):
def main():
options = read_configuration()
try:
return run_basebackup(options)
run_basebackup(options)
except Exception:
logger.exception("Clone failed")
return 1
return call_maybe_pg_upgrade()
if __name__ == '__main__':
main()
sys.exit(main())
......@@ -5,6 +5,9 @@ import csv
import logging
import os
import subprocess
import sys
from maybe_pg_upgrade import call_maybe_pg_upgrade
from collections import namedtuple
from dateutil.parser import parse
......@@ -82,11 +85,12 @@ def run_clone_from_s3(options):
def main():
options = read_configuration()
try:
return run_clone_from_s3(options)
run_clone_from_s3(options)
except Exception:
logger.exception("Clone failed")
return 1
return call_maybe_pg_upgrade()
if __name__ == '__main__':
main()
sys.exit(main())
#!/usr/bin/env python
import logging
import sys
logger = logging.getLogger(__name__)
def main():
from patroni.config import Config
from patroni.utils import polling_loop
from pg_upgrade import PostgresqlUpgrade
config = Config()
upgrade = PostgresqlUpgrade(config['postgresql'])
bin_version = upgrade.get_binary_version()
cluster_version = upgrade.get_cluster_version()
if cluster_version == bin_version:
return 0
logger.info('Cluster version: %s, bin version: %s', cluster_version, bin_version)
assert float(cluster_version) < float(bin_version)
upgrade.set_bin_dir(cluster_version)
upgrade.config['pg_ctl_timeout'] = 3600*24*7
upgrade.config['callbacks'] = {}
bootstrap_config = config['bootstrap']
bootstrap_config[bootstrap_config['method']]['command'] = 'true'
logger.info('Trying to start the cluster with old postgres')
if not upgrade.bootstrap(bootstrap_config):
raise Exception('Failed to start the cluster with old postgres')
for _ in polling_loop(upgrade.config['pg_ctl_timeout'], 10):
upgrade.reset_cluster_info_state()
if upgrade.is_leader():
break
logger.info('waiting for end of recovery of the old cluster')
if not upgrade.run_bootstrap_post_init(bootstrap_config):
raise Exception('Failed to run bootstrap.post_init')
locale = upgrade.query('SHOW lc_collate').fetchone()[0]
encoding = upgrade.query('SHOW server_encoding').fetchone()[0]
initdb_config = [{'locale': locale}, {'encoding': encoding}]
if upgrade.query('SHOW data_checksums').fetchone()[0]:
initdb_config.append('data-checksums')
logger.info('Doing a clean shutdown of the cluster before pg_upgrade')
if not upgrade.stop(block_callbacks=True, checkpoint=False):
raise Exception('Failed to stop the cluster with old postgres')
logger.info('initdb config: %s', initdb_config)
logger.info('Executing pg_upgrade')
if not upgrade.do_upgrade(bin_version, {'initdb': initdb_config}):
raise Exception('Failed to upgrade cluster from {0} to {1}'.format(cluster_version, bin_version))
logger.info('Starting the cluster with new postgres after upgrade')
if not upgrade.start():
raise Exception('Failed to start the cluster with new postgres')
upgrade.analyze()
def call_maybe_pg_upgrade():
import inspect
import os
import subprocess
my_name = os.path.abspath(inspect.getfile(inspect.currentframe()))
ret = subprocess.call([sys.executable, my_name, os.path.join(os.getenv('PGHOME'), 'postgres.yml')])
if ret != 0:
logger.error('%s script failed', my_name)
return ret
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s maybe_pg_upgrade %(levelname)s: %(message)s', level='INFO')
if len(sys.argv) > 5 and sys.argv[1] == 'pg_ctl_start':
from patroni import pg_ctl_start
pg_ctl_start(sys.argv[2:])
else:
main()
import os
import shutil
import subprocess
import re
import psutil
from patroni.postgresql import Postgresql
class PostgresqlUpgrade(Postgresql):
def get_binary_version(self):
version = subprocess.check_output([self._pgcommand('postgres'), '--version']).decode()
version = re.match('^[^\s]+ [^\s]+ (\d+)\.(\d+)', version)
return '.'.join(version.groups()) if int(version.group(1)) < 10 else version.group(1)
def get_cluster_version(self):
with open(self._version_file) as f:
return f.read().strip()
def set_bin_dir(self, version):
self._old_bin_dir = self._bin_dir
self._bin_dir = '/usr/lib/postgresql/{0}/bin'.format(version)
def copy_configs(self):
for f in os.listdir(self._old_data_dir):
if f.startswith('postgresql.') or f.startswith('pg_hba.conf') or f == 'patroni.dynamic.json':
shutil.copy(os.path.join(self._old_data_dir, f), os.path.join(self._data_dir, f))
return True
def pg_upgrade(self):
self._upgrade_dir = self._data_dir + '_upgrade'
if os.path.exists(self._upgrade_dir) and os.path.isdir(self._upgrade_dir):
shutil.rmtree(self._upgrade_dir)
os.makedirs(self._upgrade_dir)
self._old_cwd = os.getcwd()
os.chdir(self._upgrade_dir)
pg_upgrade_args = ['-k', '-j', str(psutil.cpu_count()),
'-b', self._old_bin_dir, '-B', self._bin_dir,
'-d', self._old_data_dir, '-D', self._data_dir]
if 'username' in self._superuser:
pg_upgrade_args += ['-U', self._superuser['username']]
return subprocess.call([self._pgcommand('pg_upgrade')] + pg_upgrade_args) == 0
def do_upgrade(self, version, initdb_config):
self._data_dir = os.path.abspath(self._data_dir)
self._old_data_dir = self._data_dir + '_old'
os.rename(self._data_dir, self._old_data_dir)
self.set_bin_dir(version)
if self._initdb(initdb_config) and self.copy_configs() and self.pg_upgrade():
os.chdir(self._old_cwd)
shutil.rmtree(self._upgrade_dir)
shutil.rmtree(self._old_data_dir)
return True
def analyze(self):
vacuumdb_args = ['-a', '-Z', '-j', str(psutil.cpu_count())]
if 'username' in self._superuser:
vacuumdb_args += ['-U', self._superuser['username']]
subprocess.call([self._pgcommand('vacuumdb')] + vacuumdb_args)
......@@ -2,7 +2,9 @@
cd "$(dirname "${BASH_SOURCE[0]}")"
(echo "DO \$\$
(echo "SET synchronous_commit = 'local';
DO \$\$
BEGIN
PERFORM * FROM pg_catalog.pg_authid WHERE rolname = 'admin';
IF FOUND THEN
......@@ -125,6 +127,7 @@ cat _zmon_schema.dump
while IFS= read -r db_name; do
echo "\c ${db_name}"
echo "SET synchronous_commit = 'local';"
# In case if timescaledb binary is missing the first query fails with the error
# ERROR: could not access file "$libdir/timescaledb-$OLD_VERSION": No such file or directory
TIMESCALEDB_VERSION=$(echo -e "SELECT NULL;\nSELECT extversion FROM pg_extension WHERE extname = 'timescaledb'" | psql -tAX -d ${db_name} 2> /dev/null | tail -n 1)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment