Added support for sending WAL to Azure via wal-g

Close https://github.com/zalando/spilo/issues/383

ENVIRONMENT.rst

@@ -65,6 +65,9 @@ Environment Configuration Settings
 - **SWIFT_PROJECT_DOMAIN_NAME**:
 - **SWIFT_PROJECT_DOMAIN_ID**:
 - **WALE_SWIFT_PREFIX**: (optional) the full path to the backup location on the Swift Storage in the format swift://bucket-name/very/long/path. If not specified Spilo will generate it from WAL_SWIFT_BUCKET.
+- **WALG_AZ_PREFIX**: (optional) the azure prefix to store WAL backups at in the format azure://test-container/walg-folder.
+- **AZURE_STORAGE_ACCOUNT**:(optional) the azure storage account to use for WAL backups.
+- **AZURE_STORAGE_ACCOUNT_ACCESS_KEY**:(optional) the access key for the azure storage account used for WAL backups.
 - **CALLBACK_SCRIPT**: the callback script to run on various cluster actions (on start, on stop, on restart, on role change). The script will receive the cluster name, connection string and the current action. See `Patroni <http://patroni.readthedocs.io/en/latest/SETTINGS.html?highlight=callback#postgresql>`__ documentation for details.
 - **LOG_S3_BUCKET**: path to the S3 bucket used for PostgreSQL daily log files (i.e. s3://foobar). Spilo will add /spilo/scope/pg_daily_logs to that path. Logs are shipped if this variable is set.
 - **LOG_SHIP_SCHEDULE**: cron schedule for shipping compressed logs from ``pg_log`` (if this feature is enabled, '00 02 * * *' by default)

postgres-appliance/scripts/configure_spilo.py

@@ -40,7 +40,7 @@ extensions = {
     'pg_partman':     (9.4, 12, False, True)
 }
 
-AUTO_ENABLE_WALG_RESTORE = ('WAL_S3_BUCKET', 'WALE_S3_PREFIX', 'WALG_S3_PREFIX')
+AUTO_ENABLE_WALG_RESTORE = ('WAL_S3_BUCKET', 'WALE_S3_PREFIX', 'WALG_S3_PREFIX', 'WALG_AZ_PREFIX')
 
 
 def parse_args():
@@ -406,14 +406,13 @@ def get_instance_metadata(provider):
 
 def set_extended_wale_placeholders(placeholders, prefix):
     """ checks that enough parameters are provided to configure cloning or standby with WAL-E """
-    for name in ('S3', 'GS', 'GCS', 'SWIFT'):
+    for name in ('S3', 'GS', 'GCS', 'SWIFT', 'AZ'):
         if placeholders.get('{0}WALE_{1}_PREFIX'.format(prefix, name)) or\
                 name in ('S3', 'GS') and placeholders.get('{0}WALG_{1}_PREFIX'.format(prefix, name)) or\
                 placeholders.get('{0}WAL_{1}_BUCKET'.format(prefix, name)) and placeholders.get(prefix + 'SCOPE'):
             break
     else:
         return False
-
     scope = placeholders.get(prefix + 'SCOPE')
     dirname = 'env-' + prefix[:-1].lower() + ('-' + scope if scope else '')
     placeholders[prefix + 'WALE_ENV_DIR'] = os.path.join(placeholders['RW_DIR'], 'etc', 'wal-e.d', dirname)
@@ -559,6 +558,8 @@ def get_placeholders(provider):
 
     if any(placeholders.get(n) for n in AUTO_ENABLE_WALG_RESTORE):
         placeholders.setdefault('USE_WALG_RESTORE', 'true')
+    if placeholders.get('WALG_AZ_PREFIX'):
+        placeholders.setdefault('USE_WALG_BACKUP', 'true')
     set_walg_placeholders(placeholders)
 
     placeholders['USE_WALE'] = any(placeholders.get(n) for n in AUTO_ENABLE_WALG_RESTORE +
@@ -685,6 +686,7 @@ def write_wale_environment(placeholders, prefix, overwrite):
     s3_names = ['WALE_S3_PREFIX', 'WALG_S3_PREFIX', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY',
                 'WALE_S3_ENDPOINT', 'AWS_ENDPOINT', 'AWS_REGION', 'AWS_INSTANCE_PROFILE',
                 'WALG_S3_SSE_KMS_ID', 'WALG_S3_SSE', 'WALG_DISABLE_S3_SSE', 'AWS_S3_FORCE_PATH_STYLE']
+    azure_names = ['WALG_AZ_PREFIX', 'AZURE_STORAGE_ACCOUNT', 'AZURE_STORAGE_ACCESS_KEY']
     gs_names = ['WALE_GS_PREFIX', 'WALG_GS_PREFIX', 'GOOGLE_APPLICATION_CREDENTIALS']
     swift_names = ['WALE_SWIFT_PREFIX', 'SWIFT_AUTHURL', 'SWIFT_TENANT', 'SWIFT_TENANT_ID', 'SWIFT_USER',
                    'SWIFT_USER_ID', 'SWIFT_USER_DOMAIN_NAME', 'SWIFT_USER_DOMAIN_ID', 'SWIFT_PASSWORD',
@@ -699,8 +701,8 @@ def write_wale_environment(placeholders, prefix, overwrite):
 
     wale = defaultdict(lambda: '')
     for name in ['WALE_ENV_DIR', 'SCOPE', 'WAL_BUCKET_SCOPE_PREFIX', 'WAL_BUCKET_SCOPE_SUFFIX',
-                 'WAL_S3_BUCKET', 'WAL_GCS_BUCKET', 'WAL_GS_BUCKET', 'WAL_SWIFT_BUCKET', 'BACKUP_NUM_TO_RETAIN'] +\
-            s3_names + swift_names + gs_names + walg_names:
+                 'WAL_S3_BUCKET', 'WAL_GCS_BUCKET', 'WAL_GS_BUCKET', 'WAL_SWIFT_BUCKET', 'BACKUP_NUM_TO_RETAIN',
+                 'WALG_AZ_PREFIX'] + s3_names + swift_names + gs_names + walg_names + azure_names:
         wale[name] = placeholders.get(prefix + name, '')
 
     if wale.get('WAL_S3_BUCKET') or wale.get('WALE_S3_PREFIX') or wale.get('WALG_S3_PREFIX'):
@@ -741,6 +743,8 @@ def write_wale_environment(placeholders, prefix, overwrite):
         write_envdir_names = gs_names + walg_names
     elif wale.get('WAL_SWIFT_BUCKET') or wale.get('WALE_SWIFT_PREFIX'):
         write_envdir_names = swift_names
+    elif wale.get("WALG_AZ_PREFIX"):
+        write_envdir_names = azure_names + walg_names
     else:
         return