diff --git a/bin/pgbackman_dump b/bin/pgbackman_dump index 3612c4c..719d35c 100755 --- a/bin/pgbackman_dump +++ b/bin/pgbackman_dump @@ -1308,7 +1308,11 @@ def main(): if global_parameters['backup_code'] == 'CLUSTER': pg_dumpall(db) +<<<<<<< HEAD + elif global_parameters['backup_code'] == 'RDSDATA': +======= elif global_parameters['backup_code'] == 'RDS': +>>>>>>> 7d6b4a293c3f8676f316a6c7ca3fac3f9f590097 pg_dump(db) else: pg_dump(db) @@ -1362,7 +1366,11 @@ if __name__ == '__main__': parser.add_argument('--def-id', metavar='JOBID', required=False, help='Backup job ID', dest='def_id') parser.add_argument('--snapshot-id', metavar='SNAPSHOT-ID', required=False, help='snapshot ID', dest='snapshot_id') parser.add_argument('--dbname', metavar='DBNAME', required=False, help='Database name', dest='dbname') +<<<<<<< HEAD + parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER|RDSDATAONLY]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER', 'RDS'], required=True, help='Backup code', dest='backup_code') +======= parser.add_argument('--backup-code', metavar='[FULL|SCHEMA|DATA|CLUSTER|RDS]', choices=['FULL', 'SCHEMA', 'DATA', 'CLUSTER', 'RDS'], required=True, help='Backup code', dest='backup_code') +>>>>>>> 7d6b4a293c3f8676f316a6c7ca3fac3f9f590097 parser.add_argument('--encryption', metavar='[false|true]', default=True, choices=['true', 'false'],required=True, help='Activate encryption', dest='encryption') parser.add_argument('--root-backup-dir', metavar='ROOT-BACKUP-DIR', default=True, required=True, help='Root backup dir', dest='root_backup_dir') parser.add_argument('--extra-backup-parameters', metavar='EXTRA-PARAMETERS', required=False, help='extra pg_dump parameters', dest='extra_backup_parameters') diff --git a/bin/pgbackman_restore b/bin/pgbackman_restore index c8bf522..e37e319 100755 --- a/bin/pgbackman_restore +++ b/bin/pgbackman_restore @@ -31,7 +31,7 @@ import signal import argparse from pgbackman.logs import * -from pgbackman.database import * +from pgbackman.database import * from pgbackman.config import * ''' @@ -55,15 +55,15 @@ def restore_roles(db): try: pg_restore_roles_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=global_parameters['tmp_dir']) logs.logger.debug('Restore roles temp file created %s',pg_restore_roles_temp_file.name) - + with open(global_parameters['pgdump_roles_file'], 'r') as sqldump_in: with open(pg_restore_roles_temp_file.name, 'w') as sqldump_out: sqldump_out.write('BEGIN;\n') - + for line in sqldump_in: - + for role in global_parameters['role_list']: - + # # CREATE ROLE statements # @@ -82,7 +82,7 @@ def restore_roles(db): if 'GRANT ' in line: if ' TO ' + role + ' GRANTED BY ' in line: sqldump_out.write(line) - + sqldump_out.write('COMMIT;\n') logs.logger.debug('Role restore file generated.') sqldump_out.flush() @@ -96,12 +96,12 @@ def restore_roles(db): ' -f ' + pg_restore_roles_temp_file.name with open(global_parameters['restore_log_file'],'a') as restore_log_file: - + restore_log_file.write('------------------------------------\n') restore_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') restore_log_file.write('Command: ' + role_restore_command + '\n') restore_log_file.write('------------------------------------\n\n') - + restore_log_file.flush() proc = subprocess.Popen([role_restore_command],stdout=restore_log_file,stderr=subprocess.STDOUT,shell=True) @@ -109,23 +109,23 @@ def restore_roles(db): if proc.returncode != 0: logs.logger.critical('The command used to restore the roles needed by the target database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Roles returncode: ' + str(proc.returncode) register_restore_catalog(db) sys.exit(1) - + else: logs.logger.info('Roles for database restored.') except Exception as e: logs.logger.critical('Could not restore the roles needed by the database - %s',e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems restoring roles - ' + str(e) register_restore_catalog(db) sys.exit(1) - + # ############################################ @@ -140,10 +140,10 @@ def restore_dbconfig(db): try: pg_restore_dbconfig_temp_file = tempfile.NamedTemporaryFile(delete=True,dir=global_parameters['tmp_dir']) logs.logger.debug('Temp dbconfig restore file created %s',pg_restore_dbconfig_temp_file.name) - + with open(global_parameters['pgdump_dbconfig_file'], 'r') as sqldump_in: with open(pg_restore_dbconfig_temp_file.name, 'w') as sqldump_out: - + for line in sqldump_in: # @@ -153,7 +153,7 @@ def restore_dbconfig(db): if global_parameters['source_dbname'] == global_parameters['target_dbname']: sqldump_out.write(line) - + elif global_parameters['source_dbname'] != global_parameters['target_dbname']: sqldump_out.write(line.replace('CREATE DATABASE ' + global_parameters['source_dbname'],'CREATE DATABASE ' + global_parameters['target_dbname'])) @@ -166,10 +166,10 @@ def restore_dbconfig(db): if global_parameters['source_dbname'] == global_parameters['target_dbname']: sqldump_out.write(line) - + elif global_parameters['source_dbname'] != global_parameters['target_dbname']: sqldump_out.write(line.replace(' ON DATABASE ' + global_parameters['source_dbname'],' ON DATABASE ' + global_parameters['target_dbname'])) - + # # ALTER DATABASE statements # @@ -177,12 +177,12 @@ def restore_dbconfig(db): if global_parameters['source_dbname'] == global_parameters['target_dbname']: sqldump_out.write(line) - + elif global_parameters['source_dbname'] != global_parameters['target_dbname']: sqldump_out.write(line.replace('ALTER DATABASE ' + global_parameters['source_dbname'],'ALTER DATABASE ' + global_parameters['target_dbname'])) - - sqldump_out.write('COMMIT;\n') + + sqldump_out.write('COMMIT;\n') logs.logger.debug('Temp dbconfig restore file generated.') sqldump_out.flush() @@ -196,36 +196,36 @@ def restore_dbconfig(db): ' -f ' + pg_restore_dbconfig_temp_file.name with open(global_parameters['restore_log_file'],'a') as restore_log_file: - + restore_log_file.write('------------------------------------\n') restore_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') restore_log_file.write('Command: ' + dbconfig_restore_command + '\n') restore_log_file.write('------------------------------------\n\n') restore_log_file.flush() - + proc = subprocess.Popen([dbconfig_restore_command],stdout=restore_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() if proc.returncode != 0: logs.logger.critical('The command used to restore the global database statements needed by the target database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'DBconfig for database returncode: ' + str(proc.returncode) register_restore_catalog(db) sys.exit(1) - + else: logs.logger.info('DBconfig for database restored.') except Exception as e: logs.logger.critical('Could not restore the global database statements needed by the database - %s',e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems restoring DBconfig - ' + str(e) register_restore_catalog(db) sys.exit(1) - + # ############################################ # Function restore_database() @@ -233,7 +233,7 @@ def restore_dbconfig(db): def restore_database(db): '''Restore database''' - + global global_parameters try: @@ -247,31 +247,31 @@ def restore_database(db): ' ' + global_parameters['pgdump_file'] with open(global_parameters['restore_log_file'],'a') as restore_log_file: - + restore_log_file.write('------------------------------------\n') restore_log_file.write('Timestamp:' + str(datetime.datetime.now()) + '\n') restore_log_file.write('Command: ' + database_restore_command + '\n') restore_log_file.write('------------------------------------\n\n') - + restore_log_file.flush() proc = subprocess.Popen([database_restore_command],stdout=restore_log_file,stderr=subprocess.STDOUT,shell=True) proc.wait() - + if proc.returncode != 0: logs.logger.critical('The command used to restore the database has a return value != 0') - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Database restore returncode: ' + str(proc.returncode) register_restore_catalog(db) sys.exit(1) - + else: logs.logger.info('Database restored.') except Exception as e: logs.logger.critical('Could not restore the database - %s',e) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems restoring Database - ' + str(e) register_restore_catalog(db) @@ -281,7 +281,7 @@ def restore_database(db): # ############################################ # Function get_pgsql_node_dsn() # ############################################ - + def get_pgsql_node_dsn(): '''Get the DSN values needed to connect to a PgSQL node''' @@ -296,10 +296,10 @@ def get_pgsql_node_dsn(): # ############################################ # Function get_pgsql_node_release() # ############################################ - + def get_pgsql_node_release(db,db_pgnode): '''Get the postgreSQL release version a PgSQL node is running''' - + pgsql_node_version = '' try: @@ -308,10 +308,12 @@ def get_pgsql_node_release(db,db_pgnode): db_pgnode.pg_close() except Exception as e: logs.logger.critical('Problems getting the postgreSQL version running on Pgsql node - %s',e) - + logs.logger.debug('PgSQL node version: %s',pgsql_node_version) - if pgsql_node_version == '905': + if pgsql_node_version == '100': + pgsql_node_release = '10' + elif pgsql_node_version == '905': pgsql_node_release = '9_5' elif pgsql_node_version == '904': pgsql_node_release = '9_4' @@ -335,12 +337,12 @@ def get_pgsql_node_release(db,db_pgnode): global_parameters['error_message'] = 'Problems getting pgsql release' register_restore_catalog(db) sys.exit(1) - + # ############################################ # Function get_backup_server_pgsql_bin_dir() # ############################################ - + def get_backup_server_pgsql_bin_dir(db): '''Get the directory with postgreSQL binaries to use''' @@ -349,22 +351,22 @@ def get_backup_server_pgsql_bin_dir(db): try: pgsql_bin_dir = db.get_backup_server_config_value(global_parameters['backup_server_id'],'pgsql_bin_' + global_parameters['pg_release']) logs.logger.debug('pgsql bin directory to use: %s',pgsql_bin_dir) - + return pgsql_bin_dir except Exception as e: pgsql_bin_dir = backup_server_cache_data['pgsql_bin_' + global_parameters['pg_release']] logs.logger.debug('pgsql bin directory to use: %s',pgsql_bin_dir) - + return pgsql_bin_dir - + # ################################################## # Function get_backup_server_parameters_from_cache() # ################################################## -def get_backup_server_parameters_from_cache(db,backup_server_fqdn): +def get_backup_server_parameters_from_cache(db,backup_server_fqdn): '''Get backup server parameters from cache file''' global backup_server_cache_data @@ -372,12 +374,12 @@ def get_backup_server_parameters_from_cache(db,backup_server_fqdn): try: backup_server_cache_file = global_parameters['root_backup_dir'] + '/cache_dir/backup_server_' + backup_server_fqdn + '.cache' - + with open(backup_server_cache_file,'r') as backup_server_cache: for line in backup_server_cache: (key, val) = line.split('::') backup_server_cache_data[key] = val.replace('\n','') - + except Exception as e: logs.logger.error('Could not read the cache file for the backup server: %s - %s',backup_server_fqdn,e) global_parameters['execution_status'] = 'ERROR' @@ -391,7 +393,7 @@ def get_backup_server_parameters_from_cache(db,backup_server_fqdn): # Function get_pgsql_node_parameters_from_cache() # ############################################### -def get_pgsql_node_parameters_from_cache(db): +def get_pgsql_node_parameters_from_cache(db): '''Get pgsql_node parameters from cache file''' global pgsql_node_cache_data @@ -404,7 +406,7 @@ def get_pgsql_node_parameters_from_cache(db): for line in pgsql_node_cache: (key, val) = line.split('::') pgsql_node_cache_data[key] = val.replace('\n','') - + except Exception as e: logs.logger.critical('Could not read the cache file for the PgSQL node - %s',e) global_parameters['execution_status'] = 'ERROR' @@ -427,14 +429,14 @@ def rename_existing_database(db,db_pgnode): global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Problems renaming an existing database' register_restore_catalog(db) - + sys.exit(1) - - + + # ############################################ # Function register_restore_catalog() # ############################################ - + def register_restore_catalog(db): '''Update the restore catalog information in the database''' @@ -453,7 +455,7 @@ def register_restore_catalog(db): procpid = os.getpid() db.register_restore_catalog(global_parameters['restore_def'], procpid, - global_parameters['backup_server_id'], + global_parameters['backup_server_id'], global_parameters['pgsql_node_id'], global_parameters['source_dbname'], global_parameters['target_dbname'], @@ -469,30 +471,30 @@ def register_restore_catalog(db): global_parameters['target_pgsql_node_release'].replace('_','.'), global_parameters['pg_release'].replace('_','.') ) - + logs.logger.info('Restore job catalog for RestoreDef: %s updated in the database',str(global_parameters['restore_def'])) - + except Exception as e: # # We create a pending log file if we can not update the # database. This file will be processed by # pgbackman_maintenance later. - # + # logs.logger.warning('Problems updating the restore job catalog for RestoreDef: %s in the database - %s',str(global_parameters['restore_def']),e) pending_log_file = '' - + try: procpid = os.getpid() pending_log_file = global_parameters['backup_server_pending_registration_dir'] + '/restore_jobs_pending_log_updates_nodeid_' + str(global_parameters['pgsql_node_id']) + '_' + str(procpid) + '.log' - + with open(pending_log_file,'w+') as catalog_pending: catalog_pending.write(str(global_parameters['restore_def']) + '::' + str(procpid) + '::' + - str(global_parameters['backup_server_id']) + '::' + + str(global_parameters['backup_server_id']) + '::' + str(global_parameters['pgsql_node_id']) + '::' + global_parameters['source_dbname'] + '::' + global_parameters['target_dbname'] + '::' + @@ -507,19 +509,19 @@ def register_restore_catalog(db): " ".join(global_parameters['role_list']) + '::' + global_parameters['target_pgsql_node_release'].replace('_','.') + '::' + global_parameters['pg_release'].replace('_','.') + '\n') - + logs.logger.info('Restore catalog pending log file: %s created',pending_log_file) - + except Exception as e: logs.logger.error('Could not generate the restore catalog pending log file: %s - %s',pending_log_file,e) # ############################################ # Function get_logfile_id() # ############################################ - + def get_logfile_id(): '''Generate the filename used for the log files of a restore job''' - + global global_parameters timestamp = datetime.datetime.now().strftime('%Y%m%dT%H%M%S') @@ -531,7 +533,7 @@ def get_logfile_id(): # ############################################## # Function check_pgbackman_database_connection() # ############################################## - + def check_pgbackman_database_connection(db): '''Check if we can connect to the pgbackman database''' @@ -540,7 +542,7 @@ def check_pgbackman_database_connection(db): db.pg_close() return True - except Exception as e: + except Exception as e: logs.logger.error('The pgbackman database is not available - %s',e) return False @@ -548,7 +550,7 @@ def check_pgbackman_database_connection(db): # ############################################### # Function check_pgsql_node_database_connection() # ############################################### - + def check_pgsql_node_database_connection(db_pgnode): '''Check if we can connect to the pgsql node database''' @@ -557,18 +559,18 @@ def check_pgsql_node_database_connection(db_pgnode): db_pgnode.pg_close() return True - except Exception as e: + except Exception as e: logs.logger.critical('The PgSQL node is not available - %s',e) - return False + return False # ############################################ # Function Main() # ############################################ - + def main(): '''Main function''' - + global global_parameters conf = PgbackmanConfiguration() @@ -583,11 +585,11 @@ def main(): global_parameters['target_pgsql_node_release'] = '' db = PgbackmanDB(pgbackman_dsn, 'pgbackman_restore') - + pgsql_node_dsn = get_pgsql_node_dsn() db_pgnode = PgbackmanDB(pgsql_node_dsn, 'pgbackman_restore') - # + # # The backup server FQDN to be used can be defined in the # pgbackman configuration file. If the configuration parameter # 'backup_server' is not defined, the return value of @@ -628,18 +630,18 @@ def main(): if not check_pgnode_db: logs.logger.critical('The PgSQL node is not available. Shutting down the restore job with RestoreID: %s',str(global_parameters['restore_def'])) - + global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Target PgSQL node not available' register_restore_catalog(db) - sys.exit(1) - + sys.exit(1) + try: global_parameters['pgsql_node_backup_dir'] = db.get_pgsql_node_config_value(global_parameters['pgsql_node_id'],'pgnode_backup_partition') except psycopg2.Error as e: global_parameters['pgsql_node_backup_dir'] = pgsql_node_cache_data['pgnode_backup_partition'] - + global_parameters['backup_server_pgsql_bin_dir'] = get_backup_server_pgsql_bin_dir(db) global_parameters['target_pgsql_node_release'] = get_pgsql_node_release(db,db_pgnode) @@ -663,24 +665,24 @@ def main(): global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'Backup pgsql release > Target pgsql node release' register_restore_catalog(db) - sys.exit(1) + sys.exit(1) if global_parameters['renamed_dbname'] != '': rename_existing_database(db,db_pgnode) logs.logger.info('Renaming an existing database to %s before restoring the backup in the target database',global_parameters['renamed_dbname']) - + # # Check if the target database already exists in the target PgSQL # node. Stop the backup with an error it it exists. # - + if db_pgnode.database_exists(global_parameters['target_dbname']): logs.logger.critical('The target database already exists in the target PgSQL node') global_parameters['execution_status'] = 'ERROR' global_parameters['error_message'] = 'The target database already exists in the target PgSQL node' register_restore_catalog(db) - sys.exit(1) + sys.exit(1) restore_roles(db) restore_dbconfig(db) @@ -693,14 +695,14 @@ def main(): # ############################################ # Function signal_handler() # ############################################ - + def signal_handler(signum, frame): logs.logger.info('**** pgbackman_dump stopped. ****') sys.exit(0) # ############################################ -# +# # ############################################ if __name__ == '__main__': @@ -725,7 +727,7 @@ if __name__ == '__main__': parser.add_argument('--pg-release', metavar='PG-RELEASE', required=True, help='PG release from backup', dest='pg_release') parser.add_argument('--root-backup-dir', metavar='ROOT-BACKUP-DIR', default=True, required=True, help='Root backup dir', dest='root_backup_dir') - args = parser.parse_args() + args = parser.parse_args() if args.pgsql_node_fqdn: global_parameters['pgsql_node_fqdn'] = args.pgsql_node_fqdn @@ -744,19 +746,19 @@ if __name__ == '__main__': else: print('PgSQL node port parameter not defined') sys.exit(1) - + if args.pgsql_node_admin_user: global_parameters['pgsql_node_admin_user'] = args.pgsql_node_admin_user else: print('PgSQL node admin user parameter not defined') sys.exit(1) - + if args.restore_def: global_parameters['restore_def'] = args.restore_def else: print('RestoreID parameter not defined') sys.exit(1) - + if args.pgdump_file: global_parameters['pgdump_file'] = args.pgdump_file else: @@ -779,7 +781,7 @@ if __name__ == '__main__': global_parameters['source_dbname'] = args.source_dbname else: print('Source dbname parameter not defined') - sys.exit(1) + sys.exit(1) if args.target_dbname: global_parameters['target_dbname'] = args.target_dbname @@ -820,8 +822,7 @@ if __name__ == '__main__': logs = PgbackmanLogs("pgbackman_restore", "[" + global_parameters['pgsql_node_fqdn'] + "]", "[" + global_parameters['target_dbname'] + "]") logs.logger.info('**** pgbackman_restore started. ****') - + main() logs.logger.info('**** pgbackman_restore finished. ****') - diff --git a/pgbackman/database.py b/pgbackman/database.py index 8fef535..33cce06 100644 --- a/pgbackman/database.py +++ b/pgbackman/database.py @@ -55,7 +55,7 @@ def __init__(self, dsn,application): self.output_format = 'table' - + # ############################################ # Method pg_connect() # ############################################ @@ -65,7 +65,7 @@ def pg_connect(self): try: self.conn = psycopg2.connect(self.dsn) - + if self.conn: self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) psycopg2.extras.wait_select(self.conn) @@ -75,13 +75,13 @@ def pg_connect(self): self.server_version = self.conn.server_version if (self.server_version >= 90000 and 'application_name=' not in self.dsn): - + try: self.cur.execute('SET application_name TO %s',(self.application,)) self.conn.commit() except psycopg2.Error as e: raise e - + except psycopg2.Error as e: raise e @@ -97,17 +97,17 @@ def pg_close(self): try: self.cur.close() except psycopg2.Error as e: - raise e + raise e if self.conn: try: - self.conn.close() + self.conn.close() except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def get_server_version(self): @@ -115,13 +115,13 @@ def get_server_version(self): try: return self.server_version - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def show_backup_servers(self): @@ -139,15 +139,15 @@ def show_backup_servers(self): except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def register_backup_server(self,hostname,domain,status,remarks): @@ -159,21 +159,21 @@ def register_backup_server(self,hostname,domain,status,remarks): if self.cur: try: self.cur.execute('SELECT register_backup_server(%s,%s,%s,%s)',(hostname,domain,status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def delete_backup_server(self,server_id): """A function to delete a backup server""" @@ -183,19 +183,19 @@ def delete_backup_server(self,server_id): if self.cur: try: self.cur.execute('SELECT delete_backup_server(%s)',(server_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_pgsql_nodes(self): @@ -209,19 +209,19 @@ def show_pgsql_nodes(self): self.cur.execute('SELECT * FROM show_pgsql_nodes') self.conn.commit() - return self.cur + return self.cur except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def register_pgsql_node(self,hostname,domain,port,admin_user,status,remarks): @@ -233,19 +233,19 @@ def register_pgsql_node(self,hostname,domain,port,admin_user,status,remarks): if self.cur: try: self.cur.execute('SELECT register_pgsql_node(%s,%s,%s,%s,%s,%s)',(hostname,domain,port,admin_user,status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def delete_pgsql_node(self,node_id): @@ -257,19 +257,19 @@ def delete_pgsql_node(self,node_id): if self.cur: try: self.cur.execute('SELECT delete_pgsql_node(%s)',(node_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def register_backup_definition(self,backup_server,pgsql_node,dbname,minutes_cron,hours_cron,day_month_cron, \ @@ -285,19 +285,19 @@ def register_backup_definition(self,backup_server,pgsql_node,dbname,minutes_cron self.cur.execute('SELECT register_backup_definition(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',(backup_server,pgsql_node,dbname,minutes_cron,hours_cron,day_month_cron, \ month_cron,weekday_cron,backup_code,encryption, \ retention_period,retention_redundancy,extra_backup_parameters,job_status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_backup_definition_id(self,def_id): @@ -309,19 +309,19 @@ def delete_backup_definition_id(self,def_id): if self.cur: try: self.cur.execute('SELECT delete_backup_definition_id(%s)',(def_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_force_backup_definition_id(self,def_id): @@ -333,19 +333,19 @@ def delete_force_backup_definition_id(self,def_id): if self.cur: try: self.cur.execute('SELECT delete_force_backup_definition_id(%s)',(def_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_backup_definition_dbname(self,pgsql_node_id,dbname): @@ -357,19 +357,19 @@ def delete_backup_definition_dbname(self,pgsql_node_id,dbname): if self.cur: try: self.cur.execute('SELECT delete_backup_definition_dbname(%s,%s)',(pgsql_node_id,dbname)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def delete_force_backup_definition_dbname(self,pgsql_node_id,dbname): @@ -381,19 +381,19 @@ def delete_force_backup_definition_dbname(self,pgsql_node_id,dbname): if self.cur: try: self.cur.execute('SELECT delete_force_backup_definition_dbname(%s,%s)',(pgsql_node_id,dbname)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_definitions(self,backup_server_list,pgsql_node_list,dbname_list): @@ -407,59 +407,59 @@ def show_backup_definitions(self,backup_server_list,pgsql_node_list,dbname_list) if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"DefID\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"Schedule\",\"Code\",\"Retention\",\"Status\",\"Parameters\" FROM show_backup_definitions WHERE TRUE ' + server_sql + node_sql + dbname_sql) - + return self.cur - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def register_snapshot_definition(self,backup_server,pgsql_node,dbname,at_time,backup_code,retention_period,extra_backup_parameters,remarks,pg_dump_release): @@ -472,19 +472,19 @@ def register_snapshot_definition(self,backup_server,pgsql_node,dbname,at_time,ba try: self.cur.execute('SELECT register_snapshot_definition(%s,%s,%s,%s,%s,%s,%s,%s,%s)',(backup_server,pgsql_node,dbname,at_time,backup_code, \ retention_period,extra_backup_parameters,remarks,pg_dump_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_snapshot_definitions(self,backup_server_list,pgsql_node_list,dbname_list): @@ -498,58 +498,58 @@ def show_snapshot_definitions(self,backup_server_list,pgsql_node_list,dbname_lis if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"SnapshotID\",\"Registered\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"AT time\",\"Code\",\"Retention\",\"Parameters\",\"Status\" FROM show_snapshot_definitions WHERE TRUE ' + server_sql + node_sql + dbname_sql) return self.cur - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_restore_definitions(self,backup_server_list,pgsql_node_list,dbname_list): @@ -563,59 +563,59 @@ def show_restore_definitions(self,backup_server_list,pgsql_node_list,dbname_list if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "Target DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"RestoreDef\",\"Registered\",\"BckID\",target_pgsql_node_id AS \"ID\",\"Target PgSQL node\",\"Target DBname\",\"Renamed database\",\"AT time\",\"Extra parameters\",\"Status\" FROM show_restore_definitions WHERE TRUE ' + server_sql + node_sql + dbname_sql) return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_catalog(self,backup_server_list,pgsql_node_list,dbname_list,def_id_list,status_list): @@ -629,80 +629,80 @@ def show_backup_catalog(self,backup_server_list,pgsql_node_list,dbname_list,def_ if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + if def_id_list != None: def_id_sql = 'AND (FALSE ' - + for def_id in def_id_list: def_id_sql = def_id_sql + 'OR def_id = \'' + def_id + '\' ' - + def_id_sql = def_id_sql + ') ' - + else: def_id_sql = '' if status_list != None: status_sql = 'AND (FALSE ' - + for status in status_list: status_sql = status_sql + 'OR "Status" = \'' + status + '\' ' - + status_sql = status_sql + ') ' - + else: status_sql = '' - - + + self.cur.execute('SELECT \"BckID\",\"DefID\",\"SnapshotID\",\"Finished\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"Duration\",\"Size\",\"Code\",\"Execution\",\"Status\" FROM show_backup_catalog WHERE TRUE ' + server_sql + node_sql + dbname_sql + def_id_sql + status_sql) - + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_restore_catalog(self,backup_server_list,pgsql_node_list,dbname_list): @@ -716,58 +716,58 @@ def show_restore_catalog(self,backup_server_list,pgsql_node_list,dbname_list): if backup_server_list != None: server_sql = 'AND (FALSE ' - + for server in backup_server_list: if server.isdigit(): server_sql = server_sql + 'OR backup_server_id = ' + str(server) + ' ' else: server_sql = server_sql + 'OR backup_server_id = ' + str(self.get_backup_server_id(server.lower())) + ' ' - + server_sql = server_sql + ') ' - + else: server_sql = '' - + if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR target_pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "Target DBname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + self.cur.execute('SELECT \"RestoreID\",\"RestoreDef\",\"BckID\",\"Finished\",backup_server_id AS \"ID.\",\"Backup server\",target_pgsql_node_id AS \"ID\",\"Target PgSQL node\",\"Target DBname\",\"Duration\",\"Status\" FROM show_restore_catalog WHERE TRUE ' + server_sql + node_sql + dbname_sql) return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_details(self,bck_id): @@ -779,7 +779,7 @@ def show_backup_details(self,bck_id): if self.cur: try: self.cur.execute("SELECT * FROM show_backup_details WHERE bck_id= %s",(bck_id,)) - + result = OrderedDict() for record in self.cur: @@ -827,14 +827,14 @@ def show_backup_details(self,bck_id): except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ def show_restore_details(self,restore_id): @@ -846,11 +846,11 @@ def show_restore_details(self,restore_id): if self.cur: try: self.cur.execute("SELECT * FROM show_restore_details WHERE restore_id= %s",(restore_id,)) - + result = OrderedDict() for record in self.cur: - + result['RestoreID'] = str(record[0]) result['ProcPID'] = str(record[5]) result['Registered'] = str(record[2]) @@ -883,15 +883,15 @@ def show_restore_details(self,restore_id): except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e + raise e # ############################################ - # Method + # Method # ############################################ def show_databases_without_backup_definitions(self,pgsql_node): @@ -900,11 +900,11 @@ def show_databases_without_backup_definitions(self,pgsql_node): try: result = [] - + pgsql_node_list = [] if pgsql_node.lower() in ['all','*']: - + for id,fqdn in self.get_pgsql_nodes_list(): pgsql_node_list.append(str(id)) @@ -922,7 +922,7 @@ def show_databases_without_backup_definitions(self,pgsql_node): dsn_value = self.get_pgsql_node_dsn(pgsql_node_id) db_node = self.__class__(dsn_value,'pgbackman_cli') - + database_list = [] all_databases = [] databases_with_bckdef = [] @@ -934,11 +934,11 @@ def show_databases_without_backup_definitions(self,pgsql_node): # If it is down, we continue with the next one in # pgsql_node_list # - + try: for database in db_node.get_pgsql_node_database_list(): all_databases.append(database[0]) - + except Exception as e: continue @@ -951,13 +951,13 @@ def show_databases_without_backup_definitions(self,pgsql_node): result.append((pgsql_node_fqdn,database)) return result - + except Exception as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_snapshots_in_progress(self): @@ -968,22 +968,22 @@ def show_snapshots_in_progress(self): if self.cur: try: - + self.cur.execute('SELECT \"SnapshotID\",\"Registered\",backup_server_id AS \"ID.\",\"Backup server\",pgsql_node_id AS \"ID\",\"PgSQL node\",\"DBname\",\"AT time\",\"Code\",\"Elapsed time\" FROM show_snapshots_in_progress') return self.cur - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_restores_in_progress(self): @@ -994,22 +994,22 @@ def show_restores_in_progress(self): if self.cur: try: - + self.cur.execute('SELECT \"RestoreDef\",\"Registered\",\"BckID\",backup_server_id AS \"ID.\",\"Backup server\",target_pgsql_node_id AS \"ID\",\"Target PgSQL node\",\"Target DBname\",\"AT time\",\"Elapsed time\" FROM show_restores_in_progress') - + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def get_default_backup_server_parameter(self,param): @@ -1021,7 +1021,7 @@ def get_default_backup_server_parameter(self,param): if self.cur: try: self.cur.execute('SELECT get_default_backup_server_parameter(%s)',(param,)) - + data = self.cur.fetchone()[0] return data @@ -1029,15 +1029,15 @@ def get_default_backup_server_parameter(self,param): raise e self.pg_close() - + except psycopg2.Error as e: - raise e + raise e + - # ############################################ - # Method + # Method # ############################################ - + def get_default_pgsql_node_parameter(self,param): """A function to get the default value of a configuration parameter""" @@ -1047,23 +1047,23 @@ def get_default_pgsql_node_parameter(self,param): if self.cur: try: self.cur.execute('SELECT get_default_pgsql_node_parameter(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ - + def get_minute_from_interval(self,param): """A function to get a random minute from an interval""" @@ -1073,13 +1073,13 @@ def get_minute_from_interval(self,param): if self.cur: try: self.cur.execute('SELECT get_minute_from_interval(%s)',(param,)) - + data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: @@ -1087,35 +1087,35 @@ def get_minute_from_interval(self,param): # ############################################ - # Method + # Method # ############################################ - + def get_hour_from_interval(self,param): """A function to get a random hour from an interval""" - + try: self.pg_connect() if self.cur: try: self.cur.execute('SELECT get_hour_from_interval(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_fqdn(self,param): """A function to get the FQDN of a backup server""" @@ -1125,7 +1125,7 @@ def get_backup_server_fqdn(self,param): if self.cur: try: self.cur.execute('SELECT get_backup_server_fqdn(%s)',(param,)) - + data = self.cur.fetchone()[0] return data @@ -1133,15 +1133,15 @@ def get_backup_server_fqdn(self,param): raise e self.pg_close() - + except Exception as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_id(self,param): """A function to get the ID of a backup server""" @@ -1151,23 +1151,23 @@ def get_backup_server_id(self,param): if self.cur: try: self.cur.execute('SELECT get_backup_server_id(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() - + except Exception as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_fqdn(self,param): """A function to get the FQDN of a PgSQL node""" @@ -1177,13 +1177,13 @@ def get_pgsql_node_fqdn(self,param): if self.cur: try: self.cur.execute('SELECT get_pgsql_node_fqdn(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: @@ -1191,9 +1191,9 @@ def get_pgsql_node_fqdn(self,param): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_id(self,param): """A function to get the ID of a PgSQL node""" @@ -1203,34 +1203,34 @@ def get_pgsql_node_id(self,param): if self.cur: try: self.cur.execute('SELECT get_pgsql_node_id(%s)',(param,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_next_crontab_id_to_generate(self,param): """A function to get the next PgSQL node ID to generate a crontab file for""" try: self.pg_connect() - + if self.cur: try: self.cur.execute('SELECT get_next_crontab_id_to_generate(%s)',(param,)) self.conn.commit() - + data = self.cur.fetchone()[0] return data @@ -1238,28 +1238,28 @@ def get_next_crontab_id_to_generate(self,param): raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def generate_crontab_backup_jobs(self,backup_server_id,pgsql_node_id): """A function to get the crontab file for a PgSQL node in a backup server""" try: self.pg_connect() - + if self.cur: try: self.cur.execute('SELECT generate_crontab_backup_jobs(%s,%s)',(backup_server_id,pgsql_node_id)) - + data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -1267,36 +1267,36 @@ def generate_crontab_backup_jobs(self,backup_server_id,pgsql_node_id): except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_job_queue(self,backup_server_id,pgsql_node_id): """A function to update the backup job queue if the crontab generation fails""" try: self.pg_connect() - + if self.cur: try: self.cur.execute('SELECT update_job_queue(%s,%s)',(backup_server_id,pgsql_node_id)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_dsn(self,pgsql_node_id): """A function to DSN value for a PgSQL node in a backup server""" @@ -1306,34 +1306,34 @@ def get_pgsql_node_dsn(self,pgsql_node_id): if self.cur: try: self.cur.execute('SELECT get_pgsql_node_dsn(%s)',(pgsql_node_id,)) - + data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: return None self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def register_backup_catalog_1_0_0(self,def_id,procpid,backup_server_id,pgsql_node_id,dbname,started,finished,duration,pg_dump_file, pg_dump_file_size,pg_dump_log_file,pg_dump_roles_file,pg_dump_roles_file_size,pg_dump_roles_log_file, pg_dump_dbconfig_file,pg_dump_dbconfig_file_size,pg_dump_dbconfig_log_file,global_log_file,execution_status, execution_method,error_message,snapshot_id,role_list,pgsql_node_release): - + """A function to update the backup job catalog in a 1.0.0 version""" try: self.pg_connect() - + if self.cur: try: @@ -1361,32 +1361,32 @@ def register_backup_catalog_1_0_0(self,def_id,procpid,backup_server_id,pgsql_nod snapshot_id, role_list, pgsql_node_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def register_backup_catalog(self,def_id,procpid,backup_server_id,pgsql_node_id,dbname,started,finished,duration,pg_dump_file, pg_dump_file_size,pg_dump_log_file,pg_dump_roles_file,pg_dump_roles_file_size,pg_dump_roles_log_file, pg_dump_dbconfig_file,pg_dump_dbconfig_file_size,pg_dump_dbconfig_log_file,global_log_file,execution_status, execution_method,error_message,snapshot_id,role_list,pgsql_node_release,pg_dump_release): - + """A function to update the backup job catalog""" try: self.pg_connect() - + if self.cur: try: @@ -1415,30 +1415,30 @@ def register_backup_catalog(self,def_id,procpid,backup_server_id,pgsql_node_id,d role_list, pgsql_node_release, pg_dump_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def register_restore_catalog(self,restore_def,procpid,backup_server_id,target_pgsql_node_id,source_dbname,target_dbname,renamed_dbname,started,finished,duration,restore_log_file, global_log_file,execution_status,error_message,role_list,target_pgsql_node_release,backup_pg_release): - + """A function to update the restore job catalog""" try: self.pg_connect() - + if self.cur: try: @@ -1460,32 +1460,32 @@ def register_restore_catalog(self,restore_def,procpid,backup_server_id,target_pg target_pgsql_node_release, backup_pg_release)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - self.pg_close() + self.pg_close() except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def print_results_table(self,cur,colnames,left_columns): '''A function to print a table with sql results''' - + if self.output_format == 'table': - + x = PrettyTable(colnames) x.padding_width = 1 - + for column in left_columns: x.align[column] = "l" - + for records in cur: columns = [] @@ -1493,28 +1493,28 @@ def print_results_table(self,cur,colnames,left_columns): columns.append(records[index]) x.add_row(columns) - + print x.get_string() print elif self.output_format == 'csv': - + for records in cur: columns = [] - + for index in range(len(colnames)): columns.append(str(records[index])) - + print ','.join(columns) # ############################################ - # Method + # Method # ############################################ - + def get_catalog_entries_to_delete(self,backup_server_id): """A function to get catalog information about force deletion of backup job definitions""" - + try: self.pg_connect() @@ -1522,9 +1522,9 @@ def get_catalog_entries_to_delete(self,backup_server_id): try: self.cur.execute('SELECT * FROM get_catalog_entries_to_delete WHERE backup_server_id = %s',(backup_server_id,)) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -1535,9 +1535,9 @@ def get_catalog_entries_to_delete(self,backup_server_id): # ############################################ - # Method + # Method # ############################################ - + def delete_catalog_entries_to_delete(self,del_id): """A function to delete catalog info from defid force deletions""" @@ -1547,21 +1547,21 @@ def delete_catalog_entries_to_delete(self,del_id): if self.cur: try: self.cur.execute('SELECT delete_catalog_entries_to_delete(%s)',(del_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_cron_catalog_entries_to_delete_by_retention(self,backup_server_id): """A function to get backup catalog entries to delete""" @@ -1571,23 +1571,23 @@ def get_cron_catalog_entries_to_delete_by_retention(self,backup_server_id): if self.cur: try: self.cur.execute('SELECT * FROM get_cron_catalog_entries_to_delete_by_retention WHERE backup_server_id = %s',(backup_server_id,)) - self.conn.commit() - + self.conn.commit() + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_at_catalog_entries_to_delete_by_retention(self,backup_server_id): """A function to get snapshot catalog entries to delete""" @@ -1597,26 +1597,26 @@ def get_at_catalog_entries_to_delete_by_retention(self,backup_server_id): if self.cur: try: self.cur.execute('SELECT * FROM get_at_catalog_entries_to_delete_by_retention WHERE backup_server_id = %s',(backup_server_id,)) - self.conn.commit() - + self.conn.commit() + return self.cur except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_restore_logs_to_delete(self,backup_server_id): """A function to get restore log files to delete""" - + try: self.pg_connect() @@ -1624,9 +1624,9 @@ def get_restore_logs_to_delete(self,backup_server_id): try: self.cur.execute('SELECT * FROM get_restore_logs_to_delete WHERE backup_server_id = %s',(backup_server_id,)) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -1636,9 +1636,9 @@ def get_restore_logs_to_delete(self,backup_server_id): raise e # ############################################ - # Method + # Method # ############################################ - + def delete_restore_logs_to_delete(self,del_id): """A function to delete restore logs to delete information""" @@ -1648,21 +1648,21 @@ def delete_restore_logs_to_delete(self,del_id): if self.cur: try: self.cur.execute('SELECT delete_restore_logs_to_delete(%s)',(del_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def delete_backup_catalog(self,bck_id): """A function to delete entries from backup job catalog""" @@ -1672,20 +1672,20 @@ def delete_backup_catalog(self,bck_id): if self.cur: try: self.cur.execute('SELECT delete_backup_catalog(%s)',(bck_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def delete_snapshot_definition(self,snapshot_id): """A function to delete entries from snapshot_definition""" @@ -1695,41 +1695,41 @@ def delete_snapshot_definition(self,snapshot_id): if self.cur: try: self.cur.execute('SELECT delete_snapshot_definition(%s)',(snapshot_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def add_listen(self,channel): '''Subscribe to a PostgreSQL NOTIFY''' replace_list = ['.','-'] - + for i,j in enumerate(replace_list): channel = channel.replace(j, '_') - + sql = "LISTEN %s" % channel - + try: self.cur.execute(sql) self.conn.commit() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def delete_listen(self,channel): @@ -1739,27 +1739,27 @@ def delete_listen(self,channel): for i,j in enumerate(replace_list): channel = channel.replace(j, '_') - + sql = "UNLISTEN %s" % channel try: self.cur.execute(sql) self.conn.commit() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_listen_channel_names(self,param): """A function to get a list of channels to LISTEN for a backup_server""" try: list = [] - + self.cur.execute('SELECT get_listen_channel_names(%s)',(param,)) self.conn.commit() @@ -1767,13 +1767,13 @@ def get_listen_channel_names(self,param): list.append(row[0]) return list - + except psycopg2.Error as e: return e # ############################################ - # Method + # Method # ############################################ def show_jobs_queue(self): @@ -1788,18 +1788,18 @@ def show_jobs_queue(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_backup_server_config(self,backup_server_id): @@ -1819,12 +1819,12 @@ def show_backup_server_config(self,backup_server_id): raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_pgsql_node_config(self,pgsql_node_id): @@ -1839,23 +1839,23 @@ def show_pgsql_node_config(self,pgsql_node_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def show_pgbackman_stats(self): """A function to get pgbackman global stats""" - + try: self.pg_connect() @@ -1872,16 +1872,16 @@ def show_pgbackman_stats(self): self.cur.execute("SELECT count(*) FROM pgsql_node WHERE status = 'STOPPED'") pgsql_node_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id,dbname) def_id FROM backup_definition) AS cnt") dbname_cnt = self.cur.fetchone()[0] self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'ACTIVE'") backup_jobs_active_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'STOPPED'") backup_jobs_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'CLUSTER'") backup_jobs_cluster_cnt = self.cur.fetchone()[0] @@ -1893,28 +1893,28 @@ def show_pgbackman_stats(self): self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'SCHEMA'") backup_jobs_schema_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'SUCCEEDED'") backup_catalog_succeeded_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'ERROR'") backup_catalog_error_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT pg_size_pretty(sum(pg_dump_file_size+pg_dump_roles_file_size+pg_dump_dbconfig_file_size)) FROM backup_catalog") backup_space = self.cur.fetchone()[0] - + self.cur.execute("SELECT sum(duration) FROM backup_catalog") backup_duration = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',min(finished)) from backup_catalog;") oldest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',max(finished)) from backup_catalog;") newest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM job_queue") job_queue_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM catalog_entries_to_delete") defid_force_deletion_cnt = self.cur.fetchone()[0] @@ -1943,25 +1943,25 @@ def show_pgbackman_stats(self): result['####'] = '' result['Jobs waiting to be processed by pgbackman_control'] = str(job_queue_cnt) result['Forced deletion of backups waiting to be processed'] = str(defid_force_deletion_cnt) - + return result except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ def show_backup_server_stats(self,backup_server_id): """A function to get global stats for a backup server""" - + try: self.pg_connect() @@ -1972,16 +1972,16 @@ def show_backup_server_stats(self,backup_server_id): self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id) pgsql_node_id FROM backup_definition WHERE backup_server_id = %s) AS cnt",(backup_server_id,)) pgsql_node_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id,dbname) def_id FROM backup_definition WHERE backup_server_id = %s) AS cnt",(backup_server_id,)) dbname_cnt = self.cur.fetchone()[0] self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'ACTIVE' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_active_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'STOPPED' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'CLUSTER' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_cluster_cnt = self.cur.fetchone()[0] @@ -1993,33 +1993,33 @@ def show_backup_server_stats(self,backup_server_id): self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'SCHEMA' AND backup_server_id = %s",(backup_server_id,)) backup_jobs_schema_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'SUCCEEDED' AND backup_server_id = %s",(backup_server_id,)) backup_catalog_succeeded_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'ERROR' AND backup_server_id = %s",(backup_server_id,)) backup_catalog_error_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT pg_size_pretty(sum(pg_dump_file_size+pg_dump_roles_file_size+pg_dump_dbconfig_file_size)) FROM backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) backup_space = self.cur.fetchone()[0] - + self.cur.execute("SELECT sum(duration) FROM backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) backup_duration = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',min(finished)) from backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) oldest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',max(finished)) from backup_catalog WHERE backup_server_id = %s",(backup_server_id,)) newest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM job_queue WHERE backup_server_id = %s",(backup_server_id,)) job_queue_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM catalog_entries_to_delete WHERE backup_server_id = %s",(backup_server_id,)) defid_force_deletion_cnt = self.cur.fetchone()[0] - + result = OrderedDict() - + result['Backup server'] = backup_server_fqdn result['#'] = '' result['PgSQL nodes using this backup server'] = str(pgsql_node_cnt) @@ -2043,23 +2043,23 @@ def show_backup_server_stats(self,backup_server_id): result['Forced deletion of backups waiting to be processed'] = str(defid_force_deletion_cnt) return result - + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e + raise e + - # ############################################ - # Method + # Method # ############################################ def show_pgsql_node_stats(self,pgsql_node_id): """A function to get global stats for a backup server""" - + try: self.pg_connect() @@ -2070,16 +2070,16 @@ def show_pgsql_node_stats(self,pgsql_node_id): self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (backup_server_id) backup_server_id FROM backup_definition WHERE pgsql_node_id = %s) AS cnt",(pgsql_node_id,)) backup_server_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM (SELECT DISTINCT ON (pgsql_node_id,dbname) def_id FROM backup_definition WHERE pgsql_node_id = %s) AS cnt",(pgsql_node_id,)) dbname_cnt = self.cur.fetchone()[0] self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'ACTIVE' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_active_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE job_status = 'STOPPED' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_stopped_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'CLUSTER' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_cluster_cnt = self.cur.fetchone()[0] @@ -2091,30 +2091,30 @@ def show_pgsql_node_stats(self,pgsql_node_id): self.cur.execute("SELECT count(*) FROM backup_definition WHERE backup_code = 'SCHEMA' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_jobs_schema_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'SUCCEEDED' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_catalog_succeeded_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM backup_catalog WHERE execution_status = 'ERROR' AND pgsql_node_id = %s",(pgsql_node_id,)) backup_catalog_error_cnt = self.cur.fetchone()[0] - + self.cur.execute("SELECT pg_size_pretty(sum(pg_dump_file_size+pg_dump_roles_file_size+pg_dump_dbconfig_file_size)) FROM backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) backup_space = self.cur.fetchone()[0] - + self.cur.execute("SELECT sum(duration) FROM backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) backup_duration = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',min(finished)) from backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) oldest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("select date_trunc('seconds',max(finished)) from backup_catalog WHERE pgsql_node_id = %s",(pgsql_node_id,)) newest_backup_job = self.cur.fetchone()[0] - + self.cur.execute("SELECT count(*) FROM job_queue WHERE pgsql_node_id = %s",(pgsql_node_id,)) job_queue_cnt = self.cur.fetchone()[0] - + result = OrderedDict() - + result['PgSQL node'] = "[" + str(pgsql_node_id) + "] " + pgsql_node_fqdn result['#'] = '' result['Backup servers running backups for this Node'] = str(backup_server_cnt) @@ -2135,25 +2135,25 @@ def show_pgsql_node_stats(self,pgsql_node_id): result['Newest backup in catalog'] = str(newest_backup_job) result['####'] = '' result['Jobs waiting to be processed by pgbackman_control'] = str(job_queue_cnt) - + return result except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: - raise e - + raise e + # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_to_delete(self,backup_server_id): """A function to get the PgSQL node data from nodes that has been deleted""" - + try: self.pg_connect() @@ -2161,9 +2161,9 @@ def get_pgsql_node_to_delete(self,backup_server_id): try: self.cur.execute('SELECT backup_server_id,pgsql_node_id FROM pgsql_node_to_delete WHERE backup_server_id = %s',(backup_server_id,)) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2173,12 +2173,12 @@ def get_pgsql_node_to_delete(self,backup_server_id): raise e # ############################################ - # Method + # Method # ############################################ - + def delete_pgsql_node_to_delete(self,backup_server_id,pgsql_node_id): """A function to delete the PgSQL node data from a node that has been deleted""" - + try: self.pg_connect() @@ -2186,7 +2186,7 @@ def delete_pgsql_node_to_delete(self,backup_server_id,pgsql_node_id): try: self.cur.execute('SELECT delete_pgsql_node_to_delete(%s,%s)',(backup_server_id,pgsql_node_id)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2197,12 +2197,12 @@ def delete_pgsql_node_to_delete(self,backup_server_id,pgsql_node_id): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_stopped(self): """A function to get data for PgSQL nodes stopped when pgbackman_control was down""" - + try: self.pg_connect() @@ -2210,9 +2210,9 @@ def get_pgsql_node_stopped(self): try: self.cur.execute('SELECT pgsql_node_id FROM pgsql_node_stopped') self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2223,7 +2223,7 @@ def get_pgsql_node_stopped(self): # ############################################ - # Method + # Method # ############################################ def show_empty_backup_catalogs(self): @@ -2238,18 +2238,18 @@ def show_empty_backup_catalogs(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def database_exists(self,dbname): @@ -2267,20 +2267,20 @@ def database_exists(self,dbname): if dbname_cnt > 0: return True - + elif dbname_cnt == 0: return False - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def role_exists(self,role): @@ -2298,21 +2298,21 @@ def role_exists(self,role): if role_cnt > 0: return True - + elif role_cnt == 0: return False - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_pgsql_node_database_list(self): @@ -2327,20 +2327,20 @@ def get_pgsql_node_database_list(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def update_backup_server(self,backup_server_id,remarks): """A function to update a backup server""" @@ -2350,21 +2350,21 @@ def update_backup_server(self,backup_server_id,remarks): if self.cur: try: self.cur.execute('SELECT update_backup_server(%s,%s)',(backup_server_id,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_pgsql_node(self,pgsql_node_id,port,admin_user,status,remarks): """A function to update a PgSQL node""" @@ -2374,21 +2374,21 @@ def update_pgsql_node(self,pgsql_node_id,port,admin_user,status,remarks): if self.cur: try: self.cur.execute('SELECT update_pgsql_node(%s,%s,%s,%s,%s)',(pgsql_node_id,port,admin_user,status,remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_pgsql_node_config(self,pgsql_node_id,backup_minutes_interval,backup_hours_interval,backup_weekday_cron, backup_month_cron,backup_day_month_cron,backup_code,retention_period,retention_redundancy,automatic_deletion_retention, extra_backup_parameters,extra_restore_parameters,backup_job_status,domain,logs_email,admin_user,pgport,pgnode_backup_partition, @@ -2420,22 +2420,22 @@ def update_pgsql_node_config(self,pgsql_node_id,backup_minutes_interval,backup_h pgnode_backup_partition, pgnode_crontab_file, pgsql_node_status)) - - self.conn.commit() - + + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def update_backup_server_config(self,backup_server_id,pgsql_bin_9_0,pgsql_bin_9_1,pgsql_bin_9_2,pgsql_bin_9_3,pgsql_bin_9_4,pgsql_bin_9_5,pgsql_bin_9_6,pgsql_bin_10,root_backup_partition): """A function to update the configuration of a backup server""" @@ -2454,21 +2454,21 @@ def update_backup_server_config(self,backup_server_id,pgsql_bin_9_0,pgsql_bin_9_ pgsql_bin_9_6, pgsql_bin_10, root_backup_partition)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def check_pgsql_node_status(self,pgsql_node_id): """A function to check if a PgSQL node is stopped""" @@ -2478,24 +2478,24 @@ def check_pgsql_node_status(self,pgsql_node_id): if self.cur: try: self.cur.execute('SELECT check_pgsql_node_status(%s)',(pgsql_node_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_new_snapshots(self,backup_server_id): """A function to get new snapshots to be run in a backup server""" - + try: self.pg_connect() @@ -2503,9 +2503,9 @@ def get_new_snapshots(self,backup_server_id): try: self.cur.execute('SELECT "SnapshotID","AT time" FROM show_snapshot_definitions WHERE backup_server_id = %s AND "Status" = %s',(backup_server_id,'WAITING')) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2516,12 +2516,12 @@ def get_new_snapshots(self,backup_server_id): # ############################################ - # Method + # Method # ############################################ - + def generate_snapshot_at_file(self,snapshot_id): """A function to generate a at file for a snapshot""" - + try: self.pg_connect() @@ -2529,9 +2529,9 @@ def generate_snapshot_at_file(self,snapshot_id): try: self.cur.execute('SELECT generate_snapshot_at_file(%s)',(snapshot_id,)) self.conn.commit() - + return self.cur.fetchone()[0] - + except psycopg2.Error as e: raise e @@ -2542,12 +2542,12 @@ def generate_snapshot_at_file(self,snapshot_id): # ############################################ - # Method + # Method # ############################################ - + def update_snapshot_status(self,snapshot_id,status): """A function to update the status for a snapshot""" - + try: self.pg_connect() @@ -2555,7 +2555,7 @@ def update_snapshot_status(self,snapshot_id,status): try: self.cur.execute('SELECT update_snapshot_status(%s,%s)',(snapshot_id,status)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2566,9 +2566,9 @@ def update_snapshot_status(self,snapshot_id,status): # ############################################ - # Method + # Method # ############################################ - + def get_dbname_from_bckid(self,bck_id): """A function to get the dbname from a bckID""" @@ -2578,13 +2578,13 @@ def get_dbname_from_bckid(self,bck_id): if self.cur: try: self.cur.execute('SELECT get_dbname_from_bckid(%s)',(bck_id,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: @@ -2592,9 +2592,9 @@ def get_dbname_from_bckid(self,bck_id): # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_id_from_bckid(self,bck_id): """A function to get the backup server ID from a bckID""" @@ -2604,13 +2604,13 @@ def get_backup_server_id_from_bckid(self,bck_id): if self.cur: try: self.cur.execute('SELECT get_backup_server_id_from_bckid(%s)',(bck_id,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: @@ -2618,9 +2618,9 @@ def get_backup_server_id_from_bckid(self,bck_id): # ############################################ - # Method + # Method # ############################################ - + def get_role_list_from_bckid(self,bck_id): """A function to get the role list from a bckID""" @@ -2630,21 +2630,21 @@ def get_role_list_from_bckid(self,bck_id): if self.cur: try: self.cur.execute('SELECT get_role_list_from_bckid(%s)',(bck_id,)) - + data = self.cur.fetchone()[0] return data except psycopg2.Error as e: raise e - + self.pg_close() except Exception as e: raise e - + # ############################################ - # Method + # Method # ############################################ def register_restore_definition(self,at_time,backup_server_id,pgsql_node_id,bck_id,target_dbname,renamed_dbname,extra_restore_parameters,roles_to_restore): @@ -2663,24 +2663,24 @@ def register_restore_definition(self,at_time,backup_server_id,pgsql_node_id,bck_ renamed_dbname, extra_restore_parameters, roles_to_restore)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ - + def get_new_restore(self,backup_server_id): """A function to get new restore jobs to be run in a backup server""" - + try: self.pg_connect() @@ -2688,9 +2688,9 @@ def get_new_restore(self,backup_server_id): try: self.cur.execute('SELECT "RestoreDef","AT time" FROM show_restore_definitions WHERE backup_server_id = %s AND "Status" = %s',(backup_server_id,'WAITING')) self.conn.commit() - + return self.cur - + except psycopg2.Error as e: raise e @@ -2701,12 +2701,12 @@ def get_new_restore(self,backup_server_id): # ############################################ - # Method + # Method # ############################################ - + def generate_restore_at_file(self,restore_def): """A function to generate a at file for a restore""" - + try: self.pg_connect() @@ -2714,9 +2714,9 @@ def generate_restore_at_file(self,restore_def): try: self.cur.execute('SELECT generate_restore_at_file(%s)',(restore_def,)) self.conn.commit() - + return self.cur.fetchone()[0] - + except psycopg2.Error as e: raise e @@ -2727,12 +2727,12 @@ def generate_restore_at_file(self,restore_def): # ############################################ - # Method + # Method # ############################################ - + def update_restore_status(self,restore_id,status): """A function to update the status for a restore""" - + try: self.pg_connect() @@ -2740,7 +2740,7 @@ def update_restore_status(self,restore_id,status): try: self.cur.execute('SELECT update_restore_status(%s,%s)',(restore_id,status)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2751,22 +2751,22 @@ def update_restore_status(self,restore_id,status): # ############################################ - # Method + # Method # ############################################ - + def rename_existing_database(self,from_name,to_name): """A function to rename an existing database before restoring a backup""" - + try: self.pg_connect() if self.cur: try: sql = 'ALTER DATABASE ' + from_name + ' RENAME TO ' + to_name - + self.cur.execute(sql) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -2776,9 +2776,9 @@ def rename_existing_database(self,from_name,to_name): raise e # ############################################ - # Method + # Method # ############################################ - + def update_backup_definition(self,def_id,minutes_cron,hours_cron,weekday_cron,month_cron,day_month_cron,retention_period, retention_redundancy,extra_backup_parameters,job_status,remarks): """A function to update a backup definition""" @@ -2799,19 +2799,19 @@ def update_backup_definition(self,def_id,minutes_cron,hours_cron,weekday_cron,mo extra_backup_parameters, job_status, remarks)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def move_backup_definition(self,from_backup_server, to_backup_server,pgsql_node_list,dbname_list,def_id_list): @@ -2835,61 +2835,61 @@ def move_backup_definition(self,from_backup_server, to_backup_server,pgsql_node_ if pgsql_node_list != None: node_sql = 'AND (FALSE ' - + for node in pgsql_node_list: if node.isdigit(): node_sql = node_sql + 'OR pgsql_node_id = ' + str(node) + ' ' else: node_sql = node_sql + 'OR pgsql_node_id = ' + str(self.get_pgsql_node_id(node.lower())) + ' ' - + node_sql = node_sql + ') ' - + else: - node_sql = '' + node_sql = '' if dbname_list != None: dbname_sql = 'AND (FALSE ' - + for dbname in dbname_list: dbname_sql = dbname_sql + 'OR "dbname" = \'' + dbname + '\' ' - + dbname_sql = dbname_sql + ') ' - + else: dbname_sql = '' - + if def_id_list != None: def_id_sql = 'AND (FALSE ' - + for def_id in def_id_list: def_id_sql = def_id_sql + 'OR def_id = \'' + def_id + '\' ' - + def_id_sql = def_id_sql + ') ' - + else: def_id_sql = '' - + self.cur.execute('UPDATE backup_definition SET backup_server_id = ' + to_server_id + ' WHERE backup_server_id = ' + from_server_id + ' ' + node_sql + dbname_sql + def_id_sql) - - self.conn.commit() + + self.conn.commit() except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_backup_definition_def_value(self,def_id,parameter): """A function to get the value of an attribute from a backup_definition""" - + try: self.pg_connect() @@ -2900,7 +2900,7 @@ def get_backup_definition_def_value(self,def_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2911,12 +2911,12 @@ def get_backup_definition_def_value(self,def_id,parameter): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_config_value(self,pgsql_node_id,parameter): """A function to get the value of a default configuration parameter for a PgSQL node""" - + try: self.pg_connect() @@ -2927,7 +2927,7 @@ def get_pgsql_node_config_value(self,pgsql_node_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2937,12 +2937,12 @@ def get_pgsql_node_config_value(self,pgsql_node_id,parameter): raise e # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_config_value(self,backup_server_id,parameter): """A function to get the value of a default configuration parameter for a backup server""" - + try: self.pg_connect() @@ -2953,7 +2953,7 @@ def get_backup_server_config_value(self,backup_server_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2964,12 +2964,12 @@ def get_backup_server_config_value(self,backup_server_id,parameter): # ############################################ - # Method + # Method # ############################################ - + def get_pgsql_node_def_value(self,pgsql_node_id,parameter): """A function to get the value of an attribute from pgsql_node""" - + try: self.pg_connect() @@ -2980,7 +2980,7 @@ def get_pgsql_node_def_value(self,pgsql_node_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -2991,12 +2991,12 @@ def get_pgsql_node_def_value(self,pgsql_node_id,parameter): # ############################################ - # Method + # Method # ############################################ - + def get_backup_server_def_value(self,backup_server_id,parameter): """A function to get the value of an attribute from backup_server""" - + try: self.pg_connect() @@ -3007,7 +3007,7 @@ def get_backup_server_def_value(self,backup_server_id,parameter): data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e @@ -3018,7 +3018,7 @@ def get_backup_server_def_value(self,backup_server_id,parameter): # ############################################ - # Method + # Method # ############################################ def get_pgbackman_database_version(self): @@ -3033,18 +3033,18 @@ def get_pgbackman_database_version(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def run_sql_file(self,sqlfile): @@ -3057,19 +3057,19 @@ def run_sql_file(self,sqlfile): try: self.cur.execute(open(sqlfile,'r').read()) - return - + return + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def get_pgsql_nodes_list(self): @@ -3084,19 +3084,19 @@ def get_pgsql_nodes_list(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - + # ############################################ - # Method + # Method # ############################################ def get_deleted_backup_definitions_to_delete_by_retention(self): @@ -3115,19 +3115,19 @@ def get_deleted_backup_definitions_to_delete_by_retention(self): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e - - + + # ############################################ - # Method + # Method # ############################################ def get_all_backup_definitions(self,backup_server_id,pgsql_node_id): @@ -3147,17 +3147,17 @@ def get_all_backup_definitions(self,backup_server_id,pgsql_node_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_database_backup_definitions(self,backup_server_id,pgsql_node_id,dbname): @@ -3178,12 +3178,12 @@ def get_database_backup_definitions(self,backup_server_id,pgsql_node_id,dbname): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e @@ -3191,12 +3191,12 @@ def get_database_backup_definitions(self,backup_server_id,pgsql_node_id,dbname): # ############################################ - # Method + # Method # ############################################ - + def update_backup_definition_status_to_delete(self,def_id): """A function to update the status for a backup definition to DELETE""" - + try: self.pg_connect() @@ -3204,7 +3204,7 @@ def update_backup_definition_status_to_delete(self,def_id): try: self.cur.execute('SELECT update_backup_definition_status_to_delete(%s)',(def_id,)) self.conn.commit() - + except psycopg2.Error as e: raise e @@ -3215,7 +3215,7 @@ def update_backup_definition_status_to_delete(self,def_id): # ############################################ - # Method + # Method # ############################################ def get_pgsql_node_database_with_bckdef_list(self,pgsql_node_id): @@ -3234,17 +3234,17 @@ def get_pgsql_node_database_with_bckdef_list(self,pgsql_node_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_backup_server_bckdef_list(self,backup_server_id): @@ -3262,129 +3262,140 @@ def get_backup_server_bckdef_list(self,backup_server_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def get_pgbackman_dump_count(self): """A function to get the number of pgbackman_dump processes running in a PgSQL node""" - + try: if self.cur: - + self.cur.execute('SELECT count(*) FROM pg_stat_activity WHERE application_name = \'pgbackman_dump\'') self.conn.commit() data = self.cur.fetchone()[0] return data - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def pg_recovery_in_progress(self): """A function to find out if a PgSQL node is in recovery modus. This means that the PgSQL node is a slave/standby node in a replication system. """ - + try: if self.cur: - + self.cur.execute('SELECT pg_is_in_recovery()') self.conn.commit() data = self.cur.fetchone()[0] - + if data == True: return True elif data == False: return False else: return False - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def pg_recovery_paused(self): """A function to find out if a PgSQL node in recovery modus has the recovery process paused""" try: if self.cur: - - self.cur.execute('SELECT pg_is_xlog_replay_paused()') - self.conn.commit() + if self.conn.server_version >= 100000: + self.cur.execute('SELECT pg_is_wal_replay_paused()') + self.conn.commit() + + data = self.cur.fetchone()[0] + else: + self.cur.execute('SELECT pg_is_xlog_replay_paused()') + self.conn.commit() + + data = self.cur.fetchone()[0] - data = self.cur.fetchone()[0] - if data == True: return True elif data == False: return False else: return False - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def pause_pg_recovery(self): """A function to pause a postgres recovery process""" try: if self.cur: - - self.cur.execute('SELECT pg_xlog_replay_pause()') - self.conn.commit() - + if self.conn.server_version >= 100000: + self.cur.execute('SELECT pg_wal_replay_pause()') + self.conn.commit() + else: + self.cur.execute('SELECT pg_xlog_replay_pause()') + self.conn.commit() + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def resume_pg_recovery(self): """A function to resume a postgres recovery process""" try: if self.cur: - - self.cur.execute('SELECT pg_xlog_replay_resume()') - self.conn.commit() - + if self.server_version >= 100000: + self.cur.execute('SELECT pg_wal_replay_resume()') + self.conn.commit() + else: + self.cur.execute('SELECT pg_xlog_replay_resume()') + self.conn.commit() + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_status_info(self,parameter_status,backup_server,def_id): @@ -3429,7 +3440,7 @@ def get_status_info(self,parameter_status,backup_server,def_id): # # This value can be: succeeded, error, warning. # - + elif parameter_status in ['backup_last_status']: self.cur.execute('SELECT lower("Status") FROM show_backup_catalog WHERE def_id = %s ORDER BY "Finished" DESC LIMIT 1',(def_id,)) @@ -3444,20 +3455,20 @@ def get_status_info(self,parameter_status,backup_server,def_id): else: raise Exception("Invalid parameter status [%s]" % parameter_status) - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def delete_alert(self,alert_id): """A function to delete an alert""" @@ -3467,21 +3478,21 @@ def delete_alert(self,alert_id): if self.cur: try: self.cur.execute('SELECT delete_alert(%s)',(alert_id,)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ - + def update_alert_sent(self,alert_id,status): """A function to update an alert sent status""" @@ -3491,19 +3502,19 @@ def update_alert_sent(self,alert_id,status): if self.cur: try: self.cur.execute('SELECT update_alert_sent(%s,%s)',(alert_id,status)) - self.conn.commit() - + self.conn.commit() + except psycopg2.Error as e: raise e - + self.pg_close() - + except psycopg2.Error as e: raise e # ############################################ - # Method + # Method # ############################################ def get_alerts(self,backup_server_id): @@ -3521,11 +3532,11 @@ def get_alerts(self,backup_server_id): self.conn.commit() return self.cur - + except psycopg2.Error as e: raise e self.pg_close() - + except psycopg2.Error as e: raise e