Merge branch 'REL_2_6' into REL_2_6-PBCKP-314

Author: funny-falcon

.travis.yml

@@ -33,6 +33,8 @@ env:
     - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE
     - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE
     - PG_VERSION=10 PG_BRANCH=REL_10_STABLE
+    - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup.BackupTest.test_full_backup
+    - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup.BackupTest.test_full_backup_stream
 #    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=archive
 #    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup
 #    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup

Makefile

@@ -76,6 +76,19 @@ include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
 
+ifndef S3_DIR
+  ifneq ("$(wildcard $(abspath $(top_pbk_srcdir))/../s3)", "")
+    S3_DIR = $(abspath $(CURDIR))/../s3
+  endif
+endif
+
+ifdef S3_DIR
+  LDFLAGS += -lcurl
+  CFLAGS += $(shell pkg-config --cflags libxml-2.0) -DPBCKP_S3=1
+  LDFLAGS += $(shell pkg-config --libs libxml-2.0)
+  OBJS += $(S3_DIR)/s3.o
+endif
+
 #
 PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -I$(top_pbk_srcdir)src -I$(BORROW_DIR)
 PG_CPPFLAGS += -I$(top_pbk_srcdir)src/fu_util -Wno-declaration-after-statement
@@ -87,6 +100,9 @@ PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS}
 
 # additional dependencies on borrowed files
 src/backup.o src/catchup.o src/pg_probackup.o: $(BORROW_DIR)/streamutil.h
+ifdef S3_DIR
+  src/backup.o src/catchup.o src/pg_probackup.o: $(S3_DIR)/s3.o
+endif
 src/stream.o $(BORROW_DIR)/receivelog.o $(BORROW_DIR)/streamutil.o $(BORROW_DIR)/walmethods.o: $(BORROW_DIR)/receivelog.h
 $(BORROW_DIR)/receivelog.h: $(BORROW_DIR)/walmethods.h
 

src/backup.c

@@ -46,7 +46,6 @@ static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 
 static void pg_switch_wal(PGconn *conn);
 
-static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo);
 
 static void check_external_for_tablespaces(parray *external_list,
 										   PGconn *backup_conn);
@@ -1851,7 +1850,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c
 /*
  * Notify end of backup to PostgreSQL server.
  */
-static void
+void
 pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn,
 				PGNodeInfo *nodeInfo)
 {

src/dir.c

@@ -1762,13 +1762,16 @@ is_forkname(char *name, size_t *pos, const char *forkname)
 }
 
 #define OIDCHARS 10
+#define MAXSEGNO (((uint64_t)1<<32)/RELSEG_SIZE-1)
+#define SEGNOCHARS 5 /* when BLCKSZ == (1<<15) */
 
 /* Set forkName if possible */
 bool
 set_forkname(pgFile *file)
 {
 	size_t i = 0;
 	uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */
+	uint64_t segno = 0;
 
 	/* pretend it is not relation file */
 	file->relOid = 0;
@@ -1799,8 +1802,15 @@ set_forkname(pgFile *file)
 	/* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */
 	if (file->name[i] == '.' && isdigit(file->name[i+1]))
 	{
+		size_t start = i+1;
 		for (i++; isdigit(file->name[i]); i++)
-			;
+		{
+			if (i == start && file->name[i] == '0')
+				return false;
+			segno = segno * 10 + file->name[i] - '0';
+		}
+		if (i - start > SEGNOCHARS || segno > MAXSEGNO)
+			return false;
 	}
 
 	/* CFS "fork name" */
@@ -1819,6 +1829,7 @@ set_forkname(pgFile *file)
 	}
 
 	file->relOid = oid;
+	file->segno = segno;
 	file->is_datafile = file->forkName == none;
 	return true;
 }

src/pg_probackup.c

@@ -164,6 +164,12 @@ int64 ttl = -1;
 static char *expire_time_string = NULL;
 static pgSetBackupParams *set_backup_params = NULL;
 
+#ifdef PBCKP_S3
+/* S3 options */
+S3_protocol s3_protocol;
+char* s3_target_bucket = NULL;
+#endif
+
 /* ================ backupState =========== */
 static char *backup_id_string = NULL;
 pgBackup	current;
@@ -174,6 +180,9 @@ static bool help_opt = false;
 static void opt_incr_restore_mode(ConfigOption *opt, const char *arg);
 static void opt_backup_mode(ConfigOption *opt, const char *arg);
 static void opt_show_format(ConfigOption *opt, const char *arg);
+#ifdef PBCKP_S3
+static void opt_s3_protocol(ConfigOption *opt, const char *arg);
+#endif
 
 static void compress_init(ProbackupSubcmd const subcmd);
 
@@ -270,6 +279,12 @@ static ConfigOption cmd_options[] =
 	{ 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value},
 	{ 's', 171, "expire-time",		&expire_time_string,	SOURCE_CMD_STRICT },
 
+#ifdef PBCKP_S3
+	/* S3 options */
+	{ 'f', 245, "s3",				opt_s3_protocol,	SOURCE_CMD_STRICT },
+	{ 's', 246, "target-bucket", 	&s3_target_bucket,	SOURCE_CMD_STRICT },
+#endif
+
 	/* options for backward compatibility
 	 * TODO: remove in 3.0.0
 	 */
@@ -953,6 +968,19 @@ main(int argc, char *argv[])
 
 	compress_init(backup_subcmd);
 
+#ifdef PBCKP_S3
+	if (s3_protocol != S3_INVALID_PROTOCOL)
+	{
+		char* s3_config_file="";
+		read_s3_config(s3_config_file);
+	}
+	else
+	{
+		if (s3_target_bucket != NULL)
+			elog(WARNING, "You cannot specify s3-target without using --s3 option with name of protocol");
+	}
+#endif
+
 	/* do actual operation */
 	switch (backup_subcmd)
 	{
@@ -965,11 +993,27 @@ main(int argc, char *argv[])
 						   wal_file_path, wal_file_name, batch_size, !no_validate_wal);
 			break;
 		case ADD_INSTANCE_CMD:
-			return do_add_instance(instanceState, &instance_config);
+		{
+			int err = 0;
+			err = do_add_instance(instanceState, &instance_config);
+#ifdef PBCKP_S3
+			if (err == 0 && s3_protocol != S3_INVALID_PROTOCOL)
+				err = do_S3_write_config(instanceState);
+#endif
+			return err;
+		}
 		case DELETE_INSTANCE_CMD:
 			return do_delete_instance(instanceState);
 		case INIT_CMD:
-			return do_init(catalogState);
+		{
+			int err = 0;
+			err = do_init(catalogState);
+#ifdef PBCKP_S3
+			if (err == 0 && s3_protocol != S3_INVALID_PROTOCOL)
+				err = S3_pre_start_check(config);
+#endif
+			return err;
+		}
 		case BACKUP_CMD:
 			{
 				current.stream = stream_wal;
@@ -984,13 +1028,21 @@ main(int argc, char *argv[])
 					elog(ERROR, "required parameter not specified: BACKUP_MODE "
 						 "(-b, --backup-mode)");
 
+#ifdef PBCKP_S3
+				if (s3_protocol != S3_INVALID_PROTOCOL)
+					return do_S3_backup(instanceState, set_backup_params, start_time);
+#endif
 				return do_backup(instanceState, set_backup_params,
 								 no_validate, no_sync, backup_logs, start_time);
 			}
 		case CATCHUP_CMD:
 			return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync,
 				exclude_absolute_paths_list, exclude_relative_paths_list);
 		case RESTORE_CMD:
+#ifdef PBCKP_S3
+			if (s3_protocol != S3_INVALID_PROTOCOL)
+				return do_S3_restore(instanceState, current.backup_id);
+#endif
 			return do_restore_or_validate(instanceState, current.backup_id,
 							recovery_target_options,
 							restore_params, no_sync);
@@ -1010,6 +1062,10 @@ main(int argc, char *argv[])
 						  restore_params,
 						  no_sync);
 		case SHOW_CMD:
+#ifdef PBCKP_S3
+			if (s3_protocol != S3_INVALID_PROTOCOL)
+				return do_S3_show(instanceState);
+#endif
 			return do_show(catalogState, instanceState, current.backup_id, show_archive);
 		case DELETE_CMD:
 
@@ -1198,3 +1254,34 @@ opt_exclude_path(ConfigOption *opt, const char *arg)
 	else
 		opt_parser_add_to_parray_helper(&exclude_relative_paths_list, arg);
 }
+
+#ifdef PBCKP_S3
+static S3_protocol
+parse_s3_protocol(const char *value)
+{
+	const char *v = value;
+	size_t		len;
+
+	/* Skip all spaces detected */
+	while (IsSpace(*v))
+		v++;
+	len = strlen(v);
+
+	if (len > 0 && pg_strncasecmp("MINIO", v, len) == 0)
+		return S3_MINIO_PROTOCOL;
+	if (len > 0 && pg_strncasecmp("AWS", v, len) == 0)
+		return S3_AWS_PROTOCOL;
+	else if (len > 0 && pg_strncasecmp("GOOGLE", v, len) == 0)
+		return S3_GOOGLE_PROTOCOL;
+	else if (len > 0 && pg_strncasecmp("VK", v, len) == 0)
+		return S3_VK_PROTOCOL;
+	else
+		return S3_INVALID_PROTOCOL;
+}
+
+static void
+opt_s3_protocol(ConfigOption *opt, const char *arg)
+{
+	s3_protocol = parse_s3_protocol(arg);
+}
+#endif

src/pg_probackup.h

@@ -43,6 +43,10 @@
 
 #include "pg_probackup_state.h"
 
+#ifdef PBCKP_S3
+#include "../s3/s3.h"
+#endif
+
 #if defined(WIN32) && !(defined(_UCRT) && defined(__MINGW64__))
 #error Windows port requires compilation in MinGW64 UCRT environment
 #endif
@@ -803,6 +807,11 @@ extern pgBackup current;
 /* argv of the process */
 extern char** commands_args;
 
+#ifdef PBCKP_S3
+/* S3 options */
+extern S3_protocol s3_protocol;
+#endif
+
 /* in backup.c */
 extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params,
 					 bool no_validate, bool no_sync, bool backup_logs, time_t start_time);
@@ -813,6 +822,15 @@ extern const char *deparse_backup_mode(BackupMode mode);
 extern void process_block_change(ForkNumber forknum, RelFileNode rnode,
 								 BlockNumber blkno);
 
+#ifdef PBCKP_S3
+/* in s3.c */
+extern int do_S3_backup(InstanceState *instanceState,
+								 pgSetBackupParams *set_backup_params,time_t start_time);
+extern int do_S3_show(InstanceState *instanceState);
+extern int do_S3_restore(InstanceState *instanceState, time_t target_backup_id);
+extern int do_S3_write_config(InstanceState *instanceState);
+#endif
+
 /* in catchup.c */
 extern int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files,
 	parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list);
@@ -1252,6 +1270,7 @@ extern parray *backup_files_list;
 
 extern void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
 							PGNodeInfo *nodeInfo, PGconn *conn);
+extern void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo);
 extern void pg_silent_client_messages(PGconn *conn);
 extern void pg_create_restore_point(PGconn *conn, time_t backup_start_time);
 extern void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, char **query_text);