From 0011cd4446f8e186470226e909930ca78cd1a125 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Mon, 15 May 2023 12:57:12 +0530 Subject: [PATCH 1/6] removed sudo --- VMBackup/main/Utils/SizeCalculation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index 9077811f3..e70a25741 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -68,7 +68,7 @@ def __init__(self,patching, hutil, logger,para_parser): self.logger.log("As the LunList is empty including all disks") def get_lsscsi_list(self): - command = "sudo lsscsi" + command = "lsscsi" try: self.lsscsi_list = (os.popen(command).read()).splitlines() except Exception as e: @@ -86,10 +86,10 @@ def get_lsblk_list(self): def get_pvs_list(self): try: - self.pvs_output = os.popen("sudo pvs").read().strip().split("\n") + self.pvs_output = os.popen("pvs").read().strip().split("\n") self.pvs_output = self.pvs_output[1:] except Exception as e: - error_msg = "Failed to execute the command sudo pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.pvs_output = [] @@ -112,7 +112,7 @@ def disk_list_for_billing(self): idxOfColon = item.rindex(':',0,item.index(']'))# to get the index of last ':' idxOfColon += 1 lunNumber = int(item[idxOfColon:item.index(']')]) - # item_split is the list of elements present in the one row of the cmd sudo lsscsi + # item_split is the list of elements present in the one row of the cmd lsscsi self.item_split = item.split() #storing the corresponding device name from the list device_name = self.item_split[len(self.item_split)-1] @@ -128,7 +128,7 @@ def disk_list_for_billing(self): self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) else: self.size_calc_failed = True - self.logger.log("There is some glitch in executing the command 'sudo lsscsi' and therefore size calculation is marked as failed.") + self.logger.log("There is some glitch in executing the command 'lsscsi' and therefore size calculation is marked as failed.") def get_logicalVolumes_for_billing(self): try: From d89fbfdb7dde935ee55808d1a958649714208726 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Mon, 15 May 2023 12:58:06 +0530 Subject: [PATCH 2/6] Revert "removed sudo" This reverts commit 0011cd4446f8e186470226e909930ca78cd1a125. --- VMBackup/main/Utils/SizeCalculation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index e70a25741..9077811f3 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -68,7 +68,7 @@ def __init__(self,patching, hutil, logger,para_parser): self.logger.log("As the LunList is empty including all disks") def get_lsscsi_list(self): - command = "lsscsi" + command = "sudo lsscsi" try: self.lsscsi_list = (os.popen(command).read()).splitlines() except Exception as e: @@ -86,10 +86,10 @@ def get_lsblk_list(self): def get_pvs_list(self): try: - self.pvs_output = os.popen("pvs").read().strip().split("\n") + self.pvs_output = os.popen("sudo pvs").read().strip().split("\n") self.pvs_output = self.pvs_output[1:] except Exception as e: - error_msg = "Failed to execute the command pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command sudo pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.pvs_output = [] @@ -112,7 +112,7 @@ def disk_list_for_billing(self): idxOfColon = item.rindex(':',0,item.index(']'))# to get the index of last ':' idxOfColon += 1 lunNumber = int(item[idxOfColon:item.index(']')]) - # item_split is the list of elements present in the one row of the cmd lsscsi + # item_split is the list of elements present in the one row of the cmd sudo lsscsi self.item_split = item.split() #storing the corresponding device name from the list device_name = self.item_split[len(self.item_split)-1] @@ -128,7 +128,7 @@ def disk_list_for_billing(self): self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) else: self.size_calc_failed = True - self.logger.log("There is some glitch in executing the command 'lsscsi' and therefore size calculation is marked as failed.") + self.logger.log("There is some glitch in executing the command 'sudo lsscsi' and therefore size calculation is marked as failed.") def get_logicalVolumes_for_billing(self): try: From d8f2d2ab0b98fa03fb79259d0ac1f8a9e52fd29c Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Thu, 1 Feb 2024 12:46:07 +0530 Subject: [PATCH 3/6] guest based changes --- VMBackup/main/guestsnapshotter.py | 32 ++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/VMBackup/main/guestsnapshotter.py b/VMBackup/main/guestsnapshotter.py index 50c8aab72..8b92adc7f 100644 --- a/VMBackup/main/guestsnapshotter.py +++ b/VMBackup/main/guestsnapshotter.py @@ -19,8 +19,10 @@ import os try: import urlparse as urlparser + from urlparse import urlunparse as unparser except ImportError: import urllib.parse as urlparser + from urllib.parse import urlunparse as unparser import traceback import datetime try: @@ -74,7 +76,7 @@ def snapshot(self, sasuri, sasuri_index, settings, meta_data, snapshot_result_er error_logger='' snapshot_error = SnapshotError() snapshot_info_indexer = SnapshotInfoIndexerObj(sasuri_index, False, None, None) - if(sasuri is None): + if(sasuri is None error_logger = error_logger + str(datetime.datetime.utcnow()) + " Failed to do the snapshot because sasuri is none " snapshot_error.errorcode = CommonVariables.error snapshot_error.sasuri = sasuri @@ -94,11 +96,17 @@ def snapshot(self, sasuri, sasuri_index, settings, meta_data, snapshot_result_er key = meta['Key'] value = meta['Value'] headers["x-ms-meta-" + key] = value - if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): - self.logger.log("Not passing the TTL header via Guest path though it is enabled") temp_logger = temp_logger + str(headers) http_util = HttpUtil(self.logger) sasuri_obj = urlparser.urlparse(sasuri + '&comp=snapshot') + if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): + self.logger.log("Not passing the TTL header via Guest path though it is enabled") + if(self.port != ""): + hostname = sasuri_obj.hostname + modified_netloc = "{}:{}".format(hostname, self.port) + modified_sasuri_obj = sasuri_obj._replace(netloc = modified_netloc) + sasuri_obj = urlparser.urlunparse(modified_sasuri_obj) + print(sasuri_obj) temp_logger = temp_logger + str(datetime.datetime.utcnow()) + ' start calling the snapshot rest api. ' # initiate http call for blob-snapshot and get http response result, httpResp, errMsg, responseBody = http_util.HttpCallGetResponse('PUT', sasuri_obj, body_content, headers = headers, responseBodyRequired = True) @@ -150,11 +158,17 @@ def snapshot_seq(self, sasuri, sasuri_index, settings, meta_data): key = meta['Key'] value = meta['Value'] headers["x-ms-meta-" + key] = value - if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): - self.logger.log("Not passing the TTL header via Guest path though it is enabled") self.logger.log(str(headers)) http_util = HttpUtil(self.logger) sasuri_obj = urlparser.urlparse(sasuri + '&comp=snapshot') + if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): + self.logger.log("Not passing the TTL header via Guest path though it is enabled") + if(self.port != ""): + hostname = sasuri_obj.hostname + modified_netloc = "{}:{}".format(hostname, self.port) + modified_sasuri_obj = sasuri_obj._replace(netloc = modified_netloc) + sasuri_obj = urlparser.urlunparse(modified_sasuri_obj) + print(sasuri_obj) self.logger.log("start calling the snapshot rest api") # initiate http call for blob-snapshot and get http response result, httpResp, errMsg, responseBody = http_util.HttpCallGetResponse('PUT', sasuri_obj, body_content, headers = headers, responseBodyRequired = True) @@ -333,6 +347,7 @@ def snapshotall_seq(self, paras, freezer, thaw_done, g_fsfreeze_on): def snapshotall(self, paras, freezer, g_fsfreeze_on): thaw_done = False + self.port = self.get_port(paras) if (self.hutil.get_intvalue_from_configfile('seqsnapshot',0) == 1 or self.hutil.get_intvalue_from_configfile('seqsnapshot',0) == 2 or (len(paras.blobs) <= 4)): snapshot_result, blob_snapshot_info_array, all_failed, exceptOccurred, is_inconsistent, thaw_done, unable_to_sleep, all_snapshots_failed = self.snapshotall_seq(paras, freezer, thaw_done, g_fsfreeze_on) else: @@ -384,3 +399,10 @@ def get_snapshot_info(self, snapshot_info_indexer, snapshot_info): else: snapshot_info.isSuccessful = False snapshot_info.snapshotUri = None + +def get_port(self, paras): + print(paras.createdBy) + if paras is not None and paras.createdBy == "crp": + return "8443" + else: + return "" \ No newline at end of file From b544bb3906be653c771c1a7e01b9cff8960b49fe Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Thu, 1 Feb 2024 12:50:05 +0530 Subject: [PATCH 4/6] corrected if --- VMBackup/main/guestsnapshotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VMBackup/main/guestsnapshotter.py b/VMBackup/main/guestsnapshotter.py index 8b92adc7f..52efda6a1 100644 --- a/VMBackup/main/guestsnapshotter.py +++ b/VMBackup/main/guestsnapshotter.py @@ -76,7 +76,7 @@ def snapshot(self, sasuri, sasuri_index, settings, meta_data, snapshot_result_er error_logger='' snapshot_error = SnapshotError() snapshot_info_indexer = SnapshotInfoIndexerObj(sasuri_index, False, None, None) - if(sasuri is None + if(sasuri is None): error_logger = error_logger + str(datetime.datetime.utcnow()) + " Failed to do the snapshot because sasuri is none " snapshot_error.errorcode = CommonVariables.error snapshot_error.sasuri = sasuri From ea5cb479d17a0b7310cf18efe6fa7de7b123d0ab Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Thu, 8 Feb 2024 16:08:41 +0530 Subject: [PATCH 5/6] modified the logic --- VMBackup/main/HttpUtil.py | 11 ++++++++++- VMBackup/main/guestsnapshotter.py | 30 ++++-------------------------- VMBackup/main/parameterparser.py | 22 +++++++++++++++++++++- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/VMBackup/main/HttpUtil.py b/VMBackup/main/HttpUtil.py index d27fc9c25..613a76078 100644 --- a/VMBackup/main/HttpUtil.py +++ b/VMBackup/main/HttpUtil.py @@ -27,6 +27,7 @@ import subprocess import sys from common import CommonVariables +from parameterparser import ParameterParser from subprocess import * from Utils.WAAgentUtil import waagent import Utils.HandlerUtil @@ -127,7 +128,15 @@ def HttpCallGetResponse(self, method, sasuri_obj, data, headers , responseBodyRe if(isHostCall): connection = httplibs.HTTPConnection(sasuri_obj.hostname, timeout = 40) # making call with port 80 to make it http call else: - connection = httplibs.HTTPSConnection(sasuri_obj.hostname, timeout = 10) + port = None + isTTlEnabled = False + if CommonVariables.isSnapshotTtlEnabled in ParameterParser.wellKnownSettingFlags and ParameterParser.wellKnownSettingFlags[CommonVariables.isSnapshotTtlEnabled]: + # TTL is enabled, routing to a 8443 port + port = 8443 + connection = httplibs.HTTPSConnection(sasuri_obj.hostname, port = port, timeout = 10) + else: + self.logger.log("Routing to a default port") + connection = httplibs.HTTPSConnection(sasuri_obj.hostname, timeout = 10) self.logger.log("Details of sas uri object hostname: " + str(sasuri_obj.hostname) + " path: " + str(sasuri_obj.path)) connection.request(method=method, url=(sasuri_obj.path + '?' + sasuri_obj.query), body=data, headers = headers) resp = connection.getresponse() diff --git a/VMBackup/main/guestsnapshotter.py b/VMBackup/main/guestsnapshotter.py index 52efda6a1..50c8aab72 100644 --- a/VMBackup/main/guestsnapshotter.py +++ b/VMBackup/main/guestsnapshotter.py @@ -19,10 +19,8 @@ import os try: import urlparse as urlparser - from urlparse import urlunparse as unparser except ImportError: import urllib.parse as urlparser - from urllib.parse import urlunparse as unparser import traceback import datetime try: @@ -96,17 +94,11 @@ def snapshot(self, sasuri, sasuri_index, settings, meta_data, snapshot_result_er key = meta['Key'] value = meta['Value'] headers["x-ms-meta-" + key] = value + if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): + self.logger.log("Not passing the TTL header via Guest path though it is enabled") temp_logger = temp_logger + str(headers) http_util = HttpUtil(self.logger) sasuri_obj = urlparser.urlparse(sasuri + '&comp=snapshot') - if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): - self.logger.log("Not passing the TTL header via Guest path though it is enabled") - if(self.port != ""): - hostname = sasuri_obj.hostname - modified_netloc = "{}:{}".format(hostname, self.port) - modified_sasuri_obj = sasuri_obj._replace(netloc = modified_netloc) - sasuri_obj = urlparser.urlunparse(modified_sasuri_obj) - print(sasuri_obj) temp_logger = temp_logger + str(datetime.datetime.utcnow()) + ' start calling the snapshot rest api. ' # initiate http call for blob-snapshot and get http response result, httpResp, errMsg, responseBody = http_util.HttpCallGetResponse('PUT', sasuri_obj, body_content, headers = headers, responseBodyRequired = True) @@ -158,17 +150,11 @@ def snapshot_seq(self, sasuri, sasuri_index, settings, meta_data): key = meta['Key'] value = meta['Value'] headers["x-ms-meta-" + key] = value + if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): + self.logger.log("Not passing the TTL header via Guest path though it is enabled") self.logger.log(str(headers)) http_util = HttpUtil(self.logger) sasuri_obj = urlparser.urlparse(sasuri + '&comp=snapshot') - if(CommonVariables.isSnapshotTtlEnabled in settings and settings[CommonVariables.isSnapshotTtlEnabled]): - self.logger.log("Not passing the TTL header via Guest path though it is enabled") - if(self.port != ""): - hostname = sasuri_obj.hostname - modified_netloc = "{}:{}".format(hostname, self.port) - modified_sasuri_obj = sasuri_obj._replace(netloc = modified_netloc) - sasuri_obj = urlparser.urlunparse(modified_sasuri_obj) - print(sasuri_obj) self.logger.log("start calling the snapshot rest api") # initiate http call for blob-snapshot and get http response result, httpResp, errMsg, responseBody = http_util.HttpCallGetResponse('PUT', sasuri_obj, body_content, headers = headers, responseBodyRequired = True) @@ -347,7 +333,6 @@ def snapshotall_seq(self, paras, freezer, thaw_done, g_fsfreeze_on): def snapshotall(self, paras, freezer, g_fsfreeze_on): thaw_done = False - self.port = self.get_port(paras) if (self.hutil.get_intvalue_from_configfile('seqsnapshot',0) == 1 or self.hutil.get_intvalue_from_configfile('seqsnapshot',0) == 2 or (len(paras.blobs) <= 4)): snapshot_result, blob_snapshot_info_array, all_failed, exceptOccurred, is_inconsistent, thaw_done, unable_to_sleep, all_snapshots_failed = self.snapshotall_seq(paras, freezer, thaw_done, g_fsfreeze_on) else: @@ -399,10 +384,3 @@ def get_snapshot_info(self, snapshot_info_indexer, snapshot_info): else: snapshot_info.isSuccessful = False snapshot_info.snapshotUri = None - -def get_port(self, paras): - print(paras.createdBy) - if paras is not None and paras.createdBy == "crp": - return "8443" - else: - return "" \ No newline at end of file diff --git a/VMBackup/main/parameterparser.py b/VMBackup/main/parameterparser.py index b4e0d001a..72afd829b 100644 --- a/VMBackup/main/parameterparser.py +++ b/VMBackup/main/parameterparser.py @@ -125,4 +125,24 @@ def __init__(self, protected_settings, public_settings, backup_logger): except Exception as e: errorMsg = "Exception occurred while populating settings, Exception: %s" % (str(e)) backup_logger.log(errorMsg, True) - backup_logger.log("settings to be sent " + str(self.wellKnownSettingFlags), True) \ No newline at end of file + backup_logger.log("settings to be sent " + str(self.wellKnownSettingFlags), True) + def get_port(self): + metadata = self.backup_metadata + createdByKey = "createdby" + createdByValue = "CRP" + if(metadata != None): + try: + self.logger.log("metadata received " + str(metadata), True) + for data in self.backup_metadata: + if CommonVariables.key in data and CommonVariables.value in data: + data_key = data[CommonVariables.key].lower() + if(data_key == createdByKey): + data_value = data[CommonVariables.value].upper() + if(data_value == createdByValue): + self.logger.log("it is a managed vm and request is being routed to Xstore's native path") + return "8443" + except Exception as e: + errorMsg = "Exception occurred while populating metadata, Exception: %s" % (str(e)) + self.logger.log(errorMsg, True) + else: + return "" \ No newline at end of file From 76f6baf91d601cf36c35385d4bf462f7d0c4f2ec Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 21 Feb 2024 10:42:35 +0530 Subject: [PATCH 6/6] configured using flag --- VMBackup/main/HttpUtil.py | 12 +++++------- VMBackup/main/parameterparser.py | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/VMBackup/main/HttpUtil.py b/VMBackup/main/HttpUtil.py index 613a76078..6117d1ba2 100644 --- a/VMBackup/main/HttpUtil.py +++ b/VMBackup/main/HttpUtil.py @@ -27,9 +27,9 @@ import subprocess import sys from common import CommonVariables -from parameterparser import ParameterParser from subprocess import * from Utils.WAAgentUtil import waagent +from parameterparser import ParameterParser import Utils.HandlerUtil import sys @@ -114,7 +114,7 @@ def Call(self, method, sasuri_obj, data, headers, fallback_to_curl = False): else: return CommonVariables.error_http_failure - def HttpCallGetResponse(self, method, sasuri_obj, data, headers , responseBodyRequired = False, isHostCall = False): + def HttpCallGetResponse(self, method, sasuri_obj, data, headers, responseBodyRequired = False, isHostCall = False): result = CommonVariables.error_http_failure resp = None responeBody = "" @@ -128,12 +128,10 @@ def HttpCallGetResponse(self, method, sasuri_obj, data, headers , responseBodyRe if(isHostCall): connection = httplibs.HTTPConnection(sasuri_obj.hostname, timeout = 40) # making call with port 80 to make it http call else: - port = None - isTTlEnabled = False - if CommonVariables.isSnapshotTtlEnabled in ParameterParser.wellKnownSettingFlags and ParameterParser.wellKnownSettingFlags[CommonVariables.isSnapshotTtlEnabled]: + port_info = ParameterParser.port() + if(port_info != None): # TTL is enabled, routing to a 8443 port - port = 8443 - connection = httplibs.HTTPSConnection(sasuri_obj.hostname, port = port, timeout = 10) + connection = httplibs.HTTPSConnection(sasuri_obj.hostname, port = port_info, timeout = 10) else: self.logger.log("Routing to a default port") connection = httplibs.HTTPSConnection(sasuri_obj.hostname, timeout = 10) diff --git a/VMBackup/main/parameterparser.py b/VMBackup/main/parameterparser.py index 72afd829b..b1562a767 100644 --- a/VMBackup/main/parameterparser.py +++ b/VMBackup/main/parameterparser.py @@ -16,12 +16,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pickle import NONE from common import CommonVariables import base64 import json import sys class ParameterParser(object): + port = None def __init__(self, protected_settings, public_settings, backup_logger): """ TODO: we should validate the parameter first @@ -126,23 +128,22 @@ def __init__(self, protected_settings, public_settings, backup_logger): errorMsg = "Exception occurred while populating settings, Exception: %s" % (str(e)) backup_logger.log(errorMsg, True) backup_logger.log("settings to be sent " + str(self.wellKnownSettingFlags), True) - def get_port(self): - metadata = self.backup_metadata + + createdByKey = "createdby" createdByValue = "CRP" - if(metadata != None): + if(self.backup_metadata != None): try: - self.logger.log("metadata received " + str(metadata), True) + backup_logger.log("metadata received " + str(self.backup_metadata), True) for data in self.backup_metadata: if CommonVariables.key in data and CommonVariables.value in data: data_key = data[CommonVariables.key].lower() if(data_key == createdByKey): data_value = data[CommonVariables.value].upper() if(data_value == createdByValue): - self.logger.log("it is a managed vm and request is being routed to Xstore's native path") - return "8443" + backup_logger.log("it is a managed vm and request is being routed to Xstore's native path") + ParameterParser.port = 8443 + break except Exception as e: errorMsg = "Exception occurred while populating metadata, Exception: %s" % (str(e)) - self.logger.log(errorMsg, True) - else: - return "" \ No newline at end of file + backup_logger.log(errorMsg, True) \ No newline at end of file