added log messages to various fme modules

Author: franTarkenton

FMEUtil/FMEUtil/FMEScheduleLib.py

@@ -3,15 +3,17 @@
 
 @author: kjnether
 '''
-import logging
-import FMEUtil.PyFMEServerV2
+import copy
 import datetime
-import os.path
 import json
-import deepdiff
-import copy
+import logging
+import os.path
 import pprint
 
+import FMEUtil.PyFMEServerV2
+import deepdiff
+
+
 class Schedules(object):
 
     '''
@@ -37,7 +39,8 @@ class Schedules(object):
     :type flds2Ignore:  list
     '''
 
-    def __init__(self, fmeServUrl, fmeServToken, cacheLabel, cacheDir, refreshCache=False, ignorePP=False):
+    def __init__(self, fmeServUrl, fmeServToken, cacheLabel, cacheDir,
+                 refreshCache=False, ignorePP=False):
         '''
         :param fmeServUrl: url to fme server, don't include paths
         :param fmeServToken: token to fme server
@@ -68,7 +71,7 @@ def __init__(self, fmeServUrl, fmeServToken, cacheLabel, cacheDir, refreshCache=
         # when doing comparisons between the two structures
         self.flds2Ignore = ['begin', 'enabled']
         # this switch can be set using the method setIgnoredFields()
-        # when set to true published parameters are not considered in the 
+        # when set to true published parameters are not considered in the
         # comparison of the schedules.
         self.ignorePP = ignorePP
         # this data struct will get populated with everything in self.schedStruct
@@ -90,6 +93,13 @@ def getPyFME(self):
         '''
         return self.fme
 
+    def getScheduleData(self):
+        '''
+        :return: the data structure that describes the schedules in this
+                 object.
+        '''
+        return self.schedStruct
+
     def setIgnoredFields(self, flds, ignorePublishedParameters=True):
         '''
         When doing a comparison between schedule objects you can set a
@@ -102,7 +112,7 @@ def setIgnoredFields(self, flds, ignorePublishedParameters=True):
         self.ignorePP = ignorePublishedParameters
         self.flds2Ignore = flds
         schedStructCleaned = []
-        #schedIterator = self.schedStruct[0:]
+        # schedIterator = self.schedStruct[0:]
         schedIterator = copy.deepcopy(self.schedStruct)
         for schedRef in schedIterator:
             sched = schedRef.copy()
@@ -112,12 +122,16 @@ def setIgnoredFields(self, flds, ignorePublishedParameters=True):
                     del sched[fld2Del]
             if self.ignorePP:
                 # getting rid of published parameters too!
-                #self.pp.pprint(sched)
+                # self.pp.pprint(sched)
                 del sched['request']['publishedParameters']
             schedStructCleaned.append(sched)
         self.schedStructComparison = schedStructCleaned
 
     def isEnabled(self, scheduleName):
+        '''
+        :return: indicates if the schedule is enabled or not
+        :rtype: boolean
+        '''
         schedStruct = self.getScheduleByName(scheduleName)
         return schedStruct['enabled']
 
@@ -140,6 +154,7 @@ def getSchedules(self):
             self.logger.info("retrieving the schedules from fme server, this" + \
                              "may take a while")
             schedStruct = self.scheds.getSchedules()
+            self.logger.debug("schedStruct: {0}".format(schedStruct))
             with open(self.cacheFile, 'w') as outfile:
                 msg = "dumping the schedules to the cache file {0}"
                 self.logger.info(msg.format(cacheFile))
@@ -164,7 +179,7 @@ def getScheduleByName(self, scheduleName):
                 break
         return retVal
 
-    def __contains__(self, sched):
+    def __contains__(self, sched):  # pylint: disable=invalid-name
         '''
         :param sched: returns true or false based on whether the sched
                       object exists in this collection of schedules
@@ -177,7 +192,7 @@ def __contains__(self, sched):
         for fld in sched.keys():
             if fld not in self.flds2Ignore:
                 schedCleaned[fld] = copy.deepcopy(sched[fld])
-        # if the ignore published parameters flag is set then 
+        # if the ignore published parameters flag is set then
         # don't look at them if they are defined.
         if self.ignorePP:
             if 'request' in schedCleaned:
@@ -190,10 +205,10 @@ def __contains__(self, sched):
             for curSched in self.schedStructComparison:
                 if curSched['name'] == schedCleaned['name']:
                     diffs = deepdiff.DeepDiff(schedCleaned, curSched)
-                    self.logger.info("differences for {1}: {0}".format(diffs,                                                                        curSched['name']))
+                    self.logger.info("differences for {1}: {0}".format(diffs, curSched['name']))
         return retVal
 
-    def __sub__(self, schedules):
+    def __sub__(self, schedules):  # pylint: disable=invalid-name
         '''
         identifies schedules that are in self, but not in supplied
         schedules
@@ -223,6 +238,11 @@ def __init__(self, schedule, scheduleName):
         self.scheduleStruct = self.schedule.getScheduleByName(scheduleName)
 
     def getPublishedParameters(self):
+        '''
+        :return:  the published parameters associated with the specified schedule.
+        These are retrieved not from the schedule but from the FMW that the
+        schedule calls.
+        '''
         workspcName = self.scheduleStruct['workspace']
         repoName = self.scheduleStruct['repository']
 
@@ -232,10 +252,10 @@ def getPublishedParameters(self):
         pubParams = wrkspcs.getPublishedParams4Schedule(workspcName)
         return pubParams
 
-    def fixSchedulePublishedParameters(self, setEnabled=True):
+    def fixSchedulePublishedParameters(self):
         '''
-        Returns a schedule json struct that can be sent to FME Server to define
-        a new schedule.
+        :return: a schedule json struct that can be sent to FME Server to define
+                 a new schedule.
         '''
         # published parameters retrieved from the workspace on fme server
         pubParams = self.getPublishedParameters()
@@ -246,7 +266,7 @@ def fixSchedulePublishedParameters(self, setEnabled=True):
         pp.pprint(self.scheduleStruct)
         schedulePubParams = self.scheduleStruct['request']['publishedParameters']
 
-        fixedSchedule = self.scheduleStruct['request']['publishedParameters'][0:]
+        # fixedSchedule = self.scheduleStruct['request']['publishedParameters'][0:]
 
         # Iterating through the published parameters associated with the
         # workspace and overriding values with values that were retrieved
@@ -270,22 +290,3 @@ def fixSchedulePublishedParameters(self, setEnabled=True):
         # raise
         self.scheduleStruct['request']['publishedParameters'] = params4Schedule
         return self.scheduleStruct
-
-    def setDisabled(self, schedule):
-        '''
-        :param schedule: the schedule json object
-        :return: the same schedule as was recieved but with the enabled
-                 parameter set to false.
-        '''
-
-        pass
-
-    def setDestDbEnvKey(self, schedule, key):
-        '''
-        searches the schedule for the DEST_DB_ENV_KEY if one is found
-        then the parameter is set to the value defined in 'key'
-
-        :param key: the value to assign to DEST_DB_ENV_KEY if it exists.
-        '''
-        pass
-

FMEUtil/FMEUtil/PyFMEServerV2.py

@@ -34,6 +34,8 @@
 
 import requests
 
+# pylint: disable=logging-format-interpolation
+
 
 class FMERestBase(object):
     '''
@@ -314,6 +316,7 @@ class Logs(object):  # pylint: disable=too-few-public-methods
     '''
 
     def __init__(self, baseObj):
+        self.logger = logging.getLogger(__name__)
         self.baseObj = baseObj
         # example of v1 url to a log
         # V2 logs are moved under the jobs.
@@ -323,7 +326,7 @@ def __init__(self, baseObj):
         self.url = self.baseObj.fixUrlPath(self.url)
         self.url = urlparse.urljoin(self.url, 'jobs', True)
         self.url = self.baseObj.fixUrlPath(self.url)
-
+        self.logger.debug("url is: {0}".format(self.url))
         # self.url = urlparse.urljoin(self.url, 'complete', True)
         # self.url = self.baseObj.fixUrlPath(self.url)
 
@@ -332,6 +335,7 @@ def getLog(self, logId):
         Returns a Log object for a given log id. Job id?
         '''
         log = Log(self, logId)
+        self.logger.debug("getting the log: {0}".format(logId))
         return log
 
 
@@ -341,8 +345,10 @@ class Schedules(object):
     '''
 
     def __init__(self, baseObj):
+        self.logger = logging.getLogger(__name__)
         self.baseObj = baseObj
         self.url = urlparse.urljoin(self.baseObj.restUrl, self.baseObj.scheduleDir, True)
+        self.logger.debug("base url: {0}".format(self.url))
 
     def getSchedule(self):
         '''
@@ -367,6 +373,7 @@ def exists(self, schedName, category=None):
         '''
         schedsList = self.getSchedules(detail='low')
         retVal = False
+        self.logger.debug("schedule list: {0}".format(schedsList))
         # print 'schedList:', schedsList
         for sched in schedsList:
             # self.logger.debug("current sched in iteration %s", sched)
@@ -1225,6 +1232,8 @@ def deleteWorkspace(self, wrkspcName):
         '''
         url = self.baseObj.fixUrlPath(self.url)
         url = urlparse.urljoin(url, wrkspcName)
+        self.logger.debug("url: {0}".format(url))
+        self.logger.info("attempting to delete: {0}".format(wrkspcName))
         # print 'url', url
         # print 'wrkspcName', wrkspcName
         header = {'Accept': 'application/json'}
@@ -1396,7 +1405,7 @@ def getResourceInfo(self, dirType, dirList):
         itemUrl = self.__calcURL(dirType, dirList)
         # print 'url:', itemUrl
         response = self.baseObj.getResponse(itemUrl)
-        pp = pprint.PrettyPrinter(indent=4)
+        # pp = pprint.PrettyPrinter(indent=4)
         # pp.pprint(response)
         return response