fixed bug in exception handling & string check

Author: ian-kerins

scrapeops_scrapy/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.3"
+__version__ = "0.3.4"

scrapeops_scrapy/stats/logger.py

@@ -176,10 +176,15 @@ def log_levels(self, crawler):
             self.set_value(self._overall_stats, log_key, log_value)
 
 
+    def exception_type_check(key):
+        if isinstance(key, str):
+            return key.startswith('downloader/exception_type_count/')
+        return False
+    
     def get_exception_stats(self, crawler):
         scrapy_stats = crawler.stats.get_stats()
         if scrapy_stats.get('downloader/exception_count') is not None:
-            exception_values = [ {k:v} for k,v in scrapy_stats.items() if k.startswith('downloader/exception_type_count/')]
+            exception_values = [ {k:v} for k,v in scrapy_stats.items() if self.exception_type_check(k)]
             for exception in exception_values:
                 for key, value in exception.items():
                     key_type = key.replace('downloader/exception_type_count/', '')
@@ -189,8 +194,6 @@ def get_exception_stats(self, crawler):
                         exception_type = key_type
                     self.set_value(self._overall_stats, f'responses|unknown|unknown|unknown|unknown|unknown|{exception_type}|unknown|unknown|unknown|unknown|count', value)
 
-
-
 
 
 

scrapeops_scrapy/validators/response_validator.py

@@ -36,12 +36,12 @@ def run_validation_test(response, test_array):
                 else: return True
 
             if test.get('test_type') == 'response_length_check':
-                if ResponseValidator.response_length_check(response, test.get('threshold', 0), test.get('comparison_type')):
+                if ResponseValidator.response_length_check(ResponseValidator.get_response_text(response), test.get('threshold', 0), test.get('comparison_type')):
                     fail_counter += 1
                 else: return True
 
             if test.get('test_type') == 'string_check' and test.get('test_location') == 'body':
-                if ResponseValidator.string_check(response, test.get('text_check', ''), test.get('comparison_type'), text_slice=test.get('text_slice')):
+                if ResponseValidator.string_check(ResponseValidator.get_response_text(response), test.get('text_check', ''), test.get('comparison_type'), text_slice=test.get('text_slice')):
                     fail_counter += 1
                 else: return True
 
@@ -89,9 +89,16 @@ def failed_scan(request_response_object, domains):
 
 
     @staticmethod
-    def string_check(response, text_check, comparison, text_slice=None):
-        if text_check == '': return False
-        text = response.text if isinstance(response, Response) else response
+    def get_response_text(response):
+        try:
+            if isinstance(response, Response): return response.text 
+            else: return ''
+        except AttributeError:
+            return ''
+
+
+    @staticmethod
+    def string_check(text, text_check, comparison, text_slice=None):
         if text_slice is not None:
             text = ResponseValidator.string_slice(text, text_slice)
         if comparison == 'contains' and text_check in text:
@@ -100,6 +107,7 @@ def string_check(response, text_check, comparison, text_slice=None):
             return True
         return False
 
+
     @staticmethod
     def string_slice(text, text_slice):
         if text_slice.get('active'):
@@ -155,9 +163,9 @@ def bytes_check(response, threshold, comparison):
 
 
     @staticmethod
-    def response_length_check(response, threshold, comparison):
+    def response_length_check(text, threshold, comparison):
         if threshold == 0: return False
-        response_text_length = len(response.text)
+        response_text_length = len(text)
         return ResponseValidator.comparison_operators(response_text_length, threshold, comparison)
 
 

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.3.3'
+VERSION = '0.3.4'
 DESCRIPTION = 'Scrapeops Scrapy SDK, is a monitoring tool for your Scrapy spiders.'
 
 setup(name='scrapeops_scrapy',