Merge pull request #152 from DevoInc/test-csv-separarators-quotes

Test csv separarators quotes

Author: JuanFranDevo

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [4.0.2] - 2022-09-01
+### Added
+ * Add a new command-line option for escaping double quotes with `-eq` or `--escape_quotes`.
+ * Add a check and shows a warning if the file contains double quotes and the option `-eq` is not used.
+ * Add a new command-line option in tests for testing just one module with `-m <module_name>` or `--module <module_name>`.
 ## [4.0.1] - 2022-08-26
 ### Added
  * Create GitHub action to publish package in PyPI

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (C) 2018 Devo, Inc.
+Copyright (C) 2022 Devo, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -105,20 +105,50 @@ To launch this script, you need either the environment variables loaded in the s
 
 Its normal, by the way, TCP tests fails in clients or not Devo developers systems.
 
-```bash
+```console
 ~/projects/devo-python-sdk > python setup.py test 
 ```
 
-```bash
+```console
 ~/projects/devo-python-sdk > python run_tests.py
 ```
 
 You can add option "Coverage" for create HTML report about tests.
 
-```bash
+```console
 ~/projects/devo-python-sdk > python run_tests.py --coverage
 ```
 
+You can also run the test for just one module. This is a useful feature if you are developing functionality in just one module.
+
+```console
+~/projects/devo-python-sdk > python run_tests.py -m SEND_CLI
+```
+
+Using the --help flag prints the available modules to use:
+
+```console
+~/projects/devo-python-sdk > python run_tests.py --help
+usage: run_tests.py [-h] [--coverage [COVERAGE]] [-m [MODULE]]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --coverage [COVERAGE]
+                        Generate coverage
+  -m [MODULE], --module [MODULE]
+                        Run tests for selected module: API_CLI, API_QUERY, API_TASKS, COMMON_CONFIGURATION,
+                        COMMON_DATE_PARSER, SENDER_CLI, SENDER_NUMBER_LOOKUP, SENDER_SEND_DATA, SENDER_SEND_LOOKUP
+```
+
+* API_CLI: API Command-line interface tests.
+* API_QUERY: Query API tests.
+* API_TASKS: Task API tests.
+* COMMON_CONFIGURATION: Configuration tests.
+* COMMON_DATE_PARSER: Date parser tests.
+* SENDER_CLI: Lookup command-line interface tests.
+* SENDER_NUMBER_LOOKUP: Numbers in lookup tests
+* SENDER_SEND_DATA: Data sending tests.
+* SENDER_SEND_LOOKUP: Lookup sending tests.
 
 ### Run using Unittest command
 

devo/__version__.py

@@ -1,6 +1,6 @@
 __description__ = 'Devo Python Library.'
 __url__ = 'http://www.devo.com'
-__version__ = "4.0.1"
+__version__ = "4.0.2"
 __author__ = 'Devo'
 __author_email__ = 'support@devo.com'
 __license__ = 'MIT'

devo/common/__init__.py

@@ -1,10 +1,14 @@
-from .dates.dateparser import parse, parse_string, parse_expression, \
-    default_from, default_to
-from .dates.dateoperations import month, week, day, hour, minute, second, now, \
-    now_without_ms, today, yesterday, parse_functions
-from .dates.dateutils import to_millis, trunc_time, trunc_time_minute, \
-    test_date_format, get_timestamp
+from .dates.dateparser import (
+    parse, parse_string, parse_expression,
+    default_from, default_to)
+from .dates.dateoperations import (
+    month, week, day, hour, minute, second, now,
+    now_without_ms, today, yesterday, parse_functions)
+from .dates.dateutils import (
+    to_millis, trunc_time, trunc_time_minute,
+    test_date_format, get_timestamp)
 from .generic.configuration import Configuration
-from .logging.log import get_log, set_formatter, get_rotating_file_handler, \
-    get_stream_handler
+from .logging.log import (
+    get_log, set_formatter, get_rotating_file_handler,
+    get_stream_handler)
 from .loadenv import load_env_file

devo/common/logging/log.py

@@ -38,12 +38,13 @@ def set_formatter(msg_format):
     return logging.Formatter(msg_format)
 
 
-def get_rotating_file_handler(path="./",
-                              file_name="history.log",
-                              msg_format='%(asctime)s|%(levelname)s|%(message)s',
-                              max_size=2097152,
-                              backup_count=5,
-                              level=logging.INFO):
+def get_rotating_file_handler(
+        path="./",
+        file_name="history.log",
+        msg_format='%(asctime)s|%(levelname)s|%(message)s',
+        max_size=2097152,
+        backup_count=5,
+        level=logging.INFO):
     """Initialize rotating file handler for logger
 
     :return: RotatingFileHandler object

devo/sender/data.py

@@ -311,8 +311,8 @@ def __init__(self, config=None, con_type=None,
         logging.Handler.__init__(self)
         self.logger = logger if logger else \
             get_log(handler=get_stream_handler(
-                msg_format='%(asctime)s|%(levelname)s|Devo-Sender|%(message)s'))
-
+                msg_format='%(asctime)s|%(levelname)s|Devo-Sender|%(message)s')
+                )
 
         self._sender_config = config
 
@@ -410,8 +410,8 @@ def __connect_ssl(self):
             self.reconnection += 1
             if self.debug:
                 self.logger.debug('Conected to %s|%s'
-                                  % (repr(self.socket.getpeername())
-                                     , str(self.reconnection)))
+                                  % (repr(self.socket.getpeername()),
+                                     str(self.reconnection)))
             self.timestart = int(round(time.time() * 1000))
 
         except socket.error as error:
@@ -520,7 +520,7 @@ def close(self):
         if self.socket is not None:
             try:
                 self.socket.shutdown(SHUT_RDWR)
-            except:  # Try else continue
+            except Exception as e:  # Try else continue
                 pass
             self.socket.close()
             self.socket = None
@@ -671,7 +671,8 @@ def send_bytes(self, tag, msg, **kwargs):
         """
         Send function when bytes, sure py3x. Can be zipped
         """
-        msg = COMPOSE_BYTES % (self.compose_mem(tag, bytes=True, **kwargs), msg)
+        msg = COMPOSE_BYTES % (
+            self.compose_mem(tag, bytes=True, **kwargs), msg)
         if kwargs.get('zip', False):
             return self.fill_buffer(msg)
 

devo/sender/lookup.py

@@ -6,7 +6,6 @@
 import time
 
 
-
 def find_key_index(value=None, headers=None):
     """Find index of key value in a list"""
     for index, val in enumerate(headers):
@@ -143,6 +142,20 @@ def detect_types(reader=None):
                 types[index] = "str"
         return types
 
+    def check_quotes(self, file_path) -> bool:
+        """
+        Check if there are quotes in CSV file
+        :param file_path: path to CSV file
+        :return: boolean
+        """
+        if self.escape_quotes:
+            return False
+
+        with open(file_path) as f:
+            line = next((l_temp for l_temp in f if '"' in l_temp), None)
+
+        return True if line is not None else False
+
     # Send a whole CSV file
     def send_csv(self, path=None, has_header=True, delimiter=',',
                  quotechar='"', headers=None, key="KEY", historic_tag=None,
@@ -241,7 +254,8 @@ def send_csv(self, path=None, has_header=True, delimiter=',',
         except IOError as error:
             print("I/O error({0}): {1}".format(error.errno, error.strerror))
         except Exception as error:
-            raise Exception("Unexpected error: %e \n" % error, sys.exc_info()[0])
+            raise Exception(
+                "Unexpected error: %e \n" % error, sys.exc_info()[0])
 
     # Basic process
     # --------------------------------------------------------------------------

devo/sender/scripts/sender_cli.py

@@ -50,9 +50,10 @@ def cli(version):
                                     '"2" (CERT_REQUIRED)', type=int)
 @click.option('--check_hostname', help='Verify cert hostname. Default: True',
               type=bool)
-@click.option('--multiline/--no-multiline', help='Flag for multiline (With '
-                                                 'break-line in msg). '
-                                                 'Default False', default=False)
+@click.option('--multiline/--no-multiline',
+              help='Flag for multiline (With '
+              'break-line in msg). '
+              'Default False', default=False)
 @click.option('--type', help='Connection type: SSL or TCP', default="SSL")
 @click.option('--tag', '-t', help='Tag / Table to which the data will be sent '
                                   'in Devo.', default="test.drop.ltsender")
@@ -168,18 +169,31 @@ def data(**kwargs):
               default=False)
 @click.option('--delimiter', '-d', help='CSV Delimiter char.', default=",")
 @click.option('--quotechar', '-qc', help='CSV Quote char.', default='"')
+@click.option('--escapequotes', '-eq', is_flag=True,
+              help='Escape Quotes. Default: False',
+              default=False)
 @click.option('--debug/--no-debug', help='For testing purposes', default=False)
 def lookup(**kwargs):
     """Send csv lookups to devo"""
     config = configure_lookup(kwargs)
+    warning_st = 0
+
+    # Exit errors by https://tldp.org/LDP/abs/html/exitcodes.html
+    status_msg = {
+        64: "Some field contains double quotes in this file."
+            "If you do not use -eq or --escapequotes it might not work."
+    }
+
     con = Sender(config=config)
 
-    lookup = Lookup(name=config['name'], historic_tag=None, con=con)
+    lookup = Lookup(name=config['name'], historic_tag=None,
+                    con=con, escape_quotes=config['escapequotes'])
 
-    # with open(config['file']) as file:
-    #     line = file.readline()
+    if lookup.check_quotes(config['file']):
+        warning_st = 64
 
-    lookup.send_csv(config['file'], delimiter=config['delimiter'],
+    lookup.send_csv(config['file'],
+                    delimiter=config['delimiter'],
                     quotechar=config['quotechar'],
                     has_header=True,
                     # headers=line.rstrip().split(config['delimiter']),
@@ -191,6 +205,10 @@ def lookup(**kwargs):
                                      ),
                     detect_types=config.get("detect_types", False))
 
+    if warning_st != 0:
+        print_warning(status_msg[warning_st])
+        exit(warning_st)
+
 
 def configure(args):
     """ Configuration of Sender CLI """
@@ -237,3 +255,11 @@ def print_error(error, show_help=False):
         click.echo("")
         click.echo(click.get_current_context().get_help())
     click.echo(click.style(error, fg='red'), err=True)
+
+
+def print_warning(warning, show_help=False):
+    """Class for print warning in shell"""
+    if show_help:
+        click.echo("")
+        click.echo(click.get_current_context().get_help())
+    click.echo(click.style(warning, fg='yellow'), err=True)

devo/sender/transformsyslog.py

@@ -80,4 +80,4 @@
     "NOTICE": SEVERITY_NOTICE,
     "INFO": SEVERITY_INFO,
     "DEBUG": SEVERITY_DEBUG
-}
+}