Par 16742 verify config (#171)

* feat:  Added `verify-certificates` parameter support in event sending and lookups management clients
* fix: Some documentation fixes

Author: JuanFranDevo

CHANGELOG.md

@@ -4,11 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [5.0.6] - Unreleased
+## [5.0.6] - 2023-02-21
 ### Chorus
  * Update version range support for `requests`
 ### Fixed
  * Documentation fixes in `README.md` and `docs/common.md`
+ * Documentation related to testing updated at `README.md`
+### Added
+ * Added `verify-certificates` parameter support in event sending and lookups management clients
 
 ## [5.0.5] - 2023-02-20
 ### Fix
@@ -20,7 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [5.0.4] - 2023-02-13
 ### Chorus
  * Fix documentation (implementation remains the same) for keep-alive mechanism in queries for `xls` format
- * Bump version to avoid collision with tag belonging to fomer release candidate version
+ * Bump version to avoid collision with tag belonging to former release candidate version
 
 ## [5.0.3] - 2023-01-26
 ### Fixed

README.md

@@ -125,6 +125,12 @@ You can also run the test for just one module. This is a useful feature if you a
 ~/projects/devo-python-sdk > python run_tests.py -m SEND_CLI
 ```
 
+You can also exclude one or several tests with `-M` parameter:
+
+```console
+~/projects/devo-python-sdk > python run_tests.py -M SEND_CLI,API_CLI
+```
+
 Using the --help flag prints the available modules to use:
 
 ```console
@@ -135,17 +141,27 @@ optional arguments:
   -h, --help            show this help message and exit
   --coverage [COVERAGE]
                         Generate coverage
-  -m [MODULE], --module [MODULE]
-                        Run tests for selected module: API_CLI, API_QUERY, API_TASKS, COMMON_CONFIGURATION,
-                        COMMON_DATE_PARSER, SENDER_CLI, SENDER_NUMBER_LOOKUP, SENDER_SEND_DATA, SENDER_SEND_LOOKUP
+  -m [MODULES], --modules [MODULES]
+                        Run tests for selected modules: API_CLI, API_QUERY, API_TASKS, API_ERRORS, API_PARSER_DATE,
+                        API_PROCESSORS, API_KEEPALIVE, COMMON_CONFIGURATION, COMMON_DATE_PARSER, SENDER_CLI, SENDER_CSV,
+                        SENDER_NUMBER_LOOKUP, SENDER_SEND_DATA, SENDER_SEND_LOOKUP
+  -M [EXCLUDE_MODULES], --exclude-modules [EXCLUDE_MODULES]
+                        Exclude tests for modules: API_CLI, API_QUERY, API_TASKS, API_ERRORS, API_PARSER_DATE,
+                        API_PROCESSORS, API_KEEPALIVE, COMMON_CONFIGURATION, COMMON_DATE_PARSER, SENDER_CLI, SENDER_CSV,
+                        SENDER_NUMBER_LOOKUP, SENDER_SEND_DATA, SENDER_SEND_LOOKUP
 ```
 
 * API_CLI: API Command-line interface tests.
 * API_QUERY: Query API tests.
 * API_TASKS: Task API tests.
+* API_ERRORS: Managing of API Errors tests.
+* API_PARSER_DATE: Parsing of dates in API tests.
+* API_PROCESSORS: Response processors in API tests.
+* API_KEEPALIVE: Keep Alive functionality in API tests.
 * COMMON_CONFIGURATION: Configuration tests.
 * COMMON_DATE_PARSER: Date parser tests.
 * SENDER_CLI: Lookup command-line interface tests.
+* SENDER_CSV: Lookup uploading through CSV tests.
 * SENDER_NUMBER_LOOKUP: Numbers in lookup tests
 * SENDER_SEND_DATA: Data sending tests.
 * SENDER_SEND_LOOKUP: Lookup sending tests.

devo/sender/data.py

@@ -195,7 +195,7 @@ def check_config_certificate_chain(self):
     def check_config_certificate_address(self):
         """
         Check if the certificate is compatible with the
-        address, also check is the address and port are
+        address, also check if the address and port are
         valid.
 
         :return: Boolean true or raises an exception

devo/sender/scripts/sender_cli.py

@@ -77,6 +77,9 @@ def cli(version):
 @click.option('--compression_level', help='Compression level for zipped data. '
                                           'Read readme for more info',
               type=int)
+@click.option('--no-verify-certificates', help='Do not Verify certificates '
+                                               'credentials before connection',
+              type=bool, is_flag=True)
 @click.option('--env', '-e', help='Use env vars for configuration',
               default=False, type=bool)
 @click.option('--default', '-d', help='Use default file for configuration',
@@ -176,6 +179,9 @@ def data(**kwargs):
 @click.option('--escapequotes', '-eq', is_flag=True,
               help='Escape Quotes. Default: False',
               default=False)
+@click.option('--no-verify-certificates', help='Do not Verify certificates '
+                                               'credentials before connection',
+              type=bool, is_flag=True)
 @click.option('--debug/--no-debug', help='For testing purposes', default=False)
 def lookup(**kwargs):
     """Send csv lookups to devo"""
@@ -226,6 +232,8 @@ def init_conf(args):
         if args.get('config'):
             config.load_config(args.get('config'), 'sender')
 
+        config['verify_config'] = not args.get('no_verify_certificates', False)
+
         if args.get('env'):
             config.set("address",
                        os.environ.get('DEVO_SENDER_ADDRESS',

docs/api/api.md

@@ -176,7 +176,7 @@ Query by id has the same parameters as query (), changing the field "query"
 to "query_id", which is the ID of the query in Devo.
 
 ## Not verify certificates
-If server has https certificates with problems, autogenrados or not verified, you can deactivate
+If server has https certificates with problems, autogenerated or not verified, you can deactivate
  the secure calls (Verifying the certificates https when making the call) disabling it with:
 
 ```python

tests/sender/cli.py

@@ -13,11 +13,15 @@
 
 class TestSender(unittest.TestCase):
     def setUp(self):
-        self.address = os.getenv('DEVO_SENDER_SERVER', "127.0.0.1")
-        self.port = int(os.getenv('DEVO_SENDER_PORT', 4488))
+        self.local_address = os.getenv('DEVO_SENDER_SERVER', "127.0.0.1")
+        self.local_port = int(os.getenv('DEVO_SENDER_PORT', 4488))
         self.tcp_address = os.getenv('DEVO_SENDER_TCP_SERVER', "127.0.0.1")
         self.tcp_port = int(os.getenv('DEVO_SENDER_TCP_PORT', 4489))
 
+        self.remote_address = os.getenv('DEVO_REMOTE_SENDER_SERVER',
+                                 "collector-us.devo.io")
+        self.remote_port = int(os.getenv('DEVO_REMOTE_SENDER_PORT', 443))
+
         self.key = os.getenv('DEVO_SENDER_KEY', CLIENT_KEY)
         self.cert = os.getenv('DEVO_SENDER_CERT', CLIENT_CERT)
         self.chain = os.getenv('DEVO_SENDER_CHAIN', CLIENT_CHAIN)
@@ -42,7 +46,7 @@ def setUp(self):
         configuration = Configuration()
         configuration.set("sender", {
             "key": self.key, "cert": self.cert, "chain": self.chain,
-            "address": self.address, "port": self.port,
+            "address": self.local_address, "port": self.local_port,
             "verify_mode": 0, "check_hostname": False
         })
 
@@ -57,9 +61,10 @@ def test_cli_args(self):
     def test_cli_bad_address(self):
         runner = CliRunner()
         result = runner.invoke(data, ["--debug",
-                                      "--address", self.address + "asd"])
+                                      "--type", "TCP",
+                                      "--address", self.local_address + "asd"])
         self.assertIsInstance(result.exception, DevoSenderException)
-        self.assertIn("SSL conn establishment socket error", result.stdout)
+        self.assertIn("TCP conn establishment socket error", result.stdout)
 
     def test_cli_bad_certs(self):
         runner = CliRunner()
@@ -73,6 +78,22 @@ def test_cli_bad_certs(self):
                                       "--verify_mode", 1,
                                       '--check_hostname', True])
         self.assertIsInstance(result.exception, DevoSenderException)
+        self.assertIn("Error in the configuration",
+                      result.exception.args[0])
+
+    def test_cli_bad_certs_no_verify_on(self):
+        runner = CliRunner()
+        result = runner.invoke(data, ["--debug",
+                                      "--address",
+                                      "collector-us.devo.io",
+                                      "--port", "443",
+                                      "--key", self.local_key,
+                                      "--cert", self.cert,
+                                      "--chain", self.chain,
+                                      "--verify_mode", 1,
+                                      '--check_hostname', True,
+                                      "--no-verify-certificates"])
+        self.assertIsInstance(result.exception, DevoSenderException)
         self.assertIn("SSL conn establishment socket error",
                       result.exception.args[0])
 
@@ -92,11 +113,28 @@ def test_cli_notfound_certs(self):
                       "'not_a_folder/not_a_file' does not exist.",
                       result.output)
 
-    def test_cli_normal_send(self):
+    def test_cli_normal_send_without_certificates_checking(self):
         runner = CliRunner()
         result = runner.invoke(data, ["--debug",
-                                      "--address", self.address,
-                                      "--port", self.port,
+                                      "--address", self.local_address,
+                                      "--port", self.local_port,
+                                      "--key", self.key,
+                                      "--cert", self.cert,
+                                      "--chain", self.chain,
+                                      "--tag", self.my_app,
+                                      "--verify_mode", 0,
+                                      '--check_hostname', False,
+                                      "--line", "Test line",
+                                      "--no-verify-certificates"])
+
+        self.assertIsNone(result.exception)
+        self.assertGreater(int(result.output.split("Sended: ")[-1]), 0)
+
+    def test_cli_normal_send_with_certificates_checking(self):
+        runner = CliRunner()
+        result = runner.invoke(data, ["--debug",
+                                      "--address", self.remote_address,
+                                      "--port", self.remote_port,
                                       "--key", self.key,
                                       "--cert", self.cert,
                                       "--chain", self.chain,
@@ -112,16 +150,17 @@ def test_cli_with_config_file(self):
         if self.config_path:
             runner = CliRunner()
             result = runner.invoke(data, ["--debug",
-                                          "--config", self.config_path])
+                                          "--config", self.config_path,
+                                          "--no-verify-certificates"])
 
             self.assertIsNone(result.exception)
             self.assertGreater(int(result.output.split("Sended: ")[-1]), 0)
 
     def test_cli_escape_quotes(self):
         runner = CliRunner()
         result = runner.invoke(lookup, ["--debug",
-                                        "--address", self.address,
-                                        "--port", self.port,
+                                        "--address", self.local_address,
+                                        "--port", self.local_port,
                                         "--key", self.key,
                                         "--cert", self.cert,
                                         "--chain", self.chain,
@@ -131,7 +170,8 @@ def test_cli_escape_quotes(self):
                                         "-ac", "FULL",
                                         "-f", self.lookup_file,
                                         "-lk", "KEY",
-                                        "-eq"
+                                        "-eq",
+                                        "--no-verify-certificates"
                                         ])
 
         self.assertIsNone(result.exception)
@@ -140,8 +180,8 @@ def test_cli_escape_quotes(self):
     def test_cli_not_escape_quotes(self):
         runner = CliRunner()
         result = runner.invoke(lookup, ["--debug",
-                                        "--address", self.address,
-                                        "--port", self.port,
+                                        "--address", self.local_address,
+                                        "--port", self.local_port,
                                         "--key", self.key,
                                         "--cert", self.cert,
                                         "--chain", self.chain,
@@ -150,12 +190,12 @@ def test_cli_not_escape_quotes(self):
                                         "-n", self.lookup_name,
                                         "-ac", "FULL",
                                         "-f", self.lookup_file,
-                                        "-lk", "KEY"
+                                        "-lk", "KEY",
+                                        "--no-verify-certificates"
                                         ])
 
         self.assertIsNotNone(result.exception)
         self.assertEquals(result.exit_code, 64)
 
-
 if __name__ == '__main__':
     unittest.main()