Running tests in parallel (#1130)

* Running tests in parallel

* Updated lc tests to async

* Making lc tests async

* Undo asyncio

* Flake8 fix

* Unskip lc tests for 3.10

* Skip lc tests for 310

* Updating code coverage to v3

* Test fix for 310 ut

Co-authored-by: Gavin Aguiar <gavin@GavinPC.localdomain>
Co-authored-by: Varad Meru <vrdmr@users.noreply.github.com>

Author: 3 people

.github/workflows/ci_e2e_workflow.yml

@@ -76,7 +76,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString36 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString36 }}
         run: |
-          python -m pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7
         env:
@@ -88,7 +88,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
         run: |
-          python -m pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.8 Tests
         if: matrix.python-version == 3.8
         env:
@@ -100,7 +100,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
         run: |
-          python -m pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.9 Tests
         if: matrix.python-version == 3.9
         env:
@@ -112,7 +112,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
         run: |
-          python -m pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.10 Tests
         if: matrix.python-version == 3.10
         env:
@@ -124,7 +124,7 @@ jobs:
           AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
           AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
         run: |
-          python -m pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
+          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Codecov
         uses: codecov/codecov-action@v1.0.13
         with:

.github/workflows/ogf_workflow.yml

@@ -81,7 +81,7 @@ jobs:
       - name: Install node and npm
         uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '16'
           check-latest: true
 
       - name: Azure Login

.github/workflows/ut_ci_workflow.yml

@@ -64,11 +64,11 @@ jobs:
         retry 5 python setup.py extension
     - name: Test with pytest
       env:
-        AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+        AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} # needed for installing azure-functions-durable while running setup.py
       run: |
-        python -m pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
+        python -m pytest -n auto --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests
     - name: Codecov
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v3
       with:
         file: ./coverage.xml # optional
         flags: unittests # optional

azure_functions_worker/logging.py

@@ -23,12 +23,12 @@
 
 def format_exception(exception: Exception) -> str:
     msg = str(exception) + "\n"
-    if sys.version_info.minor < 10:
+    if (sys.version_info.major, sys.version_info.minor) < (3, 10):
         msg += ''.join(traceback.format_exception(
             etype=type(exception),
             tb=exception.__traceback__,
             value=exception))
-    elif sys.version_info.minor == 10:
+    elif (sys.version_info.major, sys.version_info.minor) == (3, 10):
         msg += ''.join(traceback.format_exception(exception))
     else:
         msg = str(exception)

codecov.yml

@@ -1,3 +1,3 @@
 ignore:
-  - "azure_functions_worker/testutils.py"
-  - "azure_functions_worker/testutils_*.py"
+  - "tests/utils/testutils.py"
+  - "tests/utils/testutils_lc.py"

tests/unittests/test_enable_debug_logging_functions.py

@@ -4,6 +4,8 @@
 import os
 from unittest.mock import patch
 
+import pytest
+
 from tests.utils import testutils
 from azure_functions_worker.constants import PYTHON_ENABLE_DEBUG_LOGGING
 from tests.utils.testutils import TESTS_ROOT, remove_path
@@ -42,6 +44,7 @@ def tearDownClass(cls):
     def get_script_dir(cls):
         return testutils.UNIT_TESTS_FOLDER / 'log_filtering_functions'
 
+    @pytest.mark.flaky(reruns=3)
     def test_debug_logging_enabled(self):
         """
         Verify when cx debug logging is enabled, cx function debug logs
@@ -79,6 +82,7 @@ def tearDownClass(cls):
     def get_script_dir(cls):
         return testutils.UNIT_TESTS_FOLDER / 'log_filtering_functions'
 
+    @pytest.mark.flaky(reruns=3)
     def test_debug_logging_disabled(self):
         """
         Verify when cx debug logging is disabled, cx function debug logs
@@ -125,6 +129,7 @@ def tearDownClass(cls):
     def get_script_dir(cls):
         return testutils.UNIT_TESTS_FOLDER / 'log_filtering_functions'
 
+    @pytest.mark.flaky(reruns=3)
     def test_debug_logging_filtered(self):
         """
         Verify when cx debug logging is enabled and host logging level

tests/unittests/test_http_functions.py

@@ -100,6 +100,7 @@ def check_log_async_logging(self, host_out: typing.List[str]):
         self.assertIn('hello info', host_out)
         self.assertIn('and another error', host_out)
 
+    @pytest.mark.flaky(reruns=3)
     def test_debug_logging(self):
         r = self.webhost.request('GET', 'debug_logging')
         self.assertEqual(r.status_code, 200)
@@ -111,6 +112,7 @@ def check_log_debug_logging(self, host_out: typing.List[str]):
         self.assertIn('logging error', host_out)
         self.assertNotIn('logging debug', host_out)
 
+    @pytest.mark.flaky(reruns=3)
     def test_debug_with_user_logging(self):
         r = self.webhost.request('GET', 'debug_user_logging')
         self.assertEqual(r.status_code, 200)
@@ -306,6 +308,7 @@ def test_application_octet_stream_content_type(self):
             if (os.path.exists(received_img_file)):
                 os.remove(received_img_file)
 
+    @pytest.mark.flaky(reruns=3)
     def test_user_event_loop_error(self):
         # User event loop is not supported in HTTP trigger
         r = self.webhost.request('GET', 'user_event_loop/')

tests/unittests/test_log_filtering_functions.py

@@ -2,6 +2,8 @@
 # Licensed under the MIT License.
 import typing
 
+import pytest
+
 from tests.utils import testutils
 
 HOST_JSON_TEMPLATE_WITH_LOGLEVEL_INFO = """\
@@ -60,6 +62,7 @@ def check_log_debug_logging(self, host_out: typing.List[str]):
         # See HOST_JSON_TEMPLATE_WITH_LOGLEVEL_INFO, debug log is disabled
         self.assertNotIn('logging debug', host_out)
 
+    @pytest.mark.flaky(reruns=3)
     def test_debug_with_user_logging(self):
         r = self.webhost.request('GET', 'debug_user_logging')
         self.assertEqual(r.status_code, 200)

tests/unittests/test_shared_memory_map.py

@@ -6,6 +6,8 @@
 import unittest
 from unittest import skipIf
 
+import pytest
+
 from tests.utils import testutils
 from azure_functions_worker.bindings.shared_memory_data_transfer \
     import SharedMemoryMap
@@ -53,6 +55,7 @@ def test_init_with_invalid_inputs(self):
                                     'Invalid memory map'):
             SharedMemoryMap(self.file_accessor, mem_map_name, None)
 
+    @pytest.mark.flaky(reruns=3)
     def test_put_bytes(self):
         """
         Create a SharedMemoryMap and write bytes to it.
@@ -70,6 +73,7 @@ def test_put_bytes(self):
             dispose_status = shared_mem_map.dispose()
             self.assertTrue(dispose_status)
 
+    @pytest.mark.flaky(reruns=3)
     def test_get_bytes(self):
         """
         Create a SharedMemoryMap, write bytes to it and then read them back.

tests/unittests/test_third_party_http_functions.py

@@ -53,6 +53,7 @@ def tearDownClass(cls):
         def get_script_dir(cls):
             pass
 
+        @pytest.mark.flaky(reruns=3)
         def test_debug_logging(self):
             r = self.webhost.request('GET', 'debug_logging', no_prefix=True)
             self.assertEqual(r.status_code, 200)
@@ -64,6 +65,7 @@ def check_log_debug_logging(self, host_out: typing.List[str]):
             self.assertIn('logging error', host_out)
             self.assertNotIn('logging debug', host_out)
 
+        @pytest.mark.flaky(reruns=3)
         def test_debug_with_user_logging(self):
             r = self.webhost.request('GET', 'debug_user_logging',
                                      no_prefix=True)