Add retry pattern in Unittest and E2E pipelines

Author: Hanzhang Zeng (Roger)

.github/workflows/ci_e2e_workflow.yml

@@ -36,12 +36,31 @@ jobs:
           dotnet-version: '3.1.405'
       - name: Install dependencies and the worker
         run: |
+          retry() {
+            local -r -i max_attempts="$1"; shift
+            local -r cmd="$@"
+            local -i attempt_num=1
+            until $cmd
+            do
+              if (( attempt_num == max_attempts ))
+              then
+                echo "Attempt $attempt_num failed and there are no more attempts left!"
+                return 1
+              else
+                echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+                sleep 1
+              fi
+            done
+          }
+
           python -m pip install --upgrade pip
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
-          python setup.py build
-          python setup.py webhost
-          python setup.py extension
+
+          # Retry a couple times to avoid certificate issue
+          retry 5 python setup.py build
+          retry 5 python setup.py webhost
+          retry 5 python setup.py extension
       - name: Running 3.6 Tests
         if: matrix.python-version == 3.6
         env:

.github/workflows/ut_ci_workflow.yml

@@ -33,12 +33,31 @@ jobs:
         dotnet-version: '3.1.405'
     - name: Install dependencies and the worker
       run: |
+        retry() {
+          local -r -i max_attempts="$1"; shift
+          local -r cmd="$@"
+          local -i attempt_num=1
+          until $cmd
+          do
+            if (( attempt_num == max_attempts ))
+            then
+              echo "Attempt $attempt_num failed and there are no more attempts left!"
+              return 1
+            else
+              echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+              sleep 1
+            fi
+          done
+        }
+
         python -m pip install --upgrade pip
         python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
         python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
-        python setup.py build
-        python setup.py webhost
-        python setup.py extension
+
+        # Retry a couple times to avoid certificate issue
+        retry 5 python setup.py build
+        retry 5 python setup.py webhost
+        retry 5 python setup.py extension
     - name: Test with pytest
       run: |
         pytest  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch tests/unittests