Docker tests updates (#1457)

gavin-aguiar · hallvictoria · web-flow · commit 30789ed5cac2 · 2024-04-02T15:26:16.000-05:00
* Docker tests updates

* Fixed consumption custom image if empty

* Updated extension csproj to use .net8

* Update extensions to include System.Drawing.Common

* Update extensions to include System.Text.Json

* Added table extension

* table &amp; eh, unit, lint

* eh test

* lint

---------

Co-authored-by: hallvictoria &lt;victoriahall@microsoft.com&gt;
diff --git a/.github/workflows/ci_consumption_workflow.yml b/.github/workflows/ci_consumption_workflow.yml
@@ -5,10 +5,14 @@ name: CI Consumption E2E tests
 
 on:
   workflow_dispatch:
+    inputs:
+      custom_image:
+        description: "Use a custom image to run consumption tests"
+        required: false
   push:
-    branches: [ dev, master, main, release/* ]
+    branches: [ dev, main, release/* ]
   pull_request:
-    branches: [ dev, master, main, release/* ]
+    branches: [ dev, main, release/* ]
 
 jobs:
   build:
@@ -35,24 +39,29 @@ jobs:
         if: matrix.python-version == 3.7
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv  --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.8 Tests
         if: matrix.python-version == 3.8
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv  --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.9 Tests
         if: matrix.python-version == 3.9
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.10 Tests
         if: matrix.python-version == 3.10
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv --reruns 4 --instafail tests/consumption_tests
       - name: Running 3.11 Tests
         if: matrix.python-version == 3.11
         env:
           AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }}
+          CUSTOM_IMAGE: ${{ github.event.inputs.custom_image }}
         run: python -m pytest -n auto --dist loadfile -vv --reruns 4 --instafail tests/consumption_tests
diff --git a/.github/workflows/ci_docker_con_workflow.yml b/.github/workflows/ci_docker_con_workflow.yml
@@ -5,9 +5,8 @@ name: CI Docker Consumption tests
 on:
   workflow_dispatch:
   schedule:
-    # Monday to Thursday 2 AM PDT build
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 9 * * *"
+    # Run everyday at 5 AM CST
+    - cron: "0 10 * * *"
 
 jobs:
   build:
@@ -30,7 +29,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install -U -e .[dev]
           python setup.py build
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7
diff --git a/.github/workflows/ci_docker_custom_workflow.yml b/.github/workflows/ci_docker_custom_workflow.yml
@@ -49,7 +49,7 @@ jobs:
           python-version: ${{ env.python_version }}
       - name: Install dependencies
         run: |
-          python -m pip install -U -e .[dev]
+          python setup.py build
       - name: Running 3.7 Tests
         if: env.python_version == 3.7
         env:
diff --git a/.github/workflows/ci_docker_ded_workflow.yml b/.github/workflows/ci_docker_ded_workflow.yml
@@ -5,9 +5,8 @@ name: CI Docker Dedicated tests
 on:
   workflow_dispatch:
   schedule:
-    # Monday to Thursday 1 AM PDT build
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 8 * * *"
+    # Run everyday at 4 AM CST
+    - cron: "0 9 * * *"
 
 jobs:
   build:
@@ -30,7 +29,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install -U -e .[dev]
           python setup.py build
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7
diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml
@@ -11,11 +11,11 @@ on:
         required: false
         default: "false"
   push:
-    branches: [dev, master, main, release/*]
+    branches: [dev, main, release/*]
   pull_request:
-    branches: [dev, master, main, release/*]
+    branches: [dev, main, release/*]
   schedule:
-    # Monday to Thursday 1 AM PDT build
+    # Monday to Thursday 3 AM CST build
     # * is a special character in YAML so you have to quote this string
     - cron: "0 8 * * 1,2,3,4"
 
@@ -35,10 +35,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Set up Dotnet 6.x
-        uses: actions/setup-dotnet@v4
-        with:
-          dotnet-version: "6.x"
       - name: Set up Dotnet 8.0.x
         uses: actions/setup-dotnet@v4
         with:
diff --git a/.github/workflows/ci_ut_workflow.yml b/.github/workflows/ci_ut_workflow.yml
@@ -11,12 +11,12 @@ on:
         required: false
         default: "false"
   schedule:
-    # Monday to Thursday 1 AM PDT build
+    # Monday to Thursday 3 AM CST build
     # * is a special character in YAML so you have to quote this string
     - cron: "0 8 * * 1,2,3,4"
   push:
   pull_request:
-    branches: [ dev, master, main, release/* ]
+    branches: [ dev, main, release/* ]
 
 jobs:
   build:
@@ -63,6 +63,7 @@ jobs:
         # Retry a couple times to avoid certificate issue
         retry 5 python setup.py build
         retry 5 python setup.py webhost --branch-name=dev
+        retry 5 python setup.py extension
         mkdir logs
     - name: Test with pytest
       env:
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ What are the supported Python versions?
 |----------------------------------|------------|------------|------------|------------|-------------|-------------|
 | Azure Functions 2.0 (deprecated) | ✔          | ✔          | -          | -          | -           | -           |
 | Azure Functions 3.0 (deprecated)             | ✔          | ✔          | ✔          | ✔          | -           | -           |
-| Azure Functions 4.0              | -          | ✔          | ✔          | ✔          | ✔           | coming soon |
+| Azure Functions 4.0              | -          | -          | ✔          | ✔          | ✔           | ✔           |     
 
 For information about Azure Functions Runtime, please refer to [Azure Functions runtime versions overview](https://docs.microsoft.com/en-us/azure/azure-functions/functions-versions) page.
 
diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py b/tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py
@@ -18,10 +18,10 @@
     connection="AzureWebJobsEventHubConnectionString",
     data_type="string",
     cardinality="many")
-@app.table_output(arg_name="$return",
-                  connection="AzureWebJobsStorage",
-                  table_name="EventHubBatchTest")
-def eventhub_multiple(events):
+@app.blob_output(arg_name="$return",
+                 path="python-worker-tests/test-eventhub-batch-triggered.txt",
+                 connection="AzureWebJobsStorage")
+def eventhub_multiple(events) -> str:
     table_entries = []
     for event in events:
         json_entry = event.get_body()
@@ -46,13 +46,12 @@ def eventhub_output_batch(req: func.HttpRequest, out: func.Out[str]) -> str:
 
 # Retrieve the event data from storage blob and return it as Http response
 @app.function_name(name="get_eventhub_batch_triggered")
-@app.route(route="get_eventhub_batch_triggered/{id}")
-@app.table_input(arg_name="testEntities",
-                 connection="AzureWebJobsStorage",
-                 table_name="EventHubBatchTest",
-                 partition_key="{id}")
-def get_eventhub_batch_triggered(req: func.HttpRequest, testEntities):
-    return func.HttpResponse(status_code=200, body=testEntities)
+@app.route(route="get_eventhub_batch_triggered")
+@app.blob_input(arg_name="testEntities",
+                path="python-worker-tests/test-eventhub-batch-triggered.txt",
+                connection="AzureWebJobsStorage")
+def get_eventhub_batch_triggered(req: func.HttpRequest, testEntities: func.InputStream):
+    return func.HttpResponse(status_code=200, body=testEntities.read().decode('utf-8'))
 
 
 # Retrieve the event data from storage blob and return it as Http response
diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py b/tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py
@@ -5,7 +5,7 @@
 
 # This is an actual EventHub trigger which handles Eventhub events in batches.
 # It serializes multiple event data into a json and store it into a blob.
-def main(events):
+def main(events) -> str:
     table_entries = []
     for event in events:
         json_entry = event.get_body()
diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json b/tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json
@@ -13,9 +13,9 @@
     },
     {
       "direction": "out",
-      "type": "table",
+      "type": "blob",
       "name": "$return",
-      "tableName": "EventHubBatchTest",
+      "path": "python-worker-tests/test-eventhub-batch-triggered.txt",
       "connection": "AzureWebJobsStorage"
     }
   ]
diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py b/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py
@@ -4,5 +4,5 @@
 
 
 # Retrieve the event data from storage blob and return it as Http response
-def main(req: func.HttpRequest, testEntities):
-    return func.HttpResponse(status_code=200, body=testEntities)
+def main(req: func.HttpRequest, testEntities: func.InputStream):
+    return func.HttpResponse(status_code=200, body=testEntities.read().decode('utf-8'))
diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json b/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json
@@ -12,10 +12,9 @@
     },
     {
       "direction": "in",
-      "type": "table",
+      "type": "blob",
       "name": "testEntities",
-      "partitionKey": "WillBePopulated",
-      "tableName": "EventHubBatchTest",
+      "path": "python-worker-tests/test-eventhub-batch-triggered.txt",
       "connection": "AzureWebJobsStorage"
     },
     {
diff --git a/tests/endtoend/table_functions/table_functions_stein/function_app.py b/tests/endtoend/table_functions/table_functions_stein/function_app.py
@@ -15,8 +15,7 @@
                  row_key='{id}',
                  partition_key="test")
 def table_in_binding(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
 
 
 @app.function_name(name="table_out_binding")
@@ -28,6 +27,5 @@ def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json
diff --git a/tests/endtoend/table_functions/table_functions_stein/generic/function_app.py b/tests/endtoend/table_functions/table_functions_stein/generic/function_app.py
@@ -19,8 +19,7 @@
     row_key="{id}",
     partition_key="test")
 def table_in_binding(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
 
 
 @app.function_name(name="table_out_binding")
@@ -36,6 +35,5 @@ def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json
diff --git a/tests/endtoend/table_functions/table_in_binding/__init__.py b/tests/endtoend/table_functions/table_in_binding/__init__.py
@@ -1,10 +1,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-import json
-
 import azure.functions as func
 
 
 def main(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
diff --git a/tests/endtoend/table_functions/table_out_binding/__init__.py b/tests/endtoend/table_functions/table_out_binding/__init__.py
@@ -10,6 +10,5 @@ def main(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json
diff --git a/tests/endtoend/test_eventhub_batch_functions.py b/tests/endtoend/test_eventhub_batch_functions.py
diff --git a/tests/endtoend/test_servicebus_functions.py b/tests/endtoend/test_servicebus_functions.py
diff --git a/tests/endtoend/test_table_functions.py b/tests/endtoend/test_table_functions.py
diff --git a/tests/unittests/http_functions/debug_user_logging/function.json b/tests/unittests/http_functions/debug_user_logging/function.json
diff --git a/tests/unittests/http_functions/debug_user_logging/main.py b/tests/unittests/http_functions/debug_user_logging/main.py
diff --git a/tests/unittests/test_http_functions.py b/tests/unittests/test_http_functions.py
diff --git a/tests/utils/constants.py b/tests/utils/constants.py
diff --git a/tests/utils/testutils_lc.py b/tests/utils/testutils_lc.py

Original file line number	Diff line number	Diff line change
`@@ -13,9 +13,9 @@`
`13`	`13`	`},`
`14`	`14`	`{`
`15`	`15`	`"direction": "out",`
`16`		`- "type": "table",`
	`16`	`+ "type": "blob",`
`17`	`17`	`"name": "$return",`
`18`		`- "tableName": "EventHubBatchTest",`
	`18`	`+ "path": "python-worker-tests/test-eventhub-batch-triggered.txt",`
`19`	`19`	`"connection": "AzureWebJobsStorage"`
`20`	`20`	`}`
`21`	`21`	`]`