Update linux consumption tests to test with dev library (#990)

* Update host version for linux consumption tests

* Revert host version to v3

* Adding isolate dependencies flag in docker image

* Fixed lint tests

* Skipping common libraries test for python3.7

Author: gavin-aguiar

azure_functions_worker/testutils_lc.py

@@ -1,28 +1,34 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-
-from typing import Dict
-
 import base64
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
+import tempfile
 import time
 import uuid
+from io import BytesIO
+from typing import Dict
+from urllib.request import urlopen
+from zipfile import ZipFile
 
+import requests
 from Crypto.Cipher import AES
 from Crypto.Hash.SHA256 import SHA256Hash
 from Crypto.Util.Padding import pad
-import requests
 
 # Linux Consumption Testing Constants
 _DOCKER_PATH = "DOCKER_PATH"
 _DOCKER_DEFAULT_PATH = "docker"
 _MESH_IMAGE_URL = "https://mcr.microsoft.com/v2/azure-functions/mesh/tags/list"
 _MESH_IMAGE_REPO = "mcr.microsoft.com/azure-functions/mesh"
 _DUMMY_CONT_KEY = "MDEyMzQ1Njc4OUFCQ0RFRjAxMjM0NTY3ODlBQkNERUY="
+_FUNC_GITHUB_ZIP = "https://github.com/Azure/azure-functions-python-library" \
+                   "/archive/refs/heads/dev.zip"
+_FUNC_FILE_NAME = "azure-functions-python-library-dev"
 
 
 class LinuxConsumptionWebHostController:
@@ -80,9 +86,9 @@ def assign_container(self, env: Dict[str, str] = {}):
                                f' stdout: {stdout}')
 
     def send_request(
-        self,
-        req: requests.Request,
-        ses: requests.Session = None
+            self,
+            req: requests.Request,
+            ses: requests.Session = None
     ) -> requests.Response:
         """Send a request with authorization token. Return a Response object"""
         session = ses
@@ -129,6 +135,12 @@ def _find_latest_mesh_image(cls,
         cls._mesh_images[host_major] = image_tag
         return image_tag
 
+    @staticmethod
+    def _download_azure_functions() -> str:
+        with urlopen(_FUNC_GITHUB_ZIP) as zipresp:
+            with ZipFile(BytesIO(zipresp.read())) as zfile:
+                zfile.extractall(tempfile.gettempdir())
+
     def spawn_container(self,
                         image: str,
                         env: Dict[str, str] = {}) -> int:
@@ -137,10 +149,18 @@ def spawn_container(self,
         """
         # Construct environment variables and start the docker container
         worker_path = os.path.dirname(__file__)
+        library_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME,
+                                    'azure', 'functions')
+        self._download_azure_functions()
+
         container_worker_path = (
             f"/azure-functions-host/workers/python/{self._py_version}/"
             "LINUX/X64/azure_functions_worker"
         )
+        container_library_path = (
+            f"/azure-functions-host/workers/python/{self._py_version}/"
+            "LINUX/X64/azure/functions"
+        )
 
         run_cmd = []
         run_cmd.extend([self._docker_cmd, "run", "-p", "0:80", "-d"])
@@ -150,7 +170,9 @@ def spawn_container(self,
         run_cmd.extend(["-e", f"CONTAINER_NAME={self._uuid}"])
         run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"])
         run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"])
+        run_cmd.extend(["-e", "PYTHON_ISOLATE_WORKER_DEPENDENCIES=1"])
         run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}'])
+        run_cmd.extend(["-v", f'{library_path}:{container_library_path}'])
 
         for key, value in env.items():
             run_cmd.extend(["-e", f"{key}={value}"])
@@ -181,7 +203,7 @@ def spawn_container(self,
         self._ports[self._uuid] = port_number
 
         # Wait for three seconds for the container to be in ready state
-        time.sleep(3)
+        time.sleep(6)
         return port_number
 
     def get_container_logs(self) -> str:
@@ -259,6 +281,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         logs = self.get_container_logs()
         self.safe_kill_container()
+        shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME))
 
         if traceback:
             print(f'Test failed with container logs: {logs}',

tests/endtoend/test_linux_consumption.py

@@ -1,15 +1,16 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-from unittest import TestCase, skipIf
-
 import os
 import sys
-from requests import Request
+from unittest import TestCase, skipIf
 
 from azure_functions_worker.testutils_lc import (
     LinuxConsumptionWebHostController
 )
 from azure_functions_worker.utils.common import is_python_version
+from requests import Request
+
+_DEFAULT_HOST_VERSION = "3"
 
 
 @skipIf(is_python_version('3.10'),
@@ -42,7 +43,8 @@ def test_placeholder_mode_root_returns_ok(self):
         """In any circumstances, a placeholder container should returns 200
         even when it is not specialized.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             req = Request('GET', ctrl.url)
             resp = ctrl.send_request(req)
             self.assertTrue(resp.ok)
@@ -51,7 +53,8 @@ def test_http_no_auth(self):
         """An HttpTrigger function app with 'azure-functions' library
         should return 200.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("HttpNoAuth")
@@ -60,6 +63,8 @@ def test_http_no_auth(self):
             resp = ctrl.send_request(req)
             self.assertEqual(resp.status_code, 200)
 
+    @skipIf(is_python_version('3.7'),
+            "Skip the tests for Python 3.7.")
     def test_common_libraries(self):
         """A function app with the following requirements.txt:
 
@@ -73,7 +78,8 @@ def test_common_libraries(self):
 
         should return 200 after importing all libraries.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("CommonLibraries")
@@ -98,7 +104,8 @@ def test_new_protobuf(self):
 
         should return 200 after importing all libraries.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf")
@@ -123,10 +130,11 @@ def test_old_protobuf(self):
 
         should return 200 after importing all libraries.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
-                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf")
+                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf")
             })
             req = Request('GET', f'{ctrl.url}/api/HttpTrigger')
             resp = ctrl.send_request(req)
@@ -144,7 +152,8 @@ def test_debug_logging_disabled(self):
         should return 200 and by default customer debug logging should be
         disabled.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url("EnableDebugLogging")
@@ -170,7 +179,8 @@ def test_debug_logging_enabled(self):
         should return 200 and with customer debug logging enabled, debug logs
         should be written to container logs.
         """
-        with LinuxConsumptionWebHostController("3", self._py_version) as ctrl:
+        with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION,
+                                               self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url(