From 41b22f051c7e788d0da0d620e718929c71d5d433 Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Wed, 24 Apr 2019 20:08:45 +0100 Subject: [PATCH 1/9] Tests: Hack to run controller with coverage using the `run` script --- services/controller/run | 28 +++++++++++++------ .../controller/src/plz/controller/main.py | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/services/controller/run b/services/controller/run index d59ed837..26d78efc 100755 --- a/services/controller/run +++ b/services/controller/run @@ -20,12 +20,22 @@ if [[ "${CREATE_AWS_RESOURCES:-}" ]]; then python src/plz/controller/utils/create_aws_resources.py fi -exec gunicorn \ - --bind="0.0.0.0:${PORT}" \ - --workers=16 \ - --timeout=2000 \ - --pythonpath="${PYTHONPATH}" \ - --capture-output \ - --log-level="${LOG_LEVEL}" \ - plz.controller.main:app \ - -- $@ +CONFIGURATION+=$'\n' +CONFIGURATION+='data_dir = ../../test_cache/'$'\n' +CONFIGURATION+='results = {'$'\n' +CONFIGURATION+=' directory = ../../test_cache/results'$'\n' +CONFIGURATION+='}'$'\n' +CONFIGURATION+='redis_host=localhost'$'\n' + +echo Here +CONFIGURATION="$CONFIGURATION" PORT=$PORT coverage run --source=. src/plz/controller/main.py + +# exec gunicorn \ +# --bind="0.0.0.0:${PORT}" \ +# --workers=16 \ +# --timeout=2000 \ +# --pythonpath="${PYTHONPATH}" \ +# --capture-output \ +# --log-level="${LOG_LEVEL}" \ +# plz.controller.main:app \ +# -- $@ diff --git a/services/controller/src/plz/controller/main.py b/services/controller/src/plz/controller/main.py index c9679d33..17bbf192 100644 --- a/services/controller/src/plz/controller/main.py +++ b/services/controller/src/plz/controller/main.py @@ -22,7 +22,7 @@ ResponseGeneratorFunction = Callable[[], ResponseGenerator] config = configuration.load() -port = config.get_int('port', 8080) +port = 5123 def _setup_logging(): From d3f9ad8b2d48f9bb00582f01629f99cadc778139 Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Thu, 25 Apr 2019 15:17:23 +0100 Subject: [PATCH 2/9] Coverage: coverage report that works locally, might need tidying up --- .gitignore | 4 + Makefile | 3 + Pipfile | 1 + Pipfile.lock | 42 +++++- cli/Dockerfile | 2 +- cli/Makefile | 2 +- cli/Pipfile | 3 +- cli/Pipfile.lock | 121 ++++++++++++------ cli/setup.py | 5 +- services/controller/Pipfile | 5 +- services/controller/Pipfile.lock | 73 ++++++++--- services/controller/run | 34 +++-- .../controller/src/plz/controller/main.py | 2 +- test/coverage/coverage_report.sh | 21 +++ test/coverage/coveragerc | 19 +++ test/docker-compose.yml | 1 + test/run.py | 6 +- test/run_end_to_end_test.py | 60 +++++++-- test/test_utils.py | 14 ++ 19 files changed, 316 insertions(+), 102 deletions(-) create mode 100755 test/coverage/coverage_report.sh create mode 100644 test/coverage/coveragerc diff --git a/.gitignore b/.gitignore index 67a5299c..86b63144 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ dist # Terraform .terraform *.tfstate.backup + +# Coverage results +*.coverage +test/coverage/results/ diff --git a/Makefile b/Makefile index 4dbce3c2..4e2be7bf 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,12 @@ include vars.mk .PHONY: check check: environment pipenv run yapf -rd . + rm -rf test/coverage/results + mkdir test/coverage/results $(MAKE) -C cli check $(MAKE) -C services/controller check BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) python3 test/run.py + pipenv run test/coverage/coverage_report.sh terraform fmt -check .PHONY: format diff --git a/Pipfile b/Pipfile index 835704d2..1df6f7d0 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ verify_ssl = true name = "pypi" [packages] +coverage = "*" [dev-packages] yapf = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 6c07fb4a..6d373f89 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3ec5d8bbb099edcd508a073566bb71a44558e8bdc9c67f3755c94e9b60c927bd" + "sha256": "091a3f897accfb06d283b2158fb03f053fa62cbe5c3adf3702a024b25db1b5e5" }, "pipfile-spec": 6, "requires": { @@ -15,7 +15,45 @@ } ] }, - "default": {}, + "default": { + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "index": "pypi", + "version": "==4.5.3" + } + }, "develop": { "yapf": { "hashes": [ diff --git a/cli/Dockerfile b/cli/Dockerfile index 563e7a90..1567ebf4 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -14,7 +14,7 @@ RUN python setup.py bdist_wheel FROM python:3.6 COPY --from=builder /cli/dist/plz_cli-0.1.0-py3-none-any.whl /tmp/ -RUN pip install /tmp/plz_cli-0.1.0-py3-none-any.whl +RUN pip install --find-links /tmp/ plz_cli[test] ENV PYTHONUNBUFFERED 1 ENTRYPOINT ["plz"] diff --git a/cli/Makefile b/cli/Makefile index f871bf51..85bb0d6a 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -5,4 +5,4 @@ check: test lint .PHONY: test test: environment - pipenv run nosetests + COVERAGE_FILE=../test/coverage/results/cli_nose_tests.coverage pipenv run coverage run --source=. -m nose diff --git a/cli/Pipfile b/cli/Pipfile index ca1d6931..322612ad 100644 --- a/cli/Pipfile +++ b/cli/Pipfile @@ -8,9 +8,10 @@ python_version = "3.6" [packages] "e1839a8" = {path = ".", editable = true} -yapf = "*" [dev-packages] flake8 = "*" flask = "*" nose = "*" +coverage = "*" +yapf = "*" diff --git a/cli/Pipfile.lock b/cli/Pipfile.lock index 1ee0af47..e5dab151 100644 --- a/cli/Pipfile.lock +++ b/cli/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "51806429b4eeb894e27ed8de24d388614b5075b031e3318c5e9b03d43f534908" + "sha256": "0507776da6744c0b36c405cde301a0b58a0cc1bc24ac5d0863e862480308bf73" }, "pipfile-spec": 6, "requires": { @@ -56,36 +56,36 @@ }, "cffi": { "hashes": [ - "sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f", - "sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11", - "sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d", - "sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891", - "sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf", - "sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c", - "sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed", - "sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b", - "sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a", - "sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585", - "sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea", - "sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f", - "sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33", - "sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145", - "sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a", - "sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3", - "sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f", - "sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd", - "sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804", - "sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d", - "sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92", - "sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f", - "sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84", - "sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb", - "sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7", - "sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7", - "sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35", - "sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889" - ], - "version": "==1.12.2" + "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", + "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", + "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", + "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", + "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", + "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", + "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", + "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", + "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", + "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", + "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", + "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", + "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", + "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", + "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", + "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", + "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", + "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", + "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", + "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", + "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", + "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", + "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", + "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", + "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", + "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", + "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", + "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" + ], + "version": "==1.12.3" }, "chardet": { "hashes": [ @@ -224,10 +224,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "version": "==1.24.2" }, "websocket-client": { "hashes": [ @@ -235,14 +235,6 @@ "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" ], "version": "==0.56.0" - }, - "yapf": { - "hashes": [ - "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", - "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" - ], - "index": "pypi", - "version": "==0.27.0" } }, "develop": { @@ -253,6 +245,43 @@ ], "version": "==7.0" }, + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "index": "pypi", + "version": "==4.5.3" + }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -359,6 +388,14 @@ "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" ], "version": "==0.15.2" + }, + "yapf": { + "hashes": [ + "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", + "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" + ], + "index": "pypi", + "version": "==0.27.0" } } } diff --git a/cli/setup.py b/cli/setup.py index 1a978007..3607e37a 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -53,9 +53,10 @@ ], extras_require={ 'test': [ - 'flake8==3.5.0', - 'nose==1.3.7', + 'flake8>=3.5.0', + 'nose>=1.3.7', 'flask >= 1.0.2', + 'coverage >= 4.5.3', ], }, entry_points={ diff --git a/services/controller/Pipfile b/services/controller/Pipfile index 24c3e40d..b93ad4e5 100644 --- a/services/controller/Pipfile +++ b/services/controller/Pipfile @@ -11,7 +11,8 @@ gunicorn = "*" pyhocon = "*" redis = "*" requests = ">=2.20.0" -yapf = "*" [dev-packages] -"flake8" = "*" +coverage = "*" +flake8 = "*" +yapf = "*" diff --git a/services/controller/Pipfile.lock b/services/controller/Pipfile.lock index 39dcb638..bbb80bc4 100644 --- a/services/controller/Pipfile.lock +++ b/services/controller/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "078b039bd6a4593120e1bba669728f7a721be077ef5b46b42cba78aa17a5f753" + "sha256": "667db85574902f65d0f81f2aaeb50b589bb76313272cf9844d5e6474038ba196" }, "pipfile-spec": 6, "requires": {}, @@ -16,18 +16,18 @@ "default": { "boto3": { "hashes": [ - "sha256:bb69628f933a8dba22817c85289b3421b23ac643ff3202b13dd2e933c2717109", - "sha256:c75c45bae9dbdb2ff3fc3482d421a3901e552574a882dba1cffa064715acfbe7" + "sha256:5e5805992e873e5687b5ef3b4c56c386ccb4df1c3364f8b8601d289e2f275be1", + "sha256:95bb6e9ec31237e5eaa99c5c74dd91b5a4352566df93e857138c124a3024727a" ], "index": "pypi", - "version": "==1.9.130" + "version": "==1.9.135" }, "botocore": { "hashes": [ - "sha256:128130b12f8ba4bf07a673b119135264060eb98f6a4a7419cbd1f2c6dc926827", - "sha256:59376112fdee707927b644dd44a1771861f8fe354a48d596131ced83d7a3c05b" + "sha256:1a7d6f98420874e472d397ca6ee1ab7e6087ba501997d6295ab411d9a0004b4e", + "sha256:5a677bfe972cb258678a23a5e9b8321b0277b44933c4f0c1ea4883a5a12c09ec" ], - "version": "==1.12.130" + "version": "==1.12.135" }, "certifi": { "hashes": [ @@ -204,11 +204,11 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], "markers": "python_version >= '3.4'", - "version": "==1.24.1" + "version": "==1.24.2" }, "websocket-client": { "hashes": [ @@ -223,17 +223,46 @@ "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" ], "version": "==0.15.2" - }, - "yapf": { - "hashes": [ - "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", - "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" - ], - "index": "pypi", - "version": "==0.27.0" } }, "develop": { + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "index": "pypi", + "version": "==4.5.3" + }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -269,6 +298,14 @@ "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], "version": "==2.1.1" + }, + "yapf": { + "hashes": [ + "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", + "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" + ], + "index": "pypi", + "version": "==0.27.0" } } } diff --git a/services/controller/run b/services/controller/run index 26d78efc..29a76950 100755 --- a/services/controller/run +++ b/services/controller/run @@ -20,22 +20,18 @@ if [[ "${CREATE_AWS_RESOURCES:-}" ]]; then python src/plz/controller/utils/create_aws_resources.py fi -CONFIGURATION+=$'\n' -CONFIGURATION+='data_dir = ../../test_cache/'$'\n' -CONFIGURATION+='results = {'$'\n' -CONFIGURATION+=' directory = ../../test_cache/results'$'\n' -CONFIGURATION+='}'$'\n' -CONFIGURATION+='redis_host=localhost'$'\n' - -echo Here -CONFIGURATION="$CONFIGURATION" PORT=$PORT coverage run --source=. src/plz/controller/main.py - -# exec gunicorn \ -# --bind="0.0.0.0:${PORT}" \ -# --workers=16 \ -# --timeout=2000 \ -# --pythonpath="${PYTHONPATH}" \ -# --capture-output \ -# --log-level="${LOG_LEVEL}" \ -# plz.controller.main:app \ -# -- $@ +if [[ "${RUN_WITH_COVERAGE:-}" ]]; then + pipenv install --dev --system --deploy + COVERAGE_FILE=controller.coverage exec coverage run \ + --source=. src/plz/controller/main.py +else + exec gunicorn \ + --bind="0.0.0.0:${PORT}" \ + --workers=16 \ + --timeout=2000 \ + --pythonpath="${PYTHONPATH}" \ + --capture-output \ + --log-level="${LOG_LEVEL}" \ + plz.controller.main:app \ + -- $@; +fi diff --git a/services/controller/src/plz/controller/main.py b/services/controller/src/plz/controller/main.py index 17bbf192..c9679d33 100644 --- a/services/controller/src/plz/controller/main.py +++ b/services/controller/src/plz/controller/main.py @@ -22,7 +22,7 @@ ResponseGeneratorFunction = Callable[[], ResponseGenerator] config = configuration.load() -port = 5123 +port = config.get_int('port', 8080) def _setup_logging(): diff --git a/test/coverage/coverage_report.sh b/test/coverage/coverage_report.sh new file mode 100755 index 00000000..85a9687a --- /dev/null +++ b/test/coverage/coverage_report.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +if command -v realpath > /dev/null; then + DIR_OF_THIS_SCRIPT="$(cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")" && pwd)" +else + DIR_OF_THIS_SCRIPT="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +fi + + +RESULTS_DIR="${DIR_OF_THIS_SCRIPT}/results" +RCFILE="${DIR_OF_THIS_SCRIPT}/coveragerc" + +# Use a new shell to cd to the plz root path. Otherwise the script would +# depend on pwd, as the paths in the rcfile are constant +( \ + cd "$DIR_OF_THIS_SCRIPT"/../.. && \ + COVERAGE_FILE="${RESULTS_DIR}/total.coverage" coverage combine \ + --rcfile="${RCFILE}" "${RESULTS_DIR}"/*.coverage; + COVERAGE_FILE="${RESULTS_DIR}/total.coverage" coverage report | sed "s:`pwd`::" +) + diff --git a/test/coverage/coveragerc b/test/coverage/coveragerc new file mode 100644 index 00000000..5b8b685e --- /dev/null +++ b/test/coverage/coveragerc @@ -0,0 +1,19 @@ +[paths] +# The containers that run the CLI install plz at the system level +cli_source = + cli/src/plz/cli + /usr/*/plz/cli +# The CLI uses the api submodule of the controller +controller_api_source = + services/controller/src/plz/controller + /usr/*/plz/controller +# The container for the controller has the code inside src/src +controller_container_source = + services/controller/src/ + /src/src + +[report] +# Do not count "pass" lines in abstract methods and properties +exclude_lines = + pragma: no cover + @abstract diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 0c31b13a..c8bd37fb 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -16,6 +16,7 @@ services: context: ../services/controller container_name: $CONTROLLER_CONTAINER environment: + RUN_WITH_COVERAGE: "yes" CONFIGURATION: | # copied from ../start/docker.conf, as we can't mount files inside CircleCI log_level = DEBUG diff --git a/test/run.py b/test/run.py index d13d81f9..7fcc46aa 100644 --- a/test/run.py +++ b/test/run.py @@ -7,13 +7,15 @@ from glob import glob from typing import List, Optional, Set +import time + import test_utils from run_end_to_end_test import run_end_to_end_test from test_utils import CLI_BUILDER_IMAGE, CLI_IMAGE, \ CONTROLLER_CONTAINER, CONTROLLER_HOSTNAME, CONTROLLER_IMAGE, \ CONTROLLER_PORT, CONTROLLER_TESTS_CONTAINER, CONTROLLER_TESTS_IMAGE, \ - DATA_DIRECTORY, PLZ_ROOT_DIRECTORY, PLZ_USER, TEST_DIRECTORY, \ - docker_compose, get_network + COVERAGE_RESULTS_DIRECTORY, DATA_DIRECTORY, PLZ_ROOT_DIRECTORY, PLZ_USER, \ + TEST_DIRECTORY, docker_compose, get_network def start_controller(): diff --git a/test/run_end_to_end_test.py b/test/run_end_to_end_test.py index fc37b3a5..48483cd7 100644 --- a/test/run_end_to_end_test.py +++ b/test/run_end_to_end_test.py @@ -6,11 +6,12 @@ import shutil import signal import sys +import time from tempfile import NamedTemporaryFile, TemporaryDirectory, TemporaryFile import test_utils -from test_utils import CLI_CONTAINER_PREFIX, DATA_DIRECTORY, PLZ_USER, \ - TEST_DIRECTORY, VOLUME_PREFIX, get_network +from test_utils import CLI_CONTAINER_PREFIX, COVERAGE_RESULTS_DIRECTORY, \ + DATA_DIRECTORY, PLZ_USER, TEST_DIRECTORY, VOLUME_PREFIX, get_network def run_end_to_end_test(network: str, plz_host: str, plz_port: int, @@ -26,6 +27,10 @@ def run_end_to_end_test(network: str, plz_host: str, plz_port: int, """ if not os.path.isdir(DATA_DIRECTORY): os.mkdir(DATA_DIRECTORY) + # Keep the coverage results separate because we are going to use them after + # the tests (the data directory gets wiped after all tests have run) + if not os.path.isdir(COVERAGE_RESULTS_DIRECTORY): + os.mkdir(COVERAGE_RESULTS_DIRECTORY) # Make sure the directory has a single slash at the end test_directory = os.path.join( @@ -59,6 +64,7 @@ def run_end_to_end_test(network: str, plz_host: str, plz_port: int, app_directory=test_directory, actual_logs_file=logs_file, output_directory_name=output_directory_name) + end = datetime.datetime.now() test_utils.print_info(f'Time taken: {end-start}') @@ -118,6 +124,15 @@ def run_end_to_end_test(network: str, plz_host: str, plz_port: int, return True +# Replace non-common characters by hyphens +def hiphenize(st: str): + return re.sub(r'[^0-9a-zA-Z_]', '-', st) + + +def coverage_file_name(test_name: str): + return hiphenize(test_name) + '.coverage' + + def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, app_directory: str, actual_logs_file: TemporaryFile, output_directory_name: str) -> int: @@ -125,10 +140,9 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, :return: exit code of the test """ output_directory_name = os.path.abspath(output_directory_name) - project_name = re.sub(r'[^0-9a-zA-Z_]', '-', test_name) + project_name = hiphenize(test_name) test_config_file = f'{app_directory}/test.config.json' - suffix = re.sub(r'[^0-9a-zA-Z_]', '-', - '-'.join(os.path.split(test_name)[-2:])) + suffix = hiphenize('-'.join(os.path.split(test_name)[-2:])) cli_container = f'{CLI_CONTAINER_PREFIX}_{suffix}' volume = f'{VOLUME_PREFIX}{suffix}' @@ -157,16 +171,35 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, # Start the CLI process. test_utils.execute_command([ - 'docker', 'container', 'run', f'--name={cli_container}', f'--detach', - f'--network={network}', f'--env=PLZ_HOST={plz_host}', - f'--env=PLZ_PORT={plz_port}', f'--env=PLZ_USER={PLZ_USER}', + 'docker', + 'container', + 'run', + f'--name={cli_container}', + f'--detach', + f'--network={network}', + f'--env=PLZ_HOST={plz_host}', + f'--env=PLZ_PORT={plz_port}', + f'--env=PLZ_USER={PLZ_USER}', f'--env=PLZ_PROJECT={project_name}', f'--env=PLZ_INSTANCE_MARKET_TYPE=spot', f'--env=PLZ_MAX_BID_PRICE_IN_DOLLARS_PER_HOUR=0.5', f'--env=PLZ_INSTANCE_MAX_UPTIME_IN_MINUTES=0', - f'--env=PLZ_QUIET_BUILD=true', f'--workdir=/data/app', - f'--volume={volume}:/data', test_utils.CLI_IMAGE, 'run', - '--output=/data/output', *test_args + f'--env=PLZ_QUIET_BUILD=true', + f'--env=COVERAGE_FILE=/data/coverage', + f'--workdir=/data/app', + f'--volume={volume}:/data', + f'--entrypoint', + 'coverage', + test_utils.CLI_IMAGE, + # This run is for `coverage` + 'run', + '-m', + '--source=plz', + 'plz.cli.main', + # This run is for `plz` + 'run', + '--output=/data/output', + *test_args ]) # Capture the logs @@ -189,6 +222,11 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, hide_output=True) exit_status = int(b''.join(stdout_holder)) + test_utils.execute_command([ + 'docker', 'cp', f'{cli_container}:/data/coverage', + os.path.join(COVERAGE_RESULTS_DIRECTORY, coverage_file_name(test_name)) + ]) + test_utils.execute_command(['docker', 'container', 'rm', cli_container], hide_output=True) diff --git a/test/test_utils.py b/test/test_utils.py index 740dc8b7..ad24aca0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -7,6 +7,8 @@ import traceback from typing import Any, ContextManager, List, Optional, Tuple +import time + PROJECT_NAME = 'plztest' VOLUME_PREFIX = f'{PROJECT_NAME}_data_' CLI_BUILDER_IMAGE = f'{PROJECT_NAME}/cli-builder' @@ -29,6 +31,8 @@ TEST_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) PLZ_ROOT_DIRECTORY = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +COVERAGE_RESULTS_DIRECTORY = os.path.join(TEST_DIRECTORY, 'coverage', + 'results') DATA_DIRECTORY = f'{PLZ_ROOT_DIRECTORY}/test_cache/' REDIS_DATA_DIRECTORY = f'{DATA_DIRECTORY}/redis_data/' @@ -255,6 +259,16 @@ def stop_all_clis(): def stop_controller(): + if 'PLZ_HOST' not in os.environ: + # Unless we interrupt the server before stopping, coverage won't write + # the report + execute_command( + ['docker', 'kill', '--signal=INT', CONTROLLER_CONTAINER]) + execute_command([ + 'docker', 'container', 'cp', + f'{CONTROLLER_CONTAINER}:/src/controller.coverage', + os.path.join(COVERAGE_RESULTS_DIRECTORY, 'controller.coverage') + ]) docker_compose('stop') docker_compose('logs') docker_compose('down', '--volumes') From d6d64db249f1c6b6b86a915ceea7744ea8fd8654 Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Thu, 25 Apr 2019 15:19:19 +0100 Subject: [PATCH 3/9] Coverage: remove unused imports --- test/run.py | 4 +--- test/run_end_to_end_test.py | 1 - test/test_utils.py | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/test/run.py b/test/run.py index 7fcc46aa..e4054c39 100644 --- a/test/run.py +++ b/test/run.py @@ -7,14 +7,12 @@ from glob import glob from typing import List, Optional, Set -import time - import test_utils from run_end_to_end_test import run_end_to_end_test from test_utils import CLI_BUILDER_IMAGE, CLI_IMAGE, \ CONTROLLER_CONTAINER, CONTROLLER_HOSTNAME, CONTROLLER_IMAGE, \ CONTROLLER_PORT, CONTROLLER_TESTS_CONTAINER, CONTROLLER_TESTS_IMAGE, \ - COVERAGE_RESULTS_DIRECTORY, DATA_DIRECTORY, PLZ_ROOT_DIRECTORY, PLZ_USER, \ + DATA_DIRECTORY, PLZ_ROOT_DIRECTORY, PLZ_USER, \ TEST_DIRECTORY, docker_compose, get_network diff --git a/test/run_end_to_end_test.py b/test/run_end_to_end_test.py index 48483cd7..8446007c 100644 --- a/test/run_end_to_end_test.py +++ b/test/run_end_to_end_test.py @@ -6,7 +6,6 @@ import shutil import signal import sys -import time from tempfile import NamedTemporaryFile, TemporaryDirectory, TemporaryFile import test_utils diff --git a/test/test_utils.py b/test/test_utils.py index ad24aca0..99f77bd0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -7,8 +7,6 @@ import traceback from typing import Any, ContextManager, List, Optional, Tuple -import time - PROJECT_NAME = 'plztest' VOLUME_PREFIX = f'{PROJECT_NAME}_data_' CLI_BUILDER_IMAGE = f'{PROJECT_NAME}/cli-builder' From 3900456cb47a03abfabfe7433027c3f206bca9da Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Thu, 25 Apr 2019 15:33:40 +0100 Subject: [PATCH 4/9] Coverage: give the controller a minute before health checks --- services/controller/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/controller/Dockerfile b/services/controller/Dockerfile index 8ae79be4..de524cf6 100644 --- a/services/controller/Dockerfile +++ b/services/controller/Dockerfile @@ -26,6 +26,6 @@ ARG BUILD_TIMESTAMP=0 RUN echo -n $BUILD_TIMESTAMP > ./src/plz/controller/BUILD_TIMESTAMP -HEALTHCHECK --interval=5s CMD curl -f http://localhost/ +HEALTHCHECK --interval=5s --start-period=60s CMD curl -f http://localhost/ ENTRYPOINT ["/tini", "--", "./run"] From 1f985f83e368aa800053a74c0dcae9c8cadbd85e Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Thu, 25 Apr 2019 16:48:06 +0100 Subject: [PATCH 5/9] Coverage: fix logging for the case the server is called as a script (Instead of gunicorn) --- services/controller/src/plz/controller/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/controller/src/plz/controller/main.py b/services/controller/src/plz/controller/main.py index c9679d33..6ee21e24 100644 --- a/services/controller/src/plz/controller/main.py +++ b/services/controller/src/plz/controller/main.py @@ -8,6 +8,7 @@ import requests from flask import Flask, Response, abort, jsonify, request, stream_with_context +import plz.controller from plz.controller import configuration from plz.controller.api.exceptions import AbortedExecutionException, \ InstanceNotRunningException, JSONResponseException, \ @@ -40,8 +41,7 @@ def _setup_logging(): print(f'Setting log level to: {log_level}', file=sys.stderr, flush=True) - controller_logger = logging.getLogger('.'.join( - __name__.split('.')[:-1])) + controller_logger = logging.getLogger(plz.controller.__name__) controller_logger.setLevel(log_level) From ad67b123cdf647311fb5b709dd8c10bdb5ea859f Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Thu, 25 Apr 2019 17:16:10 +0100 Subject: [PATCH 6/9] Coverage: fix report of abstract methods --- test/coverage/coverage_report.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/coverage/coverage_report.sh b/test/coverage/coverage_report.sh index 85a9687a..12e1deb9 100755 --- a/test/coverage/coverage_report.sh +++ b/test/coverage/coverage_report.sh @@ -16,6 +16,7 @@ RCFILE="${DIR_OF_THIS_SCRIPT}/coveragerc" cd "$DIR_OF_THIS_SCRIPT"/../.. && \ COVERAGE_FILE="${RESULTS_DIR}/total.coverage" coverage combine \ --rcfile="${RCFILE}" "${RESULTS_DIR}"/*.coverage; - COVERAGE_FILE="${RESULTS_DIR}/total.coverage" coverage report | sed "s:`pwd`::" + COVERAGE_FILE="${RESULTS_DIR}/total.coverage" coverage report \ + --rcfile="${RCFILE}" | sed "s:`pwd`::" ) From f85e6583e0be270f3eeb631625cfa562a99d8305 Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Fri, 26 Apr 2019 10:41:14 +0100 Subject: [PATCH 7/9] Coverage: during `make check` run with and without coverage With coverage we run with the flask dev server, which keeps state across requests. Errors causes by relying on the state are more likely to be caught by gunicorn --- Makefile | 1 + test/docker-compose.yml | 2 +- test/run.py | 7 +++++++ test/run_end_to_end_test.py | 33 +++++++++++++++++++++------------ test/test_utils.py | 6 +++++- 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 4e2be7bf..9060dae8 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ check: environment $(MAKE) -C cli check $(MAKE) -C services/controller check BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) python3 test/run.py + RUN_WITH_COVERAGE=yes BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) python3 test/run.py pipenv run test/coverage/coverage_report.sh terraform fmt -check diff --git a/test/docker-compose.yml b/test/docker-compose.yml index c8bd37fb..141e0bb8 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -16,7 +16,7 @@ services: context: ../services/controller container_name: $CONTROLLER_CONTAINER environment: - RUN_WITH_COVERAGE: "yes" + RUN_WITH_COVERAGE: ${RUN_WITH_COVERAGE:-} CONFIGURATION: | # copied from ../start/docker.conf, as we can't mount files inside CircleCI log_level = DEBUG diff --git a/test/run.py b/test/run.py index e4054c39..e7831030 100644 --- a/test/run.py +++ b/test/run.py @@ -171,6 +171,13 @@ def main(): options = parser.parse_args(sys.argv[1:]) + test_utils.print_info('=' * 70) + if test_utils.running_with_coverage(): + test_utils.print_info('Running with flask dev server, with coverage') + else: + test_utils.print_info('Running with gunicorn, without coverage') + test_utils.print_info('=' * 70) + if options.end_to_end_only and options.controller_tests_only: raise ValueError('Options end_to_end_only and controller_tests_only ' 'can\'t be specified simultaneously') diff --git a/test/run_end_to_end_test.py b/test/run_end_to_end_test.py index 8446007c..ee1fc38c 100644 --- a/test/run_end_to_end_test.py +++ b/test/run_end_to_end_test.py @@ -168,6 +168,20 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, 'docker:stable-git', 'git', 'init', '--quiet', '/data/app' ]) + if test_utils.running_with_coverage(): + docker_run_args = [ + f'--entrypoint', + 'coverage', + test_utils.CLI_IMAGE, + # This run is for `coverage` + 'run', + '-m', + '--source=plz', + 'plz.cli.main' + ] + else: + docker_run_args = [test_utils.CLI_IMAGE] + # Start the CLI process. test_utils.execute_command([ 'docker', @@ -187,14 +201,7 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, f'--env=COVERAGE_FILE=/data/coverage', f'--workdir=/data/app', f'--volume={volume}:/data', - f'--entrypoint', - 'coverage', - test_utils.CLI_IMAGE, - # This run is for `coverage` - 'run', - '-m', - '--source=plz', - 'plz.cli.main', + *docker_run_args, # This run is for `plz` 'run', '--output=/data/output', @@ -221,10 +228,12 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, hide_output=True) exit_status = int(b''.join(stdout_holder)) - test_utils.execute_command([ - 'docker', 'cp', f'{cli_container}:/data/coverage', - os.path.join(COVERAGE_RESULTS_DIRECTORY, coverage_file_name(test_name)) - ]) + if test_utils.running_with_coverage(): + test_utils.execute_command([ + 'docker', 'cp', f'{cli_container}:/data/coverage', + os.path.join(COVERAGE_RESULTS_DIRECTORY, + coverage_file_name(test_name)) + ]) test_utils.execute_command(['docker', 'container', 'rm', cli_container], hide_output=True) diff --git a/test/test_utils.py b/test/test_utils.py index 99f77bd0..6700773f 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -257,7 +257,7 @@ def stop_all_clis(): def stop_controller(): - if 'PLZ_HOST' not in os.environ: + if running_with_coverage(): # Unless we interrupt the server before stopping, coverage won't write # the report execute_command( @@ -299,3 +299,7 @@ def docker_compose(*args): f'--file={os.path.join(TEST_DIRECTORY, "docker-compose.yml")}', *args ], env=env) + + +def running_with_coverage(): + return os.environ.get('RUN_WITH_COVERAGE', '') != '' From b52e9c221c87a08c4fac35d88b398e55d38f0171 Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Fri, 26 Apr 2019 11:05:49 +0100 Subject: [PATCH 8/9] Coverage: do not print logs of docker-compose when running with coverage --- test/test_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 6700773f..f12bb51a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -268,7 +268,11 @@ def stop_controller(): os.path.join(COVERAGE_RESULTS_DIRECTORY, 'controller.coverage') ]) docker_compose('stop') - docker_compose('logs') + # When we are running with coverage don't print the logs. They will be + # printed in the run without coverage. If we print both times, the whole + # output of the tests is too big for CircleCI + if not running_with_coverage(): + docker_compose('logs') docker_compose('down', '--volumes') From 8b60cd2f67127ea83e6a61ba103dda7e8f06a36d Mon Sep 17 00:00:00 2001 From: Sergio Giro Date: Tue, 30 Apr 2019 12:05:50 +0100 Subject: [PATCH 9/9] Coverage: fix merge of Pipfile.lock --- services/controller/Pipfile.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/controller/Pipfile.lock b/services/controller/Pipfile.lock index 04e7c0a7..7c92f424 100644 --- a/services/controller/Pipfile.lock +++ b/services/controller/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "12e843ac392d34367a32f8d6d38930c7b4e37080667c2e3ce325798d1ee2a4ff" + "sha256": "d1f18359aa6c33178f8274d8f083be57ff8d69e18caba1410993168ade0141da" }, "pipfile-spec": 6, "requires": {}, @@ -16,18 +16,18 @@ "default": { "boto3": { "hashes": [ - "sha256:2292bb7cb9de3e5a2bda9a78a4ac890961b80fc453add0ac87eca6f726b0cd6d", - "sha256:dc94237a8e9c104c86d77ec9b3ecebd864680b5a4fbd22b28722376111dbec47" + "sha256:679d8084ad40d18908a97c785d614fed554a424924d4ab30e464c16bfe95722b", + "sha256:d5ccb985caf4ea522f2fbfe4fbf270cd1e2c0c6d46ea7d13e9cda6bb6c36deb6" ], "index": "pypi", - "version": "==1.9.136" + "version": "==1.9.138" }, "botocore": { "hashes": [ - "sha256:b12e0efe214e35d0217e13e8e57677ae70b63c8bd01c960dcda92d05dd55ac6b", - "sha256:c24b9c32fdf2643a6cdeca55a43dc57e5b7b92a437d6e092807c9dc3f98c0c78" + "sha256:73bf439ba6d97606f8acbe4e037cc7a6e7a2b83f080b472c37c22d810c7dd8a8", + "sha256:ff4171f850cfb221b553f32948b93ea0a8d82e636fe121ff08945f4581c21ad9" ], - "version": "==1.12.136" + "version": "==1.12.138" }, "certifi": { "hashes": [