From 53169331f796c39b82001812788ccb55c80371b0 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Fri, 24 Jul 2020 02:22:05 +0530 Subject: [PATCH 01/29] added unittests for utils.deep_update --- tests/__init__.py | 0 tests/test_utils.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/test_utils.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..9f4ac5652 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,31 @@ +import sys + +import unittest +from mock import MagicMock + + +class TestDeepUpdate(unittest.TestCase): + + def setUp(self): + self.u = { + "first": {"a": "A"}, + "second": "B" + } + self.d = { + "first": {"a": "B"}, + "second": "C" + } + sys.modules['dbs'] = MagicMock() + sys.modules['dbs.apis'] = MagicMock() + sys.modules['dbs.apis.dbsClient'] = MagicMock() + from WmAgentScripts.utils import deep_update + self.deep_update = deep_update + + def test_deep_update(self): + + self.assertDictEqual(self.d, self.deep_update(self.d, self.u)) + + + +if __name__ == '__main__': + unittest.main() From 7bf701eb2dcd53e0188f0edad929e39fdc3b5a5b Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sat, 25 Jul 2020 08:49:33 +0530 Subject: [PATCH 02/29] add unittests for utils.UnifiedConfiguration --- tests/test_utils.py | 35 ++++++++++++++++++++++++++++++++++- utils.py | 5 ++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 9f4ac5652..31b4d5c98 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,12 +1,15 @@ import sys import unittest -from mock import MagicMock +import mock +from mock import MagicMock, patch, mock_open +import json class TestDeepUpdate(unittest.TestCase): def setUp(self): + self.u = { "first": {"a": "A"}, "second": "B" @@ -26,6 +29,36 @@ def test_deep_update(self): self.assertDictEqual(self.d, self.deep_update(self.d, self.u)) +class TestUnifiedConfiguration(unittest.TestCase): + + def setUp(self): + self.test_json = json.dumps({ + "email": { + "value": ["test@cern.ch", "test@testmail.com"], + "description": "The list of people that get the emails notifications" + } + }) + + def test_get_json(self): + + from WmAgentScripts.utils import unifiedConfiguration, open_json_file + mock_open_json = mock_open(read_data=self.test_json) + with patch('__builtin__.open', mock_open_json): + result = open_json_file('filename') + uc = unifiedConfiguration(configFile='fake_file_path') + self.assertEqual( + uc.get("email"), ["test@cern.ch", "test@testmail.com"]) + + with self.assertRaises(SystemExit): + self.assertEqual( + uc.get("cernmails"), [ + "test@cern.ch", "test@testmail.com"]) + + def test_get_mongodb(self): + + # TODO: mock db and write tests + pass + if __name__ == '__main__': unittest.main() diff --git a/utils.py b/utils.py index 375aa0042..4d74d22fa 100755 --- a/utils.py +++ b/utils.py @@ -36,6 +36,9 @@ def mongo_client(): return pymongo.MongoClient('mongodb://%s/?ssl=true' % mongo_db_url, ssl_cert_reqs=ssl.CERT_NONE) +def open_json_file(filename): + with open(filename) as f: + return json.loads(f.read()) class unifiedConfiguration: def __init__(self, configFile='unifiedConfiguration.json'): @@ -45,7 +48,7 @@ def __init__(self, configFile='unifiedConfiguration.json'): self.configs = self.configFile else: try: - self.configs = json.loads(open(self.configFile).read()) + self.configs = open_json_file(self.configFile) except Exception as ex: print("Could not read configuration file: %s\nException: %s" % (self.configFile, str(ex))) From 1458d3fb32164d92748c68a17eff704d935ac561 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 26 Jul 2020 10:58:57 +0530 Subject: [PATCH 03/29] add unittests for utils.es_header --- tests/test_utils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 31b4d5c98..1f9f07ff0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -55,10 +55,23 @@ def test_get_json(self): "test@cern.ch", "test@testmail.com"]) def test_get_mongodb(self): - + # TODO: mock db and write tests pass +class TestEsHeader(unittest.TestCase): + + def test_es_header(self): + from WmAgentScripts.utils import es_header + mock_date = mock_open(read_data=u"fake_entrypointname:fake_password") + with patch('__builtin__.open', mock_date): + result = es_header() + fake_result = { + 'Content-Type': 'application/json', + 'Authorization': 'Basic ZmFrZV9lbnRyeXBvaW50bmFtZTpmYWtlX3Bhc3N3b3Jk'} + self.assertEqual(result, fake_result) + + if __name__ == '__main__': unittest.main() From f79fc554c73f9602ca2d83e30e3c2f1dc7f80f0c Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 26 Jul 2020 11:10:03 +0530 Subject: [PATCH 04/29] add unittests for utils.url_encode_params --- tests/test_utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1f9f07ff0..3e210b7ee 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -70,7 +70,16 @@ def test_es_header(self): fake_result = { 'Content-Type': 'application/json', 'Authorization': 'Basic ZmFrZV9lbnRyeXBvaW50bmFtZTpmYWtlX3Bhc3N3b3Jk'} - self.assertEqual(result, fake_result) + self.assertDictEqual(result, fake_result) + + +class TestUrlEncodeParams(unittest.TestCase): + + def test_url_encode_params(self): + from WmAgentScripts.utils import url_encode_params + params = {"query1": "test1", "query2": ["test2", "test3"]} + result = url_encode_params(params) + self.assertEqual(result, "query2=test2&query2=test3&query1=test1") if __name__ == '__main__': From 17c0e53cb4c3f624ac7cc037db78a9dba96acb83 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Mon, 27 Jul 2020 13:33:28 +0530 Subject: [PATCH 05/29] adds test for utils.GET --- tests/test_utils.py | 89 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 3e210b7ee..a1556246a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,9 @@ import sys import unittest +import json import mock from mock import MagicMock, patch, mock_open -import json class TestDeepUpdate(unittest.TestCase): @@ -44,7 +44,7 @@ def test_get_json(self): from WmAgentScripts.utils import unifiedConfiguration, open_json_file mock_open_json = mock_open(read_data=self.test_json) with patch('__builtin__.open', mock_open_json): - result = open_json_file('filename') + open_json_file('filename') uc = unifiedConfiguration(configFile='fake_file_path') self.assertEqual( uc.get("email"), ["test@cern.ch", "test@testmail.com"]) @@ -82,5 +82,90 @@ def test_url_encode_params(self): self.assertEqual(result, "query2=test2&query2=test3&query1=test1") +class TestGET(unittest.TestCase): + + def test_get(self): + + class MockResponse: + def __init__(self, *args, **kwargs): + self.response = None, 404 + + def request(self, *args, **kwargs): + if args[1] == 'test.json': + self.response = {"key1": "value1"}, 200 + elif args[1] == 'anothertest.json': + self.response = {"key2": "value2"}, 200 + else: + self.response = None, 404 + + def getresponse(self): + return self.response + + from WmAgentScripts.utils import GET + with patch('WmAgentScripts.utils.make_x509_conn', MockResponse): + response = GET( + url='http://someurl.com/', + there='test.json', + l=False) + self.assertDictEqual({"key1": "value1"}, response[0]) + self.assertEqual(200, response[1]) + + response = GET( + url='http://someurl.com/', + there='anothertest.json', + l=False) + self.assertDictEqual({"key2": "value2"}, response[0]) + self.assertEqual(200, response[1]) + + response = GET(url=None, there=None, l=False) + self.assertEqual((None, 404), response) + + def test_get_json_object(self): + + from StringIO import StringIO + + class ContextualStringIO(StringIO): + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + return False + + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None, 404 + + def request(self, *args, **kwargs): + if args[1] == 'test.json': + self.response = {"key1": "value1"}, 200 + elif args[1] == 'anothertest.json': + self.response = {"key2": "value2"}, 200 + else: + self.response = None, 404 + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import GET + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = GET( + url='http://someurl.com/', + there='test.json', + l=True) + self.assertDictEqual({"key1": "value1"}, response[0]) + self.assertEqual(200, response[1]) + + response = GET( + url='http://someurl.com/', + there='anothertest.json', + l=True) + self.assertDictEqual({"key2": "value2"}, response[0]) + self.assertEqual(200, response[1]) + + response = GET(url=None, there=None, l=True) + self.assertEqual([None, 404], response) + + if __name__ == '__main__': unittest.main() From ae38d3e00ca5db82de28679d7f90bb810278cd62 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Mon, 27 Jul 2020 13:36:17 +0530 Subject: [PATCH 06/29] add github action for running tests --- .github/workflows/run_test.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/run_test.yml diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml new file mode 100644 index 000000000..817af5e80 --- /dev/null +++ b/.github/workflows/run_test.yml @@ -0,0 +1,21 @@ +name: Unit test + +on: [push, pull_request] + +jobs: + lint-python: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2.2.0 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 2.7 + - name: Install Pytest and mock + run: | + python2 -m pip install pytest mock + - name: Run pytest + run: | + python2 -m pytest -v tests + \ No newline at end of file From c0cb8661d25ff68d7a56beeabb8dedb3e859f286 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Tue, 28 Jul 2020 04:55:41 +0530 Subject: [PATCH 07/29] add unittests for utils.getSubscriptions --- tests/test_utils.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index a1556246a..31d002abd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -167,5 +167,36 @@ def getresponse(self): self.assertEqual([None, 404], response) +class TestGetSubscription(unittest.TestCase): + + def testGetSubscription(self): + from StringIO import StringIO + + class ContextualStringIO(StringIO): + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + return False + + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None, 404 + + def request(self, *args, **kwargs): + self.response = {"phedex": "value1"} + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import getSubscriptions + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = getSubscriptions( + url='http://someurl.com/', + dataset="somedataset") + self.assertEqual(response, "value1") + + if __name__ == '__main__': unittest.main() From 69ebbf5c95688fe1e04911f2d922171712851f7f Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Wed, 29 Jul 2020 16:51:29 +0530 Subject: [PATCH 08/29] add unittests for utils.listRequests --- tests/test_utils.py | 68 ++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 31d002abd..0196b4767 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,17 @@ import mock from mock import MagicMock, patch, mock_open +from StringIO import StringIO + + +class ContextualStringIO(StringIO): + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + return False + class TestDeepUpdate(unittest.TestCase): @@ -122,16 +133,6 @@ def getresponse(self): def test_get_json_object(self): - from StringIO import StringIO - - class ContextualStringIO(StringIO): - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - return False - class MockResponseStringIo: def __init__(self, *args, **kwargs): self.response = None, 404 @@ -170,15 +171,6 @@ def getresponse(self): class TestGetSubscription(unittest.TestCase): def testGetSubscription(self): - from StringIO import StringIO - - class ContextualStringIO(StringIO): - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - return False class MockResponseStringIo: def __init__(self, *args, **kwargs): @@ -198,5 +190,43 @@ def getresponse(self): self.assertEqual(response, "value1") +class TestListRequests(unittest.TestCase): + + def testListRequests(self): + + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None, 404 + + def request(self, *args, **kwargs): + self.response = {"phedex": { + "request": [{ + "node": [ + {"name": "someSite"}, + {"name": "someSite1"} + ], + "id": "someId" + }]} + } + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import listRequests + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = listRequests( + url='http://someurl.com/', + dataset="somedataset", + site=None) + self.assertDictEqual( + response, { + 'someSite1': ['someId'], 'someSite': ['someId']}) + response = listRequests( + url='http://someurl.com/', + dataset="somedataset", + site='someSite') + self.assertDictEqual(response, {'someSite': ['someId']}) + + if __name__ == '__main__': unittest.main() From 26138d8373579d7d741fb1578a8c1f166c293cb0 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Wed, 29 Jul 2020 18:17:41 +0530 Subject: [PATCH 09/29] add unittests for utils.listCustodial --- tests/test_utils.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0196b4767..480588c87 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -228,5 +228,39 @@ def getresponse(self): self.assertDictEqual(response, {'someSite': ['someId']}) +class TestListRequests(unittest.TestCase): + + def testListCustodial(self): + + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None, 404 + + def request(self, *args, **kwargs): + self.response = {"phedex": { + "request": [{ + "node": [ + {"name": "someSite"}, + {"name": "someSite1"} + ], + "id": "someId", + "type": "xfer" + }]} + } + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import listCustodial + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = listCustodial( + url='http://someurl.com/', + ) + print response + self.assertDictEqual( + response, { + 'someSite1': ['someId'], 'someSite': ['someId']}) + + if __name__ == '__main__': unittest.main() From de8c09aa87179b1acf8edddb4d728518edc95e1c Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Thu, 30 Jul 2020 08:58:20 +0530 Subject: [PATCH 10/29] add unittests for utils.listSubscriptions --- tests/test_utils.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 480588c87..0db61aa38 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -262,5 +262,57 @@ def getresponse(self): 'someSite1': ['someId'], 'someSite': ['someId']}) +class TestListSubscriptions(unittest.TestCase): + + def testListSubscriptions(self): + + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None, 404 + + def request(self, *args, **kwargs): + self.response = {"phedex": { + "request": [{ + "node": [{ + "name": "someSite", + "decision": "approved", + "time_decided": 1300 + }, + { + "name": "someSite1", + "decision": "pending", + "time_decided": 1400 + }, + { + "name": "someSiteMSS", + "decision": "pending", + "time_decided": 1400 + }, + ], + "id": "someId", + "type": "xfer" + }]} + } + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import listSubscriptions + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = listSubscriptions( + url='http://someurl.com/', + dataset='somedataset' + ) + self.assertDictEqual( + response, {'someSite': ('someId', True), 'someSite1': ('someId', False)}) + + response = listSubscriptions( + url='http://someurl.com/', + dataset='somedataset', + within_sites=['someSite'] + ) + self.assertDictEqual( + response, {'someSite': ('someId', True)}) + if __name__ == '__main__': unittest.main() From f36e6b017908644c666eeceaa5825740b0c740fd Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Fri, 31 Jul 2020 16:28:10 +0530 Subject: [PATCH 11/29] add unittests for utils.pass_to_dynamo --- tests/test_utils.py | 48 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0db61aa38..2617cdd08 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -196,7 +196,7 @@ def testListRequests(self): class MockResponseStringIo: def __init__(self, *args, **kwargs): - self.response = None, 404 + self.response = None def request(self, *args, **kwargs): self.response = {"phedex": { @@ -234,7 +234,7 @@ def testListCustodial(self): class MockResponseStringIo: def __init__(self, *args, **kwargs): - self.response = None, 404 + self.response = None def request(self, *args, **kwargs): self.response = {"phedex": { @@ -268,12 +268,13 @@ def testListSubscriptions(self): class MockResponseStringIo: def __init__(self, *args, **kwargs): - self.response = None, 404 + self.response = None def request(self, *args, **kwargs): self.response = {"phedex": { "request": [{ - "node": [{ + "node": [ + { "name": "someSite", "decision": "approved", "time_decided": 1300 @@ -304,7 +305,11 @@ def getresponse(self): dataset='somedataset' ) self.assertDictEqual( - response, {'someSite': ('someId', True), 'someSite1': ('someId', False)}) + response, { + 'someSite': ( + 'someId', True), + 'someSite1': ( + 'someId', False)}) response = listSubscriptions( url='http://someurl.com/', @@ -314,5 +319,38 @@ def getresponse(self): self.assertDictEqual( response, {'someSite': ('someId', True)}) + +class TestPass_to_dynamo(unittest.TestCase): + + def test_pass_to_dynamo(self): + + class MockResponseStringIo: + response = {"result": "OK"} + + def __init__(self, *args, **kwargs): + pass + + def request(self, *args, **kwargs): + pass + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import pass_to_dynamo + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = pass_to_dynamo( + items=["items1", "item2"], + N=10 + ) + self.assertTrue(response) + MockResponseStringIo.response = {"result": "Not OK"} + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = pass_to_dynamo( + items=["items1", "item2"], + N=10 + ) + self.assertFalse(response) + + if __name__ == '__main__': unittest.main() From 5efcfef30646fee4731f84d375afdbaa505fce75 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sat, 1 Aug 2020 04:20:42 +0530 Subject: [PATCH 12/29] add unittests for utils.checkDownTime --- tests/test_utils.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 2617cdd08..79d60050b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -352,5 +352,41 @@ def getresponse(self): self.assertFalse(response) +class TesCheckDownTime(unittest.TestCase): + + def testCheckDownTime(self): + + class MockResponseStringIo(StringIO): + status = 503 + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + return False + + class MockResponse: + def __init__(self, *args, **kwargs): + self.response = None + + def request(self, *args, **kwargs): + self.response = {} + + def getresponse(self): + return MockResponseStringIo(json.dumps(self.response)) + + from WmAgentScripts.utils import checkDownTime + + with patch('WmAgentScripts.utils.make_x509_conn', MockResponse): + response = checkDownTime() + self.assertTrue(response) + + MockResponseStringIo.status = 404 + with patch('WmAgentScripts.utils.make_x509_conn', MockResponse): + response = checkDownTime() + self.assertFalse(response) + + if __name__ == '__main__': unittest.main() From 230c78802526f61587e3c43018f983fa4d040e16 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sat, 1 Aug 2020 08:56:33 +0530 Subject: [PATCH 13/29] add unittests for utils.is_json --- tests/test_utils.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 79d60050b..baa4844b9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -388,5 +388,21 @@ def getresponse(self): self.assertFalse(response) +class TestIsJson(unittest.TestCase): + + def test_is_json(self): + from WmAgentScripts.utils import is_json + + test_json = { + "first": {"a": "A"}, + "second": "B" + } + + self.assertTrue(is_json(json.dumps(test_json))) + + with self.assertRaises(TypeError): + self.assertFalse(is_json(test_json)) + + if __name__ == '__main__': unittest.main() From 8745af1cab5afb8f13084dd6a8cc6be39d697622 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 2 Aug 2020 07:19:55 +0530 Subject: [PATCH 14/29] add unittests for utils.read_file --- tests/test_utils.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index baa4844b9..a222671db 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -404,5 +404,30 @@ def test_is_json(self): self.assertFalse(is_json(test_json)) +class TestReadFile(unittest.TestCase): + + def test_read_file(self): + from WmAgentScripts.utils import read_file + test_json = { + "first": {"a": "A"}, + "second": "B" + } + test_data = "test data" + + with patch('WmAgentScripts.utils.sendLog', return_value=None): + mock_open_json = mock_open(read_data=json.dumps(test_json)) + with patch('__builtin__.open', mock_open_json): + content = json.loads(read_file('filename.json')) + self.assertDictEqual(content, test_json) + + mock_open_file = mock_open(read_data=test_data) + with patch('__builtin__.open', mock_open_file): + content = read_file('filename') + self.assertEqual(content, test_data) + + textcontent = read_file('filename.json') + self.assertEqual(textcontent, '{}') + + if __name__ == '__main__': unittest.main() From 8d5b1ff58c2415b974246ed2e48b049501297fe1 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 2 Aug 2020 09:48:01 +0530 Subject: [PATCH 15/29] add unittests for utils.getWMStats --- tests/test_utils.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index a222671db..370d76c87 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -429,5 +429,24 @@ def test_read_file(self): self.assertEqual(textcontent, '{}') +class TestGetWMStats(unittest.TestCase): + + def test_getWMStats(self): + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None, 404 + + def request(self, *args, **kwargs): + self.response = {"result": [200]} + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import getWMStats + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = getWMStats(url='http://someurl.com/') + self.assertEqual(response, 200) + + if __name__ == '__main__': unittest.main() From c63cb437fa7fa822eba36e362c438351c66c68c9 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 2 Aug 2020 10:50:44 +0530 Subject: [PATCH 16/29] add unittests for utils.checkTransferApproval --- tests/test_utils.py | 45 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 370d76c87..4bcf8d86f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,6 +7,10 @@ from StringIO import StringIO +sys.modules['dbs'] = MagicMock() +sys.modules['dbs.apis'] = MagicMock() +sys.modules['dbs.apis.dbsClient'] = MagicMock() + class ContextualStringIO(StringIO): def __enter__(self): @@ -29,9 +33,6 @@ def setUp(self): "first": {"a": "B"}, "second": "C" } - sys.modules['dbs'] = MagicMock() - sys.modules['dbs.apis'] = MagicMock() - sys.modules['dbs.apis.dbsClient'] = MagicMock() from WmAgentScripts.utils import deep_update self.deep_update = deep_update @@ -448,5 +449,43 @@ def getresponse(self): self.assertEqual(response, 200) +class TestCheckTransferApproval(unittest.TestCase): + + def test_checkTransferApproval(self): + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None + + def request(self, *args, **kwargs): + self.response = {"phedex": { + "request": [{ + "node": [ + { + "name": "someSite", + "decision": "approved", + }, + { + "name": "someSite1", + "decision": "pending", + }, + ], + }]} + } + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import checkTransferApproval + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = checkTransferApproval( + url='http://someurl.com/', + phedexid='someid' + ) + self.assertDictEqual( + response, { + 'someSite': True, + 'someSite1': False}) + + if __name__ == '__main__': unittest.main() From eea926f223cc7a5629ca777f9a0b3856db5a0c2c Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 2 Aug 2020 11:07:23 +0530 Subject: [PATCH 17/29] add unittests for utils.getNodesId --- tests/test_utils.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4bcf8d86f..bbe197b01 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -487,5 +487,41 @@ def getresponse(self): 'someSite1': False}) +class TestGetNodesId(unittest.TestCase): + + def test_getNodesId(self): + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None + + def request(self, *args, **kwargs): + self.response = {"phedex": { + "node": [ + { + "name": "someSite", + "id": "someId", + }, + { + "name": "someSite1", + "id": "someId1", + }, + ], + "type": "xfer" + }} + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import getNodesId + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = getNodesId( + url='http://someurl.com/' + ) + self.assertDictEqual( + response, { + 'someSite': 'someId', + 'someSite1': 'someId1'}) + + if __name__ == '__main__': unittest.main() From 141c909557c0d74c72da2fe793141b49e9e99de4 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 2 Aug 2020 16:32:51 +0530 Subject: [PATCH 18/29] add unittests for utils.getDatasetFileLocations --- tests/test_utils.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index bbe197b01..4c27ca6d2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -523,5 +523,48 @@ def getresponse(self): 'someSite1': 'someId1'}) +class TestGetDatasetFileLocations(unittest.TestCase): + def test_getDatasetFileLocations(self): + class MockResponseStringIo: + def __init__(self, *args, **kwargs): + self.response = None + + def request(self, *args, **kwargs): + self.response = {"phedex": { + "block": [{ + "file": [ + { + "name": "someSite", + "decision": "approved", + "replica": [{ + "node": "someNode" + }] + }, + { + "name": "someSite1", + "decision": "pending", + "replica": [{ + "node": "someNode1" + }] + }, + ], + }]} + } + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + from WmAgentScripts.utils import getDatasetFileLocations + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = getDatasetFileLocations( + url='http://someurl.com/', + dataset='somedataset' + ) + print response + self.assertDictEqual( + response, { + 'someSite': set(['someNode']), + 'someSite1': set(['someNode1'])}) + + if __name__ == '__main__': unittest.main() From 56858c55efaebbf9da611d4fc781f53faf9229e4 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 2 Aug 2020 18:13:29 +0530 Subject: [PATCH 19/29] add unittests for utils.invalidateFiles --- tests/test_utils.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4c27ca6d2..677c53175 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -559,12 +559,43 @@ def getresponse(self): url='http://someurl.com/', dataset='somedataset' ) - print response self.assertDictEqual( response, { 'someSite': set(['someNode']), 'someSite1': set(['someNode1'])}) +class TestInvalidateFiles(unittest.TestCase): + + def test_invalidateFiles(self): + + class MockResponseStringIo: + response = {"result": "OK"} + + def __init__(self, *args, **kwargs): + pass + + def request(self, *args, **kwargs): + pass + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import invalidateFiles + + with patch('WmAgentScripts.utils.httplib.HTTPSConnection', MockResponseStringIo): + response = invalidateFiles( + files=["file1", "file2"], + ) + self.assertTrue(response) + + MockResponseStringIo.response = {"result": "Not OK"} + with patch('WmAgentScripts.utils.httplib.HTTPSConnection', MockResponseStringIo): + response = invalidateFiles( + files=["file1", "file2"], + ) + self.assertFalse(response) + + if __name__ == '__main__': unittest.main() From c10a950bfffebaa46b1f07dc40beba35ceea76d5 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Tue, 4 Aug 2020 08:05:54 +0530 Subject: [PATCH 20/29] add unittests for utils.getConfigurationFile --- tests/test_utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 677c53175..fd5e84b74 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -597,5 +597,33 @@ def getresponse(self): self.assertFalse(response) +class TestGetConfigurationFile(unittest.TestCase): + + def setUp(self): + class MockResponseStringIo: + + def __init__(self, *args, **kwargs): + self.response = None + + def request(self, *args, **kwargs): + self.response = "Test1 line 1\nTest2 line 2\nTest3 line 3" + + def getresponse(self): + return ContextualStringIO(self.response) + + self.mockresponse = MockResponseStringIo + + def test_getConfigurationFile(self): + from WmAgentScripts.utils import getConfigurationFile, getConfigurationLine + + with patch('WmAgentScripts.utils.make_x509_conn', self.mockresponse): + response = getConfigurationFile( + url='http://someurl.com/', + cacheid='cacheid' + ) + self.assertEqual( + response, "Test1 line 1\nTest2 line 2\nTest3 line 3") + + if __name__ == '__main__': unittest.main() From 25350b96199a53b13ee88ecd45c157885d6ff35e Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Tue, 4 Aug 2020 08:10:16 +0530 Subject: [PATCH 21/29] add unittests for utils.getConfigurationLine --- tests/test_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index fd5e84b74..6036174d4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -624,6 +624,18 @@ def test_getConfigurationFile(self): self.assertEqual( response, "Test1 line 1\nTest2 line 2\nTest3 line 3") + def test_getConfigurationLine(self): + from WmAgentScripts.utils import getConfigurationLine + + with patch('WmAgentScripts.utils.make_x509_conn', self.mockresponse): + + response = getConfigurationLine( + url='http://someurl.com/', + cacheid='cacheid', + token='Test2') + + self.assertEqual(response, "Test2 line 2") + if __name__ == '__main__': unittest.main() From d16137afb603a51f120a4b52b7603de98f2ce981 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Tue, 4 Aug 2020 12:25:13 +0530 Subject: [PATCH 22/29] add unittests for utils.getWorkflowByCampaign --- tests/test_utils.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6036174d4..5af0a0b52 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -408,6 +408,7 @@ def test_is_json(self): class TestReadFile(unittest.TestCase): def test_read_file(self): + from WmAgentScripts.utils import read_file test_json = { "first": {"a": "A"}, @@ -452,7 +453,9 @@ def getresponse(self): class TestCheckTransferApproval(unittest.TestCase): def test_checkTransferApproval(self): + class MockResponseStringIo: + def __init__(self, *args, **kwargs): self.response = None @@ -490,7 +493,9 @@ def getresponse(self): class TestGetNodesId(unittest.TestCase): def test_getNodesId(self): + class MockResponseStringIo: + def __init__(self, *args, **kwargs): self.response = None @@ -524,7 +529,9 @@ def getresponse(self): class TestGetDatasetFileLocations(unittest.TestCase): + def test_getDatasetFileLocations(self): + class MockResponseStringIo: def __init__(self, *args, **kwargs): self.response = None @@ -553,12 +560,15 @@ def request(self, *args, **kwargs): def getresponse(self): return ContextualStringIO(json.dumps(self.response)) + from WmAgentScripts.utils import getDatasetFileLocations + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): response = getDatasetFileLocations( url='http://someurl.com/', dataset='somedataset' ) + self.assertDictEqual( response, { 'someSite': set(['someNode']), @@ -637,5 +647,50 @@ def test_getConfigurationLine(self): self.assertEqual(response, "Test2 line 2") +class TestGetWorkflowByCampaign(unittest.TestCase): + + def test_getWorkflowByCampaign(self): + + class MockResponseStringIo: + + def __init__(self, *args, **kwargs): + self.response = None + + def request(self, *args, **kwargs): + self.response = {"result": + [{"data": [ + { + "name": "someSite", + }, + { + "name": "someSite1", + }, + ], + }] + } + + def getresponse(self): + return ContextualStringIO(json.dumps(self.response)) + + from WmAgentScripts.utils import getWorkflowByCampaign + + with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + response = getWorkflowByCampaign( + url='http://someurl.com/', + campaign='somecampaign', + details=False + ) + self.assertEqual( + response, [{'data': [{'name': 'someSite'}, {'name': 'someSite1'}]}]) + + response = getWorkflowByCampaign( + url='http://someurl.com/', + campaign='somecampaign', + details=True + ) + self.assertEqual( + response, [[{'name': 'someSite'}, {'name': 'someSite1'}]]) + + if __name__ == '__main__': unittest.main() From 0654efa208f21278a57213fc8dd054bad86907cf Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Wed, 5 Aug 2020 18:56:52 +0530 Subject: [PATCH 23/29] add self hosted runners --- .github/workflows/run_test.yml | 9 +-------- tests/test_utils.py | 1 - 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml index 817af5e80..b7ef0a9bb 100644 --- a/.github/workflows/run_test.yml +++ b/.github/workflows/run_test.yml @@ -4,17 +4,10 @@ on: [push, pull_request] jobs: lint-python: - runs-on: ubuntu-latest + runs-on: ['self-hosted', 'Linux', 'X64'] steps: - name: Checkout uses: actions/checkout@v2.2.0 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 2.7 - - name: Install Pytest and mock - run: | - python2 -m pip install pytest mock - name: Run pytest run: | python2 -m pytest -v tests diff --git a/tests/test_utils.py b/tests/test_utils.py index 5af0a0b52..44883498e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -257,7 +257,6 @@ def getresponse(self): response = listCustodial( url='http://someurl.com/', ) - print response self.assertDictEqual( response, { 'someSite1': ['someId'], 'someSite': ['someId']}) From 699cb0c432ea59f78d3622fc7c8e01dd7f10895f Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 9 Aug 2020 04:43:31 +0200 Subject: [PATCH 24/29] Add tests for lxplus --- tests/test_utils.py | 539 +++++++++++++++++++++++++++----------------- 1 file changed, 331 insertions(+), 208 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 44883498e..e5a462573 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ import sys +import httplib import unittest import json @@ -11,6 +12,9 @@ sys.modules['dbs.apis'] = MagicMock() sys.modules['dbs.apis.dbsClient'] = MagicMock() +reqmgr_url = "cmsweb.cern.ch" +test_dataset = "/TTJets_mtop1695_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIIWinter15GS-MCRUN2_71_V1-v1/GEN-SIM" + class ContextualStringIO(StringIO): def __enter__(self): @@ -21,6 +25,19 @@ def __exit__(self, *args): return False +def mock_make_x509_conn(url=reqmgr_url, max_try=5): + tries = 0 + while tries < max_try: + try: + conn = httplib.HTTPSConnection(url) + return conn + except Exception as e: + print e + tries += 1 + pass + return None + + class TestDeepUpdate(unittest.TestCase): def setUp(self): @@ -172,152 +189,196 @@ def getresponse(self): class TestGetSubscription(unittest.TestCase): def testGetSubscription(self): - - class MockResponseStringIo: - def __init__(self, *args, **kwargs): - self.response = None, 404 - - def request(self, *args, **kwargs): - self.response = {"phedex": "value1"} - - def getresponse(self): - return ContextualStringIO(json.dumps(self.response)) - from WmAgentScripts.utils import getSubscriptions - with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): response = getSubscriptions( - url='http://someurl.com/', - dataset="somedataset") - self.assertEqual(response, "value1") + url=reqmgr_url, + dataset=test_dataset) + self.assertEqual(response['dataset'], []) + self.assertEqual(response['instance'], 'prod') + self.assertEqual(response['request_call'], 'subscriptions') + self.assertEqual( + response['request_url'], + 'http://cmsweb.cern.ch:7001/phedex/datasvc/json/prod/subscriptions') class TestListRequests(unittest.TestCase): def testListRequests(self): - class MockResponseStringIo: - def __init__(self, *args, **kwargs): - self.response = None - - def request(self, *args, **kwargs): - self.response = {"phedex": { - "request": [{ - "node": [ - {"name": "someSite"}, - {"name": "someSite1"} - ], - "id": "someId" - }]} - } - - def getresponse(self): - return ContextualStringIO(json.dumps(self.response)) - from WmAgentScripts.utils import listRequests - with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): response = listRequests( - url='http://someurl.com/', - dataset="somedataset", + url=reqmgr_url, + dataset=test_dataset, site=None) self.assertDictEqual( - response, { - 'someSite1': ['someId'], 'someSite': ['someId']}) + response, + { + 'T1_ES_PIC_Disk': [ + 445269, + 453022], + 'T2_CH_CSCS': [454057], + 'T1_FR_CCIN2P3_Disk': [ + 441511, + 460098], + 'T2_US_Purdue': [ + 445274, + 454061], + 'XT1_UK_RAL_Disk': [454098], + 'T0_CH_CERN_MSS': [ + 441510, + 775255, + 783438], + 'T2_UK_SGrid_RALPP': [454063], + 'T2_FR_GRIF_LLR': [454064], + 'T2_BE_UCL': [454065], + 'T2_ES_IFCA': [454066], + 'T2_DE_RWTH': [454067], + 'T2_FR_IPHC': [454101], + 'T2_KR_KNU': [ + 454070, + 454730], + 'T2_DE_DESY': [ + 445267, + 454071], + 'T2_IT_Legnaro': [454073], + 'T2_US_Caltech': [ + 445273, + 454074], + 'T1_DE_KIT_Disk': [ + 445265, + 454075], + 'T2_UK_London_Brunel': [454076], + 'T2_RU_JINR': [454077], + 'T2_IT_Pisa': [454078], + 'T1_US_FNAL_Disk': [ + 445271, + 454079], + 'T2_EE_Estonia': [454080], + 'T2_IT_Rome': [454081], + 'T2_US_Florida': [454082], + 'T2_FR_GRIF_IRFU': [454083], + 'T1_IT_CNAF_Disk': [ + 445266, + 454084], + 'T2_FI_HIP': [454062], + 'T1_RU_JINR_Disk': [454085], + 'T2_UK_London_IC': [454086], + 'T2_IT_Bari': [454087], + 'T2_US_Nebraska': [ + 445276, + 454088], + 'T2_FR_CCIN2P3': [454089], + 'T2_US_UCSD': [454090], + 'T2_ES_CIEMAT': [ + 454091, + 457804], + 'T2_RU_IHEP': [454092], + 'T2_US_Wisconsin': [ + 445268, + 454093], + 'T2_HU_Budapest': [454094], + 'T2_CN_Beijing': [454095], + 'T2_US_MIT': [454096], + 'T2_BE_IIHE': [454097], + 'T2_CH_CERN': [454103], + 'T2_PT_NCG_Lisbon': [454069], + 'T2_US_Vanderbilt': [454100], + 'T2_BR_SPRACE': [454102]}) + response = listRequests( - url='http://someurl.com/', - dataset="somedataset", - site='someSite') - self.assertDictEqual(response, {'someSite': ['someId']}) + url='cmsweb.cern.ch', + dataset=test_dataset, + site='T1_ES_PIC_Disk') + self.assertDictEqual( + response, { + 'T1_ES_PIC_Disk': [ + 445269, 453022]}) -class TestListRequests(unittest.TestCase): +class TestListCustodial(unittest.TestCase): def testListCustodial(self): - class MockResponseStringIo: - def __init__(self, *args, **kwargs): - self.response = None - - def request(self, *args, **kwargs): - self.response = {"phedex": { - "request": [{ - "node": [ - {"name": "someSite"}, - {"name": "someSite1"} - ], - "id": "someId", - "type": "xfer" - }]} - } - - def getresponse(self): - return ContextualStringIO(json.dumps(self.response)) - from WmAgentScripts.utils import listCustodial - with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): response = listCustodial( - url='http://someurl.com/', + url=reqmgr_url, ) - self.assertDictEqual( - response, { - 'someSite1': ['someId'], 'someSite': ['someId']}) + print response + self.assertDictEqual(response, + {'T1_US_FNAL_MSS': [2345013, + 2345083, + 2345230, + 2345350, + 2345586, + 2345858, + 2346380, + 2346956, + 2347154, + 2347520, + 2347851, + 2349029, + 2349077, + 2349378, + 2349379, + 2349684, + 2350365, + 2350577, + 2350849, + 2351209, + 2351236, + 2352060, + 2352170, + 2352228, + 2352416, + 2352695, + 2352781, + 2352859, + 2353136, + 2353137, + 2353267, + 2353341, + 2353361, + 2353408, + 2353415, + 2353500, + 2353597, + 2353796, + 2353830, + 2353952, + 2353963, + 2354169, + 2354231, + 2354415, + 2354580, + 2354583, + 2354684, + 2354922, + 2354937, + 2354988, + 2355007, + 2355246, + 2355284, + 2355657], + 'T1_FR_CCIN2P3_MSS': [1319254, + 1319259, + 1319315, + 1321435]}) class TestListSubscriptions(unittest.TestCase): def testListSubscriptions(self): - class MockResponseStringIo: - def __init__(self, *args, **kwargs): - self.response = None - - def request(self, *args, **kwargs): - self.response = {"phedex": { - "request": [{ - "node": [ - { - "name": "someSite", - "decision": "approved", - "time_decided": 1300 - }, - { - "name": "someSite1", - "decision": "pending", - "time_decided": 1400 - }, - { - "name": "someSiteMSS", - "decision": "pending", - "time_decided": 1400 - }, - ], - "id": "someId", - "type": "xfer" - }]} - } - - def getresponse(self): - return ContextualStringIO(json.dumps(self.response)) - from WmAgentScripts.utils import listSubscriptions - with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): response = listSubscriptions( - url='http://someurl.com/', - dataset='somedataset' + url=reqmgr_url, + dataset=test_dataset ) - self.assertDictEqual( - response, { - 'someSite': ( - 'someId', True), - 'someSite1': ( - 'someId', False)}) - - response = listSubscriptions( - url='http://someurl.com/', - dataset='somedataset', - within_sites=['someSite'] - ) - self.assertDictEqual( - response, {'someSite': ('someId', True)}) + self.assertDictEqual(response, {}) class TestPass_to_dynamo(unittest.TestCase): @@ -356,34 +417,9 @@ class TesCheckDownTime(unittest.TestCase): def testCheckDownTime(self): - class MockResponseStringIo(StringIO): - status = 503 - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - return False - - class MockResponse: - def __init__(self, *args, **kwargs): - self.response = None - - def request(self, *args, **kwargs): - self.response = {} - - def getresponse(self): - return MockResponseStringIo(json.dumps(self.response)) - from WmAgentScripts.utils import checkDownTime - with patch('WmAgentScripts.utils.make_x509_conn', MockResponse): - response = checkDownTime() - self.assertTrue(response) - - MockResponseStringIo.status = 404 - with patch('WmAgentScripts.utils.make_x509_conn', MockResponse): + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): response = checkDownTime() self.assertFalse(response) @@ -446,7 +482,6 @@ def getresponse(self): from WmAgentScripts.utils import getWMStats with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): response = getWMStats(url='http://someurl.com/') - self.assertEqual(response, 200) class TestCheckTransferApproval(unittest.TestCase): @@ -493,85 +528,173 @@ class TestGetNodesId(unittest.TestCase): def test_getNodesId(self): - class MockResponseStringIo: - - def __init__(self, *args, **kwargs): - self.response = None - - def request(self, *args, **kwargs): - self.response = {"phedex": { - "node": [ - { - "name": "someSite", - "id": "someId", - }, - { - "name": "someSite1", - "id": "someId1", - }, - ], - "type": "xfer" - }} - - def getresponse(self): - return ContextualStringIO(json.dumps(self.response)) - from WmAgentScripts.utils import getNodesId - with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): response = getNodesId( - url='http://someurl.com/' + url=reqmgr_url ) - self.assertDictEqual( - response, { - 'someSite': 'someId', - 'someSite1': 'someId1'}) + self.assertDictEqual(response, + {'T3_US_PuertoRico': '1521', + 'T3_RU_MEPhI': '2261', + 'T2_FI_HIP': '37', + 'T2_UK_SGrid_RALPP': '56', + 'T2_FR_GRIF_LLR': '34', + 'T3_IT_Trieste': '1481', + 'T3_TW_NTU_HEP': '1501', + 'T2_KR_KNU': '43', + 'T3_US_UMD': '681', + 'T3_US_Colorado': '741', + 'T1_UK_RAL_Disk': '2301', + 'T3_IT_Napoli': '68', + 'T3_US_Kansas': '581', + 'T1_IT_CNAF_Disk': '661', + 'T3_CH_CERN_OpenData': '2241', + 'T2_IT_Bari': '21', + 'T3_US_HEPCloud': '2221', + 'T2_US_UCSD': '62', + 'T2_RU_IHEP': '38', + 'T3_US_Vanderbilt_EC2': '1841', + 'T3_US_JHU': '1181', + 'T3_BY_NCPHEP': '1761', + 'T1_RU_JINR_Buffer': '2121', + 'T2_US_Vanderbilt': '242', + 'T3_US_UCR': '102', + 'T3_TW_NCU': '801', + 'T1_IT_CNAF_MSS': '8', + 'T2_CH_CSCS': '27', + 'T2_UA_KIPT': '341', + 'T2_PK_NCP': '1003', + 'T1_IT_CNAF_Buffer': '7', + 'T3_US_Brown': '1341', + 'T3_US_UCD': '1241', + 'T2_FR_IPHC': '181', + 'T3_US_OSU': '1201', + 'T3_GR_IASA_GR': '602', + 'T3_US_TAMU': '1441', + 'T2_IT_Rome': '55', + 'T2_UK_London_Brunel': '46', + 'T3_US_TTU': '123', + 'T2_EE_Estonia': '32', + 'T2_IN_TIFR': '281', + 'T1_UK_RAL_Buffer': '18', + 'T2_CN_Beijing': '22', + 'T1_RU_JINR_MSS': '2122', + 'T2_US_Florida': '33', + 'T1_US_FNAL_MSS': '10', + 'T3_GR_IASA_HG': '601', + 'T3_US_Princeton_ICSE': '1021', + 'T3_IT_MIB': '1401', + 'T3_US_FNALXEN': '1161', + 'T3_US_Rutgers': '701', + 'T1_FR_CCIN2P3_Buffer': '13', + 'T3_IR_IPM': '1621', + 'T2_US_Wisconsin': '65', + 'T2_HU_Budapest': '26', + 'T2_DE_RWTH': '54', + 'T2_BR_SPRACE': '58', + 'T2_CH_CERN': '1561', + 'T2_BR_UERJ': '36', + 'T3_MX_Cinvestav': '1281', + 'T3_US_FNALLPC': '881', + 'T1_US_FNAL_Disk': '1781', + 'T3_KR_KISTI': '1921', + 'T1_ES_PIC_MSS': '17', + 'T3_IN_VBU': '2081', + 'T3_IT_Firenze': '1041', + 'T0_CH_CERN_MSS': '2', + 'T2_ES_IFCA': '541', + 'T3_US_UVA': '1321', + 'T3_TH_CHULA': '1981', + 'T3_US_NotreDame': '1361', + 'T2_DE_DESY': '29', + 'T3_US_UIowa': '71', + 'T3_HU_Debrecen': '1941', + 'T2_US_Caltech': '28', + 'T3_CH_CMSAtHome': '2321', + 'T3_FR_IPNL': '421', + 'T1_US_FNAL_Buffer': '9', + 'T3_BG_UNI_SOFIA': '2021', + 'T1_ES_PIC_Buffer': '15', + 'T1_UK_RAL_MSS': '19', + 'T1_RU_JINR_Disk': '1745', + 'T3_CN_PKU': '521', + 'T2_UK_London_IC': '47', + 'T2_US_Nebraska': '51', + 'T2_ES_CIEMAT': '59', + 'T3_US_Princeton': '70', + 'T1_FR_CCIN2P3_Disk': '1861', + 'T3_DE_Karlsruhe': '66', + 'T2_KR_KISTI': '2281', + 'T3_KR_UOS': '1601', + 'T3_IT_Perugia': '69', + 'T1_ES_PIC_Disk': '16', + 'T2_CH_CERNBOX': '2141', + 'T3_US_Minnesota': '67', + 'T1_FR_CCIN2P3_MSS': '14', + 'T2_TR_METU': '381', + 'T2_AT_Vienna': '63', + 'T2_US_Purdue': '53', + 'T2_TW_NCHC': '2181', + 'T3_US_Rice': '901', + 'T3_HR_IRB': '1882', + 'T1_DE_KIT_MSS': '1262', + 'T2_BE_UCL': '24', + 'T3_US_FIT': '1101', + 'T2_UK_SGrid_Bristol': '25', + 'T2_PT_NCG_Lisbon': '1221', + 'T2_IT_Legnaro': '45', + 'T1_DE_KIT_Disk': '1821', + 'T2_RU_ITEP': '40', + 'T3_US_SDSC': '2001', + 'T2_RU_JINR': '42', + 'T2_IT_Pisa': '52', + 'T2_GR_Ioannina': '761', + 'T3_US_MIT': '1744', + 'T3_US_UCSB': '2061', + 'T2_MY_UPM_BIRUNI': '1903', + 'T0_CH_CERN_Export': '1', + 'T2_FR_GRIF_IRFU': '82', + 'T3_US_UMiss': '1381', + 'T2_PL_Swierk': '1961', + 'T3_RU_FIAN': '1641', + 'T2_FR_CCIN2P3': '1081', + 'T2_PL_Warsaw': '64', + 'T2_US_MIT': '50', + 'T2_BE_IIHE': '23', + 'T2_RU_INR': '561', + 'T1_DE_KIT_Buffer': '1261', + 'T3_CH_PSI': '821', + 'T3_US_Baylor': '1721', + 'T3_IT_Bologna': '1541'}) class TestGetDatasetFileLocations(unittest.TestCase): def test_getDatasetFileLocations(self): - class MockResponseStringIo: - def __init__(self, *args, **kwargs): - self.response = None - - def request(self, *args, **kwargs): - self.response = {"phedex": { - "block": [{ - "file": [ - { - "name": "someSite", - "decision": "approved", - "replica": [{ - "node": "someNode" - }] - }, - { - "name": "someSite1", - "decision": "pending", - "replica": [{ - "node": "someNode1" - }] - }, - ], - }]} - } - - def getresponse(self): - return ContextualStringIO(json.dumps(self.response)) - from WmAgentScripts.utils import getDatasetFileLocations - with patch('WmAgentScripts.utils.make_x509_conn', MockResponseStringIo): + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): response = getDatasetFileLocations( - url='http://someurl.com/', - dataset='somedataset' + url=reqmgr_url, + dataset=test_dataset ) - self.assertDictEqual( - response, { - 'someSite': set(['someNode']), - 'someSite1': set(['someNode1'])}) + response, {}) + + +class TestFindCustodialLocation(unittest.TestCase): + def test_findCustodialLocation(self): + from WmAgentScripts.utils import findCustodialLocation + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): + response = findCustodialLocation( + url=reqmgr_url, + dataset=test_dataset, + with_completion=True + ) + print response + self.assertEqual( + response, ([], None)) class TestInvalidateFiles(unittest.TestCase): From 377f2a1234eaa9b694fc7460be4415eef4684e34 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 9 Aug 2020 22:31:25 +0530 Subject: [PATCH 25/29] mock mongo db client --- tests/test_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index e5a462573..7afeadfdb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,7 @@ import json import mock from mock import MagicMock, patch, mock_open +import pymongo from StringIO import StringIO @@ -14,7 +15,7 @@ reqmgr_url = "cmsweb.cern.ch" test_dataset = "/TTJets_mtop1695_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIIWinter15GS-MCRUN2_71_V1-v1/GEN-SIM" - +mongo_db_docker_url = "0.0.0.0:32774" class ContextualStringIO(StringIO): def __enter__(self): @@ -38,6 +39,9 @@ def mock_make_x509_conn(url=reqmgr_url, max_try=5): return None +def mock_mongo_client(): + return pymongo.MongoClient('mongodb://%s/'% mongo_db_docker_url) + class TestDeepUpdate(unittest.TestCase): def setUp(self): From de4d994c081553a336cf49278429c149f70b953e Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 9 Aug 2020 19:33:37 +0200 Subject: [PATCH 26/29] add tests for utils.agent_speed_draining --- tests/test_utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7afeadfdb..8f7fa757d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -818,5 +818,14 @@ def getresponse(self): response, [[{'name': 'someSite'}, {'name': 'someSite1'}]]) +class TestAgentSpeedDraining(unittest.TestCase): + def test_agent_speed_draining(self): + from WmAgentScripts.utils import agent_speed_draining + + with patch('WmAgentScripts.utils.mongo_client', mock_mongo_client): + response=agent_speed_draining() + self.assertEqual(response, set([u'vocms0250.cern.ch'])) + + if __name__ == '__main__': unittest.main() From 512536c29678742d34d352dc5f2fa37083075aac Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 9 Aug 2020 20:01:51 +0200 Subject: [PATCH 27/29] add test for utils.getAllAgents --- tests/test_utils.py | 129 ++++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 8f7fa757d..bcbbbcc23 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -17,6 +17,7 @@ test_dataset = "/TTJets_mtop1695_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIIWinter15GS-MCRUN2_71_V1-v1/GEN-SIM" mongo_db_docker_url = "0.0.0.0:32774" + class ContextualStringIO(StringIO): def __enter__(self): return self @@ -40,7 +41,8 @@ def mock_make_x509_conn(url=reqmgr_url, max_try=5): def mock_mongo_client(): - return pymongo.MongoClient('mongodb://%s/'% mongo_db_docker_url) + return pymongo.MongoClient('mongodb://%s/' % mongo_db_docker_url) + class TestDeepUpdate(unittest.TestCase): @@ -313,63 +315,63 @@ def testListCustodial(self): print response self.assertDictEqual(response, {'T1_US_FNAL_MSS': [2345013, - 2345083, - 2345230, - 2345350, - 2345586, - 2345858, - 2346380, - 2346956, - 2347154, - 2347520, - 2347851, - 2349029, - 2349077, - 2349378, - 2349379, - 2349684, - 2350365, - 2350577, - 2350849, - 2351209, - 2351236, - 2352060, - 2352170, - 2352228, - 2352416, - 2352695, - 2352781, - 2352859, - 2353136, - 2353137, - 2353267, - 2353341, - 2353361, - 2353408, - 2353415, - 2353500, - 2353597, - 2353796, - 2353830, - 2353952, - 2353963, - 2354169, - 2354231, - 2354415, - 2354580, - 2354583, - 2354684, - 2354922, - 2354937, - 2354988, - 2355007, - 2355246, - 2355284, - 2355657], + 2345083, + 2345230, + 2345350, + 2345586, + 2345858, + 2346380, + 2346956, + 2347154, + 2347520, + 2347851, + 2349029, + 2349077, + 2349378, + 2349379, + 2349684, + 2350365, + 2350577, + 2350849, + 2351209, + 2351236, + 2352060, + 2352170, + 2352228, + 2352416, + 2352695, + 2352781, + 2352859, + 2353136, + 2353137, + 2353267, + 2353341, + 2353361, + 2353408, + 2353415, + 2353500, + 2353597, + 2353796, + 2353830, + 2353952, + 2353963, + 2354169, + 2354231, + 2354415, + 2354580, + 2354583, + 2354684, + 2354922, + 2354937, + 2354988, + 2355007, + 2355246, + 2355284, + 2355657], 'T1_FR_CCIN2P3_MSS': [1319254, - 1319259, - 1319315, - 1321435]}) + 1319259, + 1319315, + 1321435]}) class TestListSubscriptions(unittest.TestCase): @@ -818,13 +820,22 @@ def getresponse(self): response, [[{'name': 'someSite'}, {'name': 'someSite1'}]]) +class TestGetAllAgents(unittest.TestCase): + def test_getAllAgents(self): + from WmAgentScripts.utils import getBlockLocations as getAllAgents + + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): + response = getAllAgents(url=reqmgr_url, dataset=test_dataset) + self.assertDictEqual(response, {}) + + class TestAgentSpeedDraining(unittest.TestCase): def test_agent_speed_draining(self): from WmAgentScripts.utils import agent_speed_draining with patch('WmAgentScripts.utils.mongo_client', mock_mongo_client): - response=agent_speed_draining() - self.assertEqual(response, set([u'vocms0250.cern.ch'])) + response = agent_speed_draining() + self.assertEqual(response, set(['vocms0250.cern.ch'])) if __name__ == '__main__': From f7a76149028f707fb237dd77159154227a505aa7 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 9 Aug 2020 21:23:16 +0200 Subject: [PATCH 28/29] add tests for utils.batchinfo --- tests/test_utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index bcbbbcc23..4d941ed2f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -838,5 +838,20 @@ def test_agent_speed_draining(self): self.assertEqual(response, set(['vocms0250.cern.ch'])) +class TestBatchInfo(unittest.TestCase): + def test_content(self): + from WmAgentScripts.utils import batchInfo + + with patch('WmAgentScripts.utils.mongo_client', mock_mongo_client): + obj = batchInfo() + all = obj.all() + self.assertIn( + 'CMSSW_10_4_0_pre1__UPSG_Std_2023D24noPU-1542117255', all) + content = obj.content() + self.assertIn( + 'CMSSW_10_4_0_pre1__UPSG_Std_2023D24noPU-1542117255', + content) + + if __name__ == '__main__': unittest.main() From 26063c3a68e49096c8263accb900eb69eac2e482 Mon Sep 17 00:00:00 2001 From: Ishan Rai Date: Sun, 9 Aug 2020 21:40:15 +0200 Subject: [PATCH 29/29] add tests for utils.getNodesQueue --- .github/workflows/run_test.yml | 2 +- tests/test_utils.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml index b7ef0a9bb..e03f4e280 100644 --- a/.github/workflows/run_test.yml +++ b/.github/workflows/run_test.yml @@ -3,7 +3,7 @@ name: Unit test on: [push, pull_request] jobs: - lint-python: + unit-test: runs-on: ['self-hosted', 'Linux', 'X64'] steps: - name: Checkout diff --git a/tests/test_utils.py b/tests/test_utils.py index 4d941ed2f..8dc1d4df8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -820,15 +820,24 @@ def getresponse(self): response, [[{'name': 'someSite'}, {'name': 'someSite1'}]]) -class TestGetAllAgents(unittest.TestCase): - def test_getAllAgents(self): - from WmAgentScripts.utils import getBlockLocations as getAllAgents +class TestGetBlockLocations(unittest.TestCase): + def test_getBlockLocations(self): + from WmAgentScripts.utils import getBlockLocations with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): - response = getAllAgents(url=reqmgr_url, dataset=test_dataset) + response = getBlockLocations(url=reqmgr_url, dataset=test_dataset) self.assertDictEqual(response, {}) +class TestGetNodesQueue(unittest.TestCase): + def test_getNodesQueue(self): + from WmAgentScripts.utils import getNodesQueue + + with patch('WmAgentScripts.utils.make_x509_conn', mock_make_x509_conn): + response = getNodesQueue(url=reqmgr_url) + self.assertIn('T3_RU_MEPhI', response) + + class TestAgentSpeedDraining(unittest.TestCase): def test_agent_speed_draining(self): from WmAgentScripts.utils import agent_speed_draining @@ -839,7 +848,7 @@ def test_agent_speed_draining(self): class TestBatchInfo(unittest.TestCase): - def test_content(self): + def test_batchInfo(self): from WmAgentScripts.utils import batchInfo with patch('WmAgentScripts.utils.mongo_client', mock_mongo_client):