From 4105a1f045d3ec0a6808f0bf295e06298b2857e8 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Thu, 14 Nov 2024 15:41:24 +0000 Subject: [PATCH 01/28] pytest init --- pytest.ini | 3 +++ tests/test_workspace.py | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 pytest.ini create mode 100644 tests/test_workspace.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..e3ad283a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +addopts = --ignore=data/ + diff --git a/tests/test_workspace.py b/tests/test_workspace.py new file mode 100644 index 00000000..01e8b161 --- /dev/null +++ b/tests/test_workspace.py @@ -0,0 +1,4 @@ +import pytest + +def test_workspace(): + assert 2 + 2 == 4 From 84ac4af28f77e61ff8ae0b5487211fb9dd625fa9 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Thu, 14 Nov 2024 15:46:52 +0000 Subject: [PATCH 02/28] setup action --- .github/workflows/python-package-conda.yml | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml new file mode 100644 index 00000000..bc51b865 --- /dev/null +++ b/.github/workflows/python-package-conda.yml @@ -0,0 +1,35 @@ +name: Python Package using Conda + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v3 + with: + python-version: '3.12' + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + - name: Lint with flake8 + run: | + conda install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + conda install pytest + pytest + From bcc29308a1fc059b7035de3b2650fe65c8a2a569 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Thu, 14 Nov 2024 16:17:22 +0000 Subject: [PATCH 03/28] test workflow --- .github/workflows/python-package-conda.yml | 35 ---------------------- .github/workflows/tests.yml | 2 +- 2 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml deleted file mode 100644 index bc51b865..00000000 --- a/.github/workflows/python-package-conda.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Python Package using Conda - -on: [push] - -jobs: - build-linux: - runs-on: ubuntu-latest - strategy: - max-parallel: 5 - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.12 - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - - name: Install dependencies - run: | - conda env update --file environment.yml --name base - - name: Lint with flake8 - run: | - conda install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - conda install pytest - pytest - diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d5712b43..b2502db2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ name: "Tests" on: push: - branches: [ "main" ] + branches: [ "main", "tests"] pull_request: # The branches below must be a subset of the branches above branches: [ "main" ] From 43d0d0336e748634b8998f96aa8a26f574e904f0 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Thu, 14 Nov 2024 16:19:02 +0000 Subject: [PATCH 04/28] tests folder --- {tests => froggy/tests}/test_workspace.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {tests => froggy/tests}/test_workspace.py (100%) diff --git a/tests/test_workspace.py b/froggy/tests/test_workspace.py similarity index 100% rename from tests/test_workspace.py rename to froggy/tests/test_workspace.py From d8926f2a19160f821f32c874cca28f3a6ad2cbe3 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Thu, 14 Nov 2024 18:34:11 +0000 Subject: [PATCH 05/28] repo env --- .vscode/settings.json | 7 +++++++ froggy/tests/test_workspace.py | 20 ++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..87ae1057 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "froggy" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index 01e8b161..d92d4219 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -1,4 +1,20 @@ -import pytest +from unittest.mock import patch +from froggy.envs import RepoEnv +from froggy.utils import load_config +from pathlib import PosixPath +@patch('sys.argv', ['run.py', 'scripts/config.yaml', '--agent', 'cot', '--debug', '-v']) def test_workspace(): - assert 2 + 2 == 4 + config, args = load_config() + config = config[args.agent] + + assert args.config_file == 'scripts/config.yaml' + assert args.agent == 'cot' + assert args.debug == True + assert args.verbose == True + + env = RepoEnv(**config["env_kwargs"]) + + assert isinstance(env.path, PosixPath) + assert env.path == PosixPath('data/pytorch') + \ No newline at end of file From eefa89804a46a68f4c42940320e50010e3c80b78 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Fri, 15 Nov 2024 00:25:07 +0000 Subject: [PATCH 06/28] mock functions --- froggy/tests/test_workspace.py | 77 +++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index d92d4219..ae4296ba 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -1,20 +1,69 @@ -from unittest.mock import patch +import unittest +from unittest.mock import patch, MagicMock +from pathlib import Path, PosixPath from froggy.envs import RepoEnv from froggy.utils import load_config -from pathlib import PosixPath -@patch('sys.argv', ['run.py', 'scripts/config.yaml', '--agent', 'cot', '--debug', '-v']) -def test_workspace(): - config, args = load_config() - config = config[args.agent] +class TestRepoEnv(unittest.TestCase): + @patch('sys.argv', ['run.py', 'scripts/config.yaml', '--agent', 'cot', '--debug', '-v']) + def test_workspace(self): + config, args = load_config() + config = config[args.agent] - assert args.config_file == 'scripts/config.yaml' - assert args.agent == 'cot' - assert args.debug == True - assert args.verbose == True - - env = RepoEnv(**config["env_kwargs"]) + assert args.config_file == 'scripts/config.yaml' + assert args.agent == 'cot' + assert args.debug == True + assert args.verbose == True + + env = RepoEnv(**config["env_kwargs"]) - assert isinstance(env.path, PosixPath) - assert env.path == PosixPath('data/pytorch') + assert isinstance(env.path, PosixPath) + assert env.path == PosixPath('data/pytorch') + + @patch('tempfile.TemporaryDirectory') + @patch('atexit.register') + @patch('shutil.copytree') + def test_setup_workspace(self, mock_copytree, mock_atexit_register, mock_tempdir): + # Mock the temporary directory + mock_tempdir.return_value.name = '/mock/tempdir' + + # Create an instance of RepoEnv + repo_env = RepoEnv(run_timeout=10, dir_tree_depth=2, auto_view_change=True) + + # Call setup_workspace + repo_env.setup_workspace('/mock/path', 'python', ['readonly_pattern']) + + # Assertions + self.assertEqual(repo_env.path, Path('/mock/path')) + self.assertEqual(repo_env.working_dir, Path('/mock/tempdir')) + + # Check if the temporary directory was created + mock_tempdir.assert_called_once_with(prefix='RepoEnv-') + + # Check if atexit.register was called to cleanup the temporary directory + mock_atexit_register.assert_called_once_with(repo_env.tempdir.cleanup) + + # Check if shutil.copytree was called to copy the directory + mock_copytree.assert_called_once_with(Path('/mock/path'), Path('/mock/tempdir'), dirs_exist_ok=True) + + @patch('tempfile.TemporaryDirectory') + @patch('atexit.register') + @patch('shutil.copytree') + def test_setup_workspace_with_none_path(self, mock_copytree, mock_atexit_register, mock_tempdir): + # Create an instance of RepoEnv + repo_env = RepoEnv(run_timeout=10, dir_tree_depth=2, auto_view_change=True) + + # Call setup_workspace with None path + repo_env.setup_workspace(None) + + # Assertions + self.assertIsNone(repo_env.path) + + # Check that copytree and tempdir were not called + mock_tempdir.assert_not_called() + mock_copytree.assert_not_called() + mock_atexit_register.assert_not_called() + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From ab298de71cd98f7f1d321e2acdb7b43545840415 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Fri, 15 Nov 2024 03:35:21 +0000 Subject: [PATCH 07/28] test pdb --- froggy/tests/test_pdb.py | 117 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 froggy/tests/test_pdb.py diff --git a/froggy/tests/test_pdb.py b/froggy/tests/test_pdb.py new file mode 100644 index 00000000..dabdbf3f --- /dev/null +++ b/froggy/tests/test_pdb.py @@ -0,0 +1,117 @@ +import unittest +from unittest.mock import patch, MagicMock, call +from froggy.tools.pdb import PDBTool +from froggy.utils import TimeoutException + +class TestPDBTool(unittest.TestCase): + def setUp(self): + self.pdb_tool = PDBTool() + self.pdb_tool.environment = MagicMock() + self.pdb_tool.environment.working_dir = "/home/user/project" + self.pdb_tool.master = 1 # Mock the master attribute + + @patch('os.write') + @patch('os.read', return_value=b'/home/user/project/constants.py(6)()\n-> ACTION_TO_INDEX = {\n(Pdb)') + def test_get_current_frame_file(self, mock_os_read, mock_os_write): + self.pdb_tool.has_pseudo_terminal = MagicMock(return_value=True) + self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value=( + "> /home/user/project/constants.py(6)()\n" + "-> ACTION_TO_INDEX = {\n(Pdb)" + )) + + # Call the method + self.pdb_tool.get_current_frame_file() + + # Assertions + mock_os_write.assert_called_once_with(self.pdb_tool.master, b"where\n") + self.assertEqual(self.pdb_tool.current_frame_file, "constants.py") + + @patch('os.write') + @patch('os.read', return_value=b'/home/user/project/constants.py(6)()\n-> ACTION_TO_INDEX = {\n(Pdb)') + def test_get_current_frame_file_no_pseudo_terminal(self, mock_os_read, mock_os_write): + self.pdb_tool.has_pseudo_terminal = MagicMock(return_value=False) + self.pdb_tool.start_pseudo_terminal = MagicMock() + self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value=( + "> /home/user/project/constants.py(6)()\n" + "-> ACTION_TO_INDEX = {\n" + )) + + # Call the method + self.pdb_tool.get_current_frame_file() + + # Assertions + self.pdb_tool.start_pseudo_terminal.assert_called_once() + mock_os_write.assert_called_once_with(self.pdb_tool.master, b"where\n") + self.assertEqual(self.pdb_tool.current_frame_file, "constants.py") + + @patch('os.write') + @patch('os.read', return_value=b'/other/path/constants.py(6)()\n-> ACTION_TO_INDEX = {\n(Pdb)') + def test_get_current_frame_file_no_sep_in_output(self, mock_os_read, mock_os_write): + self.pdb_tool.has_pseudo_terminal = MagicMock(return_value=True) + self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value=( + "/other/path/constants.py(6)()\n" + "-> ACTION_TO_INDEX = {\n" + )) + + # Call the method + self.pdb_tool.get_current_frame_file() + + # Assertions + mock_os_write.assert_called_once_with(self.pdb_tool.master, b"where\n") + self.assertIsNone(self.pdb_tool.current_frame_file) + + @patch('subprocess.Popen') + @patch('os.close') + @patch('os.read', return_value=b'(Pdb)') + def test_start_pseudo_terminal(self, mock_os_read, mock_os_close, mock_popen): + mock_popen.return_value = MagicMock() + self.pdb_tool.environment.entrypoint = ["python", "script.py"] + self.pdb_tool.environment.working_dir = "/home/user/project" + + # Call the method + initial_output = self.pdb_tool.start_pseudo_terminal() + + # Assertions + self.assertIn("(Pdb)", initial_output) + mock_popen.assert_called_once_with( + ["python", "-m", "pdb", "script.py"], + env=unittest.mock.ANY, + cwd="/home/user/project", + stdin=unittest.mock.ANY, + stdout=unittest.mock.ANY, + stderr=unittest.mock.ANY, + text=True, + close_fds=True, + ) + #TODO mock_os_close.assert_called_once_with(self.pdb_tool.master) + + @patch('os.write') + @patch('os.read', return_value=b'(Pdb)') + def test_interact_with_pseudo_terminal(self, mock_os_read, mock_os_write): + self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value="(Pdb)") + + # Call the method + output = self.pdb_tool.interact_with_pseudo_terminal("list") + + # Assertions + mock_os_write.assert_called_once_with(self.pdb_tool.master, b"list\n") + self.assertEqual(output, "(Pdb)") + + @patch('os.write') + @patch('os.read', side_effect=[b'(Pdb)', b'The program finished and will be restarted\n(Pdb)']) + def test_interact_with_pseudo_terminal_restart(self, mock_os_read, mock_os_write): + self.pdb_tool.read_pseudo_terminal_output = MagicMock(side_effect=[ + "(Pdb)", + "The program finished and will be restarted\n(Pdb)" + ]) + self.pdb_tool.start_pseudo_terminal = MagicMock(return_value="(Pdb)") + + # Call the method + output = self.pdb_tool.interact_with_pseudo_terminal("quit") + + # Assertions + # TODO mock_os_write.assert_has_calls([call(self.pdb_tool.master, b"quit\n")]) + # TODO self.assertIn("The program finished and will be restarted", output) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From d304b84d60d16fda43a5d60358517f061e29d277 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Mon, 18 Nov 2024 17:07:10 +0000 Subject: [PATCH 08/28] code coverage --- .github/workflows/tests.yml | 3 ++- pytest.ini | 1 + requirements.txt | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b2502db2..c3beb302 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,4 +32,5 @@ jobs: pip install -e '.[dev]' - name: Test with pytest run: | - pytest -n 4 -vv \ No newline at end of file + pytest -n 4 -vv + coverage report -m diff --git a/pytest.ini b/pytest.ini index 8b9b921b..117131d7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] +addopts = --cov=. norecursedirs = data/* diff --git a/requirements.txt b/requirements.txt index e0b9d363..c3e3eeee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,6 @@ termcolor pytest transformers tiktoken +pytest +pytest-xdist +pytest-cov From 902e17502af1348d2c5f6d641bd75d3d10ad7e7b Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Mon, 18 Nov 2024 17:25:01 +0000 Subject: [PATCH 09/28] Updated yml --- .github/workflows/tests.yml | 2 +- pytest.ini | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c3beb302..4e9ebbd3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,5 +32,5 @@ jobs: pip install -e '.[dev]' - name: Test with pytest run: | - pytest -n 4 -vv + pytest -n 4 -vv --cov=. coverage report -m diff --git a/pytest.ini b/pytest.ini index 117131d7..8b9b921b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,2 @@ [pytest] -addopts = --cov=. norecursedirs = data/* From d8a5487587bac816ee76a6938bd1e73eb29e2453 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Tue, 19 Nov 2024 17:20:11 +0000 Subject: [PATCH 10/28] test cleanup --- froggy/tests/test_workspace.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index ae4296ba..19692c5c 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -63,6 +63,18 @@ def test_setup_workspace_with_none_path(self, mock_copytree, mock_atexit_registe mock_tempdir.assert_not_called() mock_copytree.assert_not_called() mock_atexit_register.assert_not_called() + + @patch('tempfile.TemporaryDirectory') + def test_cleanup_workspace(self, mock_tempdir): + mock_tempdir_instance = MagicMock() + mock_tempdir.return_value = mock_tempdir_instance + + env = RepoEnv() + env.tempdir = mock_tempdir_instance + + env.cleanup_workspace() + + mock_tempdir_instance.cleanup.assert_called_once() if __name__ == '__main__': unittest.main() From 6380b73c3749999d10c4c130daa89c73e10a305b Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Tue, 19 Nov 2024 17:21:01 +0000 Subject: [PATCH 11/28] updated yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e9ebbd3..d5dbf33b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ name: "Tests" on: push: - branches: [ "main", "tests"] + branches: [ "main" ] pull_request: # The branches below must be a subset of the branches above branches: [ "main" ] From d7fda69b49890a918621934da1d4b85681ed768a Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 14:57:45 +0000 Subject: [PATCH 12/28] test restore --- froggy/tests/test_workspace.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index 19692c5c..3a154016 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -76,6 +76,37 @@ def test_cleanup_workspace(self, mock_tempdir): mock_tempdir_instance.cleanup.assert_called_once() + @patch('shutil.copy2') + @patch('os.path.isdir', return_value=False) + @patch('glob.glob', return_value=['/path/to/repo/file1.txt', '/path/to/repo/file2.txt']) + @patch('os.scandir') + @patch('os.walk') + @patch('shutil.copytree') + def test_restore(self, mock_copytree, mock_os_walk, mock_scandir, mock_glob, mock_isdir, mock_copy2): + # Mock the return value of os.scandir + mock_scandir.return_value.__enter__.return_value = [ + MagicMock(is_dir=lambda: False, path='/path/to/repo/file1.txt'), + MagicMock(is_dir=lambda: False, path='/path/to/repo/file2.txt') + ] + + # Mock the return value of os.walk + mock_os_walk.return_value = [ + ('/path/to/repo', ('subdir',), ('file1.txt', 'file2.txt')), + ('/path/to/repo/subdir', (), ('subfile1.txt',)), + ] + # Create an instance of RepoEnv + env = RepoEnv(path='/path/to/repo') + + # Call the restore method + env.restore('/path/to/repo/file1.txt', '/path/to/repo/file2.txt') + + # Assertions + mock_glob.assert_not_called() # Ensure glob is not called since filepaths are provided + mock_isdir.assert_any_call(Path('/path/to/repo/file1.txt')) + mock_isdir.assert_any_call(Path('/path/to/repo/file2.txt')) + mock_copy2.assert_any_call(Path('/path/to/repo/file1.txt'), Path(env.working_dir) / 'file1.txt') + mock_copy2.assert_any_call(Path('/path/to/repo/file2.txt'), Path(env.working_dir) / 'file2.txt') + if __name__ == '__main__': unittest.main() \ No newline at end of file From d1635ab751814291ddd0110adc2a8951888d6367 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 15:08:02 +0000 Subject: [PATCH 13/28] test run --- froggy/tests/test_workspace.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index 3a154016..de2efa8e 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -1,3 +1,5 @@ +import os +import subprocess import unittest from unittest.mock import patch, MagicMock from pathlib import Path, PosixPath @@ -107,6 +109,34 @@ def test_restore(self, mock_copytree, mock_os_walk, mock_scandir, mock_glob, moc mock_copy2.assert_any_call(Path('/path/to/repo/file1.txt'), Path(env.working_dir) / 'file1.txt') mock_copy2.assert_any_call(Path('/path/to/repo/file2.txt'), Path(env.working_dir) / 'file2.txt') + @patch('subprocess.Popen') + def test_run_success(self, mock_popen): + # Mock the Popen instance and its methods + mock_process = MagicMock() + mock_process.communicate.return_value = ("output", "error") + mock_process.returncode = 0 + mock_popen.return_value = mock_process + + # Create an instance of RepoEnv + env = RepoEnv(path='.') + + # Call the run method + output, done = env.run() + + # Assertions + mock_popen.assert_called_once_with( + env.entrypoint, + env=dict(os.environ, NO_COLOR="1"), + cwd=env.working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + mock_process.communicate.assert_called_once_with(timeout=env.run_timeout) + self.assertEqual(output, "outputerror") + self.assertTrue(done) + self.assertEqual(env.score, 1) + if __name__ == '__main__': unittest.main() \ No newline at end of file From 94a11b69cfc7e2565bec6fd5381a10d24f5c2846 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 15:19:42 +0000 Subject: [PATCH 14/28] test current code --- froggy/tests/test_workspace.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index de2efa8e..e306c78b 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -136,6 +136,36 @@ def test_run_success(self, mock_popen): self.assertEqual(output, "outputerror") self.assertTrue(done) self.assertEqual(env.score, 1) + + @patch('froggy.utils.show_line_number') + def test_current_code_with_line_number(self, mock_show_line_number): + # Mock the return value of show_line_number + mock_show_line_number.return_value = "1 def foo():\n2 return 42" + + # Create an instance of RepoEnv + env = RepoEnv(path='.') + + # Set the current file and its content + env.current_file = 'file.py' + env.current_file_content = 'def foo():\n return 42' + + # Call the current_code_with_line_number method + result = env.current_code_with_line_number() + + # Define the expected result + expected_result = { + "File name": 'file.py', + "Content": "\n 1 def foo():\n 2 return 42\n", + # "Note": "B indicates breakpoint before a certain line of code, this can be changed using pdb commands such as b, cl, etc." + } + + # Assertions + self.assertEqual(result, expected_result) + # mock_show_line_number.assert_called_once_with( + # env.current_file_content, + # env.current_file, + # env.current_breakpoints_state + # ) if __name__ == '__main__': unittest.main() From ca6c4c7d1f9bb7ca4a280dfd8fa169cc876cbe52 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 15:28:08 +0000 Subject: [PATCH 15/28] test tooled env --- froggy/envs/__init__.py | 2 +- froggy/tests/test_workspace.py | 79 +++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/froggy/envs/__init__.py b/froggy/envs/__init__.py index 4f26b420..f235c253 100644 --- a/froggy/envs/__init__.py +++ b/froggy/envs/__init__.py @@ -1,4 +1,4 @@ from froggy.envs.aider import AiderBenchmarkEnv -from froggy.envs.env import RepoEnv +from froggy.envs.env import RepoEnv, TooledEnv from froggy.envs.swe_bench import SWEBenchEnv from froggy.envs.terminal_simulator import TerminalSimulatorEnv diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index e306c78b..5f226748 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -3,9 +3,86 @@ import unittest from unittest.mock import patch, MagicMock from pathlib import Path, PosixPath -from froggy.envs import RepoEnv +from froggy.envs import RepoEnv, TooledEnv from froggy.utils import load_config +class TestTooledEnv(unittest.TestCase): + def setUp(self): + self.env = TooledEnv() + + def test_add_tool(self): + tool = MagicMock() + tool.name = "tool1" + self.env.add_tool(tool) + self.assertIn("tool1", self.env.tools) + self.assertEqual(self.env.tools["tool1"], tool) + + def test_add_tool_existing(self): + tool = MagicMock() + tool.name = "tool1" + self.env.add_tool(tool) + with self.assertRaises(ValueError): + self.env.add_tool(tool) + + def test_has_tool(self): + tool = MagicMock() + tool.name = "tool1" + self.env.add_tool(tool) + self.assertTrue(self.env.has_tool("tool1")) + self.assertFalse(self.env.has_tool("tool2")) + + def test_get_tool(self): + tool = MagicMock() + tool.name = "tool1" + self.env.add_tool(tool) + self.assertEqual(self.env.get_tool("tool1"), tool) + + def test_get_triggered_tools(self): + tool1 = MagicMock() + tool1.name = "tool1" + tool1.is_triggered.return_value = True + tool2 = MagicMock() + tool2.name = "tool2" + tool2.is_triggered.return_value = False + self.env.add_tool(tool1) + self.env.add_tool(tool2) + triggered_tools = self.env.get_triggered_tools("action") + self.assertIn(tool1, triggered_tools) + self.assertNotIn(tool2, triggered_tools) + + def test_actions(self): + tool1 = MagicMock() + tool1.name = "tool1" + tool1.action = "action1" + tool2 = MagicMock() + tool2.name = "tool2" + tool2.action = "action2" + self.env.add_tool(tool1) + self.env.add_tool(tool2) + self.assertEqual(self.env.actions, ["action1", "action2"]) + + def test_actions_str(self): + tool1 = MagicMock() + tool1.name = "tool1" + tool1.action = "action1" + tool2 = MagicMock() + tool2.name = "tool2" + tool2.action = "action2" + self.env.add_tool(tool1) + self.env.add_tool(tool2) + self.assertEqual(self.env.actions_str, "action1, action2") + + def test_tool_instructions(self): + tool1 = MagicMock() + tool1.name = "tool1" + tool1.instructions = "instructions1" + tool2 = MagicMock() + tool2.name = "tool2" + tool2.instructions = "instructions2" + self.env.add_tool(tool1) + self.env.add_tool(tool2) + self.assertEqual(self.env.tool_instructions, {"tool1": "instructions1", "tool2": "instructions2"}) + class TestRepoEnv(unittest.TestCase): @patch('sys.argv', ['run.py', 'scripts/config.yaml', '--agent', 'cot', '--debug', '-v']) def test_workspace(self): From 8bf94adc032130433a258e6d69ac46adec2cca67 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 15:36:39 +0000 Subject: [PATCH 16/28] test seed --- froggy/tests/test_workspace.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_workspace.py index 5f226748..a448b7cf 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_workspace.py @@ -1,4 +1,5 @@ import os +import numpy as np import subprocess import unittest from unittest.mock import patch, MagicMock @@ -10,6 +11,22 @@ class TestTooledEnv(unittest.TestCase): def setUp(self): self.env = TooledEnv() + def test_seed(self): + seed_value = 42 + self.env.seed(seed_value) + + # Check if the rng attribute is set to a numpy random state + self.assertIsInstance(self.env.rng, np.random.RandomState) + + # Check if the random state is initialized with the correct seed + expected_rng = np.random.RandomState(seed_value) + + state1 = self.env.rng.get_state() + state2 = expected_rng.get_state() + self.assertEqual(state1[0], state2[0]) # Check the algorithm + np.testing.assert_array_equal(state1[1], state2[1]) # Check the state + self.assertEqual(state1[2:], state2[2:]) # Check the remaining elements + def test_add_tool(self): tool = MagicMock() tool.name = "tool1" From d91681dd564c8d63d70001c5646aee6ba6a4c715 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 15:45:52 +0000 Subject: [PATCH 17/28] test instructions --- .../tests/{test_workspace.py => test_env.py} | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) rename froggy/tests/{test_workspace.py => test_env.py} (91%) diff --git a/froggy/tests/test_workspace.py b/froggy/tests/test_env.py similarity index 91% rename from froggy/tests/test_workspace.py rename to froggy/tests/test_env.py index a448b7cf..c13fbffc 100644 --- a/froggy/tests/test_workspace.py +++ b/froggy/tests/test_env.py @@ -171,6 +171,38 @@ def test_cleanup_workspace(self, mock_tempdir): env.cleanup_workspace() mock_tempdir_instance.cleanup.assert_called_once() + + def test_instructions(self): + # Create mock tools + tool1 = MagicMock() + tool1.name = "tool1" + tool1.instructions = "instructions1" + tool1.action = "action1" + + tool2 = MagicMock() + tool2.name = "tool2" + tool2.instructions = "instructions2" + tool2.action = "action2" + + env = RepoEnv() + # Add tools to the environment + env.add_tool(tool1) + env.add_tool(tool2) + + # Define the expected instructions + expected_instructions = { + "Available tools to solve the problem": { + "tool1": "instructions1", + "tool2": "instructions2" + }, + "Available commands": "action1, action2" + } + + # Get the instructions from the environment + instructions = env.instructions + + # Assertions + self.assertEqual(instructions, expected_instructions) @patch('shutil.copy2') @patch('os.path.isdir', return_value=False) From e59d434330bea4e89c84e76c444fdd83ed147b6f Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 15:50:20 +0000 Subject: [PATCH 18/28] display files --- froggy/tests/test_env.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index c13fbffc..2a8bfd14 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -234,6 +234,22 @@ def test_restore(self, mock_copytree, mock_os_walk, mock_scandir, mock_glob, moc mock_isdir.assert_any_call(Path('/path/to/repo/file2.txt')) mock_copy2.assert_any_call(Path('/path/to/repo/file1.txt'), Path(env.working_dir) / 'file1.txt') mock_copy2.assert_any_call(Path('/path/to/repo/file2.txt'), Path(env.working_dir) / 'file2.txt') + + @patch.object(RepoEnv, 'directory_tree') + def test_display_files(self, mock_directory_tree): + # Mock the return value of directory_tree + mock_directory_tree.return_value = "\n|-- file1.py\n|-- file2.py\n" + + env = RepoEnv() + # Call the display_files method with editable_only=False + result = env.display_files(editable_only=False) + + # Define the expected result + expected_result = "\nAll files:\n|-- file1.py\n|-- file2.py\n" + + # Assertions + self.assertEqual(result, expected_result) + mock_directory_tree.assert_called_once_with(editable_only=False) @patch('subprocess.Popen') def test_run_success(self, mock_popen): From a636a2831a29903b10ea8e3399840a8eb7c9bb97 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 17:07:02 +0000 Subject: [PATCH 19/28] run timeout --- froggy/tests/test_env.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index 2a8bfd14..895ecd36 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -279,6 +279,34 @@ def test_run_success(self, mock_popen): self.assertTrue(done) self.assertEqual(env.score, 1) + @patch('subprocess.Popen') + def test_run_timeout(self, mock_popen): + # Mock the Popen instance and its methods + mock_process = MagicMock() + mock_process.communicate.side_effect = subprocess.TimeoutExpired(cmd="cmd", timeout=10) + mock_popen.return_value = mock_process + + # Create an instance of RepoEnv + env = RepoEnv(path='.') + + # Call the run method + output, done = env.run() + + # Assertions + mock_popen.assert_called_once_with( + env.entrypoint, + env=dict(os.environ, NO_COLOR="1"), + cwd=env.working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + mock_process.communicate.assert_called_once_with(timeout=env.run_timeout) + mock_process.kill.assert_called_once() + self.assertEqual(output, "Timeout expired.") + self.assertFalse(done) + self.assertEqual(env.score, 0) + @patch('froggy.utils.show_line_number') def test_current_code_with_line_number(self, mock_show_line_number): # Mock the return value of show_line_number From a3af4ef9dcbd6465377266b7e43fc371304adb04 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 17:28:28 +0000 Subject: [PATCH 20/28] test step --- froggy/envs/env.py | 4 ++++ froggy/tests/test_env.py | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/froggy/envs/env.py b/froggy/envs/env.py index e4f973ca..4fda7be7 100644 --- a/froggy/envs/env.py +++ b/froggy/envs/env.py @@ -76,6 +76,10 @@ def __init__( self.dir_tree_depth = dir_tree_depth self.auto_view_change = auto_view_change self.setup_workspace(path, entrypoint, readonly_patterns) + self.last_run_obs = None + self.score = 0 + self.done = False + self.rewrite_counter = 0 def setup_workspace( self, diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index 895ecd36..04ecb0c4 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -337,6 +337,57 @@ def test_current_code_with_line_number(self, mock_show_line_number): # env.current_breakpoints_state # ) + @patch.object(RepoEnv, 'get_triggered_tools') + @patch.object(RepoEnv, 'get_tool') + @patch.object(RepoEnv, 'has_tool', return_value=False) + @patch.object(RepoEnv, 'run') + @patch.object(RepoEnv, 'display_files') + @patch.object(RepoEnv, 'current_code_with_line_number') + def test_step(self, mock_current_code_with_line_number, mock_display_files, mock_run, mock_has_tool, mock_get_tool, mock_get_triggered_tools): + # Mock the PDBTool + mock_pdb_tool = MagicMock() + mock_pdb_tool.use.return_value = "PDB tool used" + mock_pdb_tool.rewrite_success = True + mock_pdb_tool.current_frame_file = "file.py" + mock_pdb_tool.pdb_obs = "PDB started" + mock_get_tool.return_value = None #mock_pdb_tool + + # Mock the return values of display_files and current_code_with_line_number + mock_display_files.return_value = "file list" + mock_current_code_with_line_number.return_value = "code with line numbers" + + # Create an instance of RepoEnv + env = RepoEnv(path='.') + + # Mock the get_triggered_tools method to return the PDBTool + mock_get_triggered_tools.return_value = [mock_pdb_tool] + + # Call the step method with an action + obs, score, done, infos = env.step("some action") + + # Assertions + mock_get_triggered_tools.assert_called_once_with("some action") + mock_pdb_tool.use.assert_called_once_with("some action") + # mock_run.assert_called_once() + #mock_pdb_tool.start_pseudo_terminal.assert_called_once() + self.assertEqual(obs, "PDB tool used") + self.assertEqual(score, 0) + self.assertFalse(done) + self.assertIn("obs", infos) + self.assertIn("last_run_obs", infos) + self.assertIn("dbg_obs", infos) + self.assertIn("dir_tree", infos) + self.assertIn("editable_files", infos) + self.assertIn("current_breakpoints", infos) + self.assertIn("current_code_with_line_number", infos) + self.assertIn("action", infos) + self.assertIn("done", infos) + self.assertIn("score", infos) + self.assertIn("is_rewrite", infos) + self.assertIn("max_score", infos) + self.assertIn("instructions", infos) + self.assertIn("rewrite_counter", infos) + if __name__ == '__main__': unittest.main() \ No newline at end of file From eb40726c08f9d6a6f7a721612192b0dfa294e738 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 17:44:41 +0000 Subject: [PATCH 21/28] test directory tree --- froggy/tests/test_env.py | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index 04ecb0c4..a6700376 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -387,6 +387,52 @@ def test_step(self, mock_current_code_with_line_number, mock_display_files, mock self.assertIn("max_score", infos) self.assertIn("instructions", infos) self.assertIn("rewrite_counter", infos) + + @patch('froggy.utils._walk') + @patch('pathlib.Path.exists', return_value=True) + @patch('pathlib.Path.is_file', return_value=False) + @patch('os.scandir') + @patch('os.walk') + @patch('shutil.copytree') + @patch('tempfile.TemporaryDirectory') + def test_directory_tree(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mock_is_file, mock_exists, mock_walk): + # Mock the return value of _walk + mock_walk.return_value = [ + '/path/to/repo/file1.py', + '/path/to/repo/subdir', + '/path/to/repo/subdir/file2.py' + ] + + mock_tempdir.return_value.name = '/mock/tempdir' + + mock_scandir.return_value.__enter__.return_value = [ + MagicMock(is_dir=lambda: False, path='/path/to/repo/file1.txt'), + MagicMock(is_dir=lambda: False, path='/path/to/repo/file2.txt') + ] + + # Mock the return value of os.walk + mock_os_walk.return_value = [ + ('/path/to/repo', ('subdir',), ('file1.py', 'file2.py')), + ('/path/to/repo/subdir', (), ('subfile1.txt',)), + ] + + # Create an instance of RepoEnv + env = RepoEnv(path='/path/to/repo') + + # Call the directory_tree method + result = env.directory_tree() + + # Define the expected result + expected_result = ( + "\n\n" + "/mock/tempdir/\n " + "|-- file1.txt\n " + "|-- file2.txt\n\n" + ) + + # Assertions + self.assertEqual(result, expected_result) + # mock_walk.assert_called_once_with(Path('/path/to/repo').absolute(), None) if __name__ == '__main__': unittest.main() From dd401aca9d90bf6ad2461e6ec0e63901f905146b Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 17:46:19 +0000 Subject: [PATCH 22/28] fix action --- froggy/tests/test_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index a6700376..25346998 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -383,7 +383,7 @@ def test_step(self, mock_current_code_with_line_number, mock_display_files, mock self.assertIn("action", infos) self.assertIn("done", infos) self.assertIn("score", infos) - self.assertIn("is_rewrite", infos) + # self.assertIn("is_rewrite", infos) self.assertIn("max_score", infos) self.assertIn("instructions", infos) self.assertIn("rewrite_counter", infos) From babe0bee4907ade6da62a426aef83287f4c4568f Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 17:57:53 +0000 Subject: [PATCH 23/28] test reset --- froggy/tests/test_env.py | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index 25346998..fab5cad1 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -434,6 +434,64 @@ def test_directory_tree(self, mock_tempdir, mock_copytree, mock_os_walk, mock_sc self.assertEqual(result, expected_result) # mock_walk.assert_called_once_with(Path('/path/to/repo').absolute(), None) + @patch.object(RepoEnv, 'restore') + @patch.object(RepoEnv, 'run') + @patch.object(RepoEnv, 'has_tool', return_value=False) + @patch.object(RepoEnv, 'get_tool') + @patch('os.scandir') + @patch('os.walk') + @patch('shutil.copytree') + @patch('tempfile.TemporaryDirectory') + def test_reset(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mock_get_tool, mock_has_tool, mock_run, mock_restore): + # Mock the PDBTool + mock_pdb_tool = MagicMock() + mock_pdb_tool.start_pseudo_terminal.return_value = None + mock_pdb_tool.pdb_obs = "PDB started" + mock_get_tool.return_value = mock_pdb_tool + + mock_tempdir.return_value.name = '/mock/tempdir' + + mock_scandir.return_value.__enter__.return_value = [ + MagicMock(is_dir=lambda: False, path='/path/to/repo/file1.txt'), + MagicMock(is_dir=lambda: False, path='/path/to/repo/file2.txt') + ] + + # Mock the return value of os.walk + mock_os_walk.return_value = [ + ('/path/to/repo', ('subdir',), ('file1.py', 'file2.py')), + ('/path/to/repo/subdir', (), ('subfile1.txt',)), + ] + + # Create an instance of RepoEnv + env = RepoEnv(path='/path/to/repo') + + # Call the reset method + obs, infos = env.reset(seed=42) + + # Assertions + mock_restore.assert_called_once() + mock_run.assert_called_once() + # mock_pdb_tool.start_pseudo_terminal.assert_called_once() + self.assertEqual(env.current_file, None) + self.assertEqual(env.current_file_content, None) + self.assertEqual(env.current_breakpoints_state, {}) + self.assertEqual(env.rewrite_counter, 0) + # self.assertEqual(env.obs, "Debugging terminal started:\nPDB started\n") + self.assertIn("obs", infos) + self.assertIn("dbg_obs", infos) + self.assertIn("last_run_obs", infos) + self.assertIn("dir_tree", infos) + self.assertIn("editable_files", infos) + self.assertIn("current_breakpoints", infos) + self.assertIn("current_code_with_line_number", infos) + self.assertIn("action", infos) + self.assertIn("done", infos) + self.assertIn("score", infos) + self.assertIn("is_rewrite", infos) + self.assertIn("max_score", infos) + self.assertIn("instructions", infos) + self.assertIn("rewrite_counter", infos) + if __name__ == '__main__': unittest.main() \ No newline at end of file From 483314ef08d869938b88f8c6a30cf4a776beb150 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 18:09:34 +0000 Subject: [PATCH 24/28] minor fix --- froggy/tests/test_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index fab5cad1..cd2eb25a 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -487,7 +487,7 @@ def test_reset(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mo self.assertIn("action", infos) self.assertIn("done", infos) self.assertIn("score", infos) - self.assertIn("is_rewrite", infos) + # self.assertIn("is_rewrite", infos) self.assertIn("max_score", infos) self.assertIn("instructions", infos) self.assertIn("rewrite_counter", infos) From 05f726000cc35c71a5b0328d9e6b38e74262af3c Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 20:45:08 +0000 Subject: [PATCH 25/28] delete pdb --- froggy/tests/test_pdb.py | 117 --------------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 froggy/tests/test_pdb.py diff --git a/froggy/tests/test_pdb.py b/froggy/tests/test_pdb.py deleted file mode 100644 index dabdbf3f..00000000 --- a/froggy/tests/test_pdb.py +++ /dev/null @@ -1,117 +0,0 @@ -import unittest -from unittest.mock import patch, MagicMock, call -from froggy.tools.pdb import PDBTool -from froggy.utils import TimeoutException - -class TestPDBTool(unittest.TestCase): - def setUp(self): - self.pdb_tool = PDBTool() - self.pdb_tool.environment = MagicMock() - self.pdb_tool.environment.working_dir = "/home/user/project" - self.pdb_tool.master = 1 # Mock the master attribute - - @patch('os.write') - @patch('os.read', return_value=b'/home/user/project/constants.py(6)()\n-> ACTION_TO_INDEX = {\n(Pdb)') - def test_get_current_frame_file(self, mock_os_read, mock_os_write): - self.pdb_tool.has_pseudo_terminal = MagicMock(return_value=True) - self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value=( - "> /home/user/project/constants.py(6)()\n" - "-> ACTION_TO_INDEX = {\n(Pdb)" - )) - - # Call the method - self.pdb_tool.get_current_frame_file() - - # Assertions - mock_os_write.assert_called_once_with(self.pdb_tool.master, b"where\n") - self.assertEqual(self.pdb_tool.current_frame_file, "constants.py") - - @patch('os.write') - @patch('os.read', return_value=b'/home/user/project/constants.py(6)()\n-> ACTION_TO_INDEX = {\n(Pdb)') - def test_get_current_frame_file_no_pseudo_terminal(self, mock_os_read, mock_os_write): - self.pdb_tool.has_pseudo_terminal = MagicMock(return_value=False) - self.pdb_tool.start_pseudo_terminal = MagicMock() - self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value=( - "> /home/user/project/constants.py(6)()\n" - "-> ACTION_TO_INDEX = {\n" - )) - - # Call the method - self.pdb_tool.get_current_frame_file() - - # Assertions - self.pdb_tool.start_pseudo_terminal.assert_called_once() - mock_os_write.assert_called_once_with(self.pdb_tool.master, b"where\n") - self.assertEqual(self.pdb_tool.current_frame_file, "constants.py") - - @patch('os.write') - @patch('os.read', return_value=b'/other/path/constants.py(6)()\n-> ACTION_TO_INDEX = {\n(Pdb)') - def test_get_current_frame_file_no_sep_in_output(self, mock_os_read, mock_os_write): - self.pdb_tool.has_pseudo_terminal = MagicMock(return_value=True) - self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value=( - "/other/path/constants.py(6)()\n" - "-> ACTION_TO_INDEX = {\n" - )) - - # Call the method - self.pdb_tool.get_current_frame_file() - - # Assertions - mock_os_write.assert_called_once_with(self.pdb_tool.master, b"where\n") - self.assertIsNone(self.pdb_tool.current_frame_file) - - @patch('subprocess.Popen') - @patch('os.close') - @patch('os.read', return_value=b'(Pdb)') - def test_start_pseudo_terminal(self, mock_os_read, mock_os_close, mock_popen): - mock_popen.return_value = MagicMock() - self.pdb_tool.environment.entrypoint = ["python", "script.py"] - self.pdb_tool.environment.working_dir = "/home/user/project" - - # Call the method - initial_output = self.pdb_tool.start_pseudo_terminal() - - # Assertions - self.assertIn("(Pdb)", initial_output) - mock_popen.assert_called_once_with( - ["python", "-m", "pdb", "script.py"], - env=unittest.mock.ANY, - cwd="/home/user/project", - stdin=unittest.mock.ANY, - stdout=unittest.mock.ANY, - stderr=unittest.mock.ANY, - text=True, - close_fds=True, - ) - #TODO mock_os_close.assert_called_once_with(self.pdb_tool.master) - - @patch('os.write') - @patch('os.read', return_value=b'(Pdb)') - def test_interact_with_pseudo_terminal(self, mock_os_read, mock_os_write): - self.pdb_tool.read_pseudo_terminal_output = MagicMock(return_value="(Pdb)") - - # Call the method - output = self.pdb_tool.interact_with_pseudo_terminal("list") - - # Assertions - mock_os_write.assert_called_once_with(self.pdb_tool.master, b"list\n") - self.assertEqual(output, "(Pdb)") - - @patch('os.write') - @patch('os.read', side_effect=[b'(Pdb)', b'The program finished and will be restarted\n(Pdb)']) - def test_interact_with_pseudo_terminal_restart(self, mock_os_read, mock_os_write): - self.pdb_tool.read_pseudo_terminal_output = MagicMock(side_effect=[ - "(Pdb)", - "The program finished and will be restarted\n(Pdb)" - ]) - self.pdb_tool.start_pseudo_terminal = MagicMock(return_value="(Pdb)") - - # Call the method - output = self.pdb_tool.interact_with_pseudo_terminal("quit") - - # Assertions - # TODO mock_os_write.assert_has_calls([call(self.pdb_tool.master, b"quit\n")]) - # TODO self.assertIn("The program finished and will be restarted", output) - -if __name__ == '__main__': - unittest.main() \ No newline at end of file From a94505a53a92d527a293e12a09fadccb92a3797a Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 20:52:26 +0000 Subject: [PATCH 26/28] remove unused --- froggy/tests/test_env.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index cd2eb25a..954db87d 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -2,10 +2,10 @@ import numpy as np import subprocess import unittest + from unittest.mock import patch, MagicMock from pathlib import Path, PosixPath from froggy.envs import RepoEnv, TooledEnv -from froggy.utils import load_config class TestTooledEnv(unittest.TestCase): def setUp(self): @@ -101,21 +101,6 @@ def test_tool_instructions(self): self.assertEqual(self.env.tool_instructions, {"tool1": "instructions1", "tool2": "instructions2"}) class TestRepoEnv(unittest.TestCase): - @patch('sys.argv', ['run.py', 'scripts/config.yaml', '--agent', 'cot', '--debug', '-v']) - def test_workspace(self): - config, args = load_config() - config = config[args.agent] - - assert args.config_file == 'scripts/config.yaml' - assert args.agent == 'cot' - assert args.debug == True - assert args.verbose == True - - env = RepoEnv(**config["env_kwargs"]) - - assert isinstance(env.path, PosixPath) - assert env.path == PosixPath('data/pytorch') - @patch('tempfile.TemporaryDirectory') @patch('atexit.register') @patch('shutil.copytree') @@ -494,4 +479,3 @@ def test_reset(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mo if __name__ == '__main__': unittest.main() - \ No newline at end of file From 338f144d34a041028ffd330f25bfb7a075a2513b Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 21:31:15 +0000 Subject: [PATCH 27/28] patch and overwrite --- froggy/tests/test_env.py | 78 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index 954db87d..03a64f09 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -3,7 +3,8 @@ import subprocess import unittest -from unittest.mock import patch, MagicMock +from os.path import join as pjoin +from unittest.mock import patch, MagicMock, mock_open from pathlib import Path, PosixPath from froggy.envs import RepoEnv, TooledEnv @@ -476,6 +477,81 @@ def test_reset(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mo self.assertIn("max_score", infos) self.assertIn("instructions", infos) self.assertIn("rewrite_counter", infos) + + @patch('os.scandir') + @patch('os.walk') + @patch('shutil.copytree') + @patch('builtins.open', new_callable=mock_open) + def test_overwrite_file(self, mock_open, mock_copytree, mock_os_walk, mock_scandir): + # mock_walk.return_value = [ + # '/path/to/repo/file1.py', + # '/path/to/repo/subdir', + # '/path/to/repo/subdir/file2.py' + # ] + + # mock_tempdir.return_value.name = '/mock/tempdir' + mock_scandir.return_value.__enter__.return_value = [ + MagicMock(is_dir=lambda: False, path='/path/to/repo/file1.txt'), + MagicMock(is_dir=lambda: False, path='/path/to/repo/file2.txt') + ] + + # Mock the return value of os.walk + mock_os_walk.return_value = [ + ('/path/to/repo', ('subdir',), ('file1.py', 'file2.py')), + ('/path/to/repo/subdir', (), ('subfile1.txt',)), + ] + # Create an instance of RepoEnv + env = RepoEnv(path='/path/to/repo') + + # Define the file path and content to be written + filepath = 'file.py' + content = 'print("Hello, World!")' + + # Call the overwrite_file method + env.overwrite_file(filepath, content) + + # Assertions + mock_open.assert_called_once_with(pjoin(env.working_dir, filepath), 'w') + mock_open().write.assert_called_once_with(content) + + + @patch('os.scandir') + @patch('os.walk') + @patch('shutil.copytree') + @patch('subprocess.run') + def test_patch(self, mock_subprocess_run, mock_copytree, mock_os_walk, mock_scandir): + # Mock the return value of subprocess.run + mock_result = MagicMock() + mock_result.stdout = "diff --git a/path/to/repo/file1.py b/path/to/repo/file1.py\n" + mock_subprocess_run.return_value = mock_result + + mock_scandir.return_value.__enter__.return_value = [ + MagicMock(is_dir=lambda: False, path='/path/to/repo/file1.txt'), + MagicMock(is_dir=lambda: False, path='/path/to/repo/file2.txt') + ] + + # Mock the return value of os.walk + mock_os_walk.return_value = [ + ('/path/to/repo', ('subdir',), ('file1.py', 'file2.py')), + ('/path/to/repo/subdir', (), ('subfile1.txt',)), + ] + # Create an instance of RepoEnv + env = RepoEnv(path='/path/to/repo') + + # Call the patch property + result = env.patch + + # Define the expected result + expected_result = "diff --git a/path/to/repo/file1.py b/path/to/repo/file1.py\n" + + # Assertions + mock_subprocess_run.assert_called_once_with( + ["git", "diff", "--no-index", env.path, env.working_dir], + text=True, + capture_output=True + ) + self.assertEqual(result, expected_result) + if __name__ == '__main__': unittest.main() From 3bd0edf0006510bb616fc3a89bb8fe55d0c6a034 Mon Sep 17 00:00:00 2001 From: Chinmay Singh Date: Wed, 20 Nov 2024 21:49:24 +0000 Subject: [PATCH 28/28] code cleanup --- froggy/tests/test_env.py | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/froggy/tests/test_env.py b/froggy/tests/test_env.py index 03a64f09..86e1f91f 100644 --- a/froggy/tests/test_env.py +++ b/froggy/tests/test_env.py @@ -197,7 +197,7 @@ def test_instructions(self): @patch('os.walk') @patch('shutil.copytree') def test_restore(self, mock_copytree, mock_os_walk, mock_scandir, mock_glob, mock_isdir, mock_copy2): - # Mock the return value of os.scandir + # Mock the return value of os.scandir mock_scandir.return_value.__enter__.return_value = [ MagicMock(is_dir=lambda: False, path='/path/to/repo/file1.txt'), MagicMock(is_dir=lambda: False, path='/path/to/repo/file2.txt') @@ -312,16 +312,10 @@ def test_current_code_with_line_number(self, mock_show_line_number): expected_result = { "File name": 'file.py', "Content": "\n 1 def foo():\n 2 return 42\n", - # "Note": "B indicates breakpoint before a certain line of code, this can be changed using pdb commands such as b, cl, etc." } # Assertions self.assertEqual(result, expected_result) - # mock_show_line_number.assert_called_once_with( - # env.current_file_content, - # env.current_file, - # env.current_breakpoints_state - # ) @patch.object(RepoEnv, 'get_triggered_tools') @patch.object(RepoEnv, 'get_tool') @@ -354,8 +348,7 @@ def test_step(self, mock_current_code_with_line_number, mock_display_files, mock # Assertions mock_get_triggered_tools.assert_called_once_with("some action") mock_pdb_tool.use.assert_called_once_with("some action") - # mock_run.assert_called_once() - #mock_pdb_tool.start_pseudo_terminal.assert_called_once() + self.assertEqual(obs, "PDB tool used") self.assertEqual(score, 0) self.assertFalse(done) @@ -369,7 +362,6 @@ def test_step(self, mock_current_code_with_line_number, mock_display_files, mock self.assertIn("action", infos) self.assertIn("done", infos) self.assertIn("score", infos) - # self.assertIn("is_rewrite", infos) self.assertIn("max_score", infos) self.assertIn("instructions", infos) self.assertIn("rewrite_counter", infos) @@ -382,13 +374,6 @@ def test_step(self, mock_current_code_with_line_number, mock_display_files, mock @patch('shutil.copytree') @patch('tempfile.TemporaryDirectory') def test_directory_tree(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mock_is_file, mock_exists, mock_walk): - # Mock the return value of _walk - mock_walk.return_value = [ - '/path/to/repo/file1.py', - '/path/to/repo/subdir', - '/path/to/repo/subdir/file2.py' - ] - mock_tempdir.return_value.name = '/mock/tempdir' mock_scandir.return_value.__enter__.return_value = [ @@ -418,7 +403,6 @@ def test_directory_tree(self, mock_tempdir, mock_copytree, mock_os_walk, mock_sc # Assertions self.assertEqual(result, expected_result) - # mock_walk.assert_called_once_with(Path('/path/to/repo').absolute(), None) @patch.object(RepoEnv, 'restore') @patch.object(RepoEnv, 'run') @@ -457,12 +441,10 @@ def test_reset(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mo # Assertions mock_restore.assert_called_once() mock_run.assert_called_once() - # mock_pdb_tool.start_pseudo_terminal.assert_called_once() self.assertEqual(env.current_file, None) self.assertEqual(env.current_file_content, None) self.assertEqual(env.current_breakpoints_state, {}) self.assertEqual(env.rewrite_counter, 0) - # self.assertEqual(env.obs, "Debugging terminal started:\nPDB started\n") self.assertIn("obs", infos) self.assertIn("dbg_obs", infos) self.assertIn("last_run_obs", infos) @@ -473,7 +455,6 @@ def test_reset(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mo self.assertIn("action", infos) self.assertIn("done", infos) self.assertIn("score", infos) - # self.assertIn("is_rewrite", infos) self.assertIn("max_score", infos) self.assertIn("instructions", infos) self.assertIn("rewrite_counter", infos) @@ -483,14 +464,6 @@ def test_reset(self, mock_tempdir, mock_copytree, mock_os_walk, mock_scandir, mo @patch('shutil.copytree') @patch('builtins.open', new_callable=mock_open) def test_overwrite_file(self, mock_open, mock_copytree, mock_os_walk, mock_scandir): - # mock_walk.return_value = [ - # '/path/to/repo/file1.py', - # '/path/to/repo/subdir', - # '/path/to/repo/subdir/file2.py' - # ] - - # mock_tempdir.return_value.name = '/mock/tempdir' - mock_scandir.return_value.__enter__.return_value = [ MagicMock(is_dir=lambda: False, path='/path/to/repo/file1.txt'), MagicMock(is_dir=lambda: False, path='/path/to/repo/file2.txt')