diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..cb9f69c37 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# style: fix flake8 violations across the codebase +b58be01bc127516870106b9ccd3dab324d1fcd51 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..23c38d7a7 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,50 @@ +name: Tests + +env: + PYTHON_VERSION: "3.10" + +on: + push: + branches: ["**-redpanda"] + pull_request: + branches: ["**-redpanda"] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-test.txt + pip install -e . + + - name: Run tests + run: pytest + + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install flake8 + run: | + python -m pip install --upgrade pip + pip install flake8~=6.1.0 + + - name: Run flake8 + run: flake8 --config tox.ini diff --git a/ducktape/cluster/remoteaccount.py b/ducktape/cluster/remoteaccount.py index f498a518d..51371b116 100644 --- a/ducktape/cluster/remoteaccount.py +++ b/ducktape/cluster/remoteaccount.py @@ -12,17 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import paramiko -# Constant that is responsible for updating ssh session keys after -# more than REKEY_BYTES data passed through the connection -# Changing it due to https://github.com/redpanda-data/redpanda/issues/6792 -paramiko.packet.Packetizer.REKEY_BYTES = pow(2, 32) # noqa - from contextlib import contextmanager import logging import os -from paramiko import SSHClient, SSHConfig, MissingHostKeyPolicy -from paramiko.ssh_exception import SSHException, NoValidConnectionsError import shutil import signal import socket @@ -30,10 +22,19 @@ import tempfile import warnings +import paramiko +from paramiko import SSHClient, SSHConfig, MissingHostKeyPolicy +from paramiko.ssh_exception import SSHException, NoValidConnectionsError + from ducktape.utils.http_utils import HttpMixin from ducktape.utils.util import wait_until from ducktape.errors import DucktapeError +# Constant that is responsible for updating ssh session keys after +# more than REKEY_BYTES data passed through the connection +# Changing it due to https://github.com/redpanda-data/redpanda/issues/6792 +paramiko.packet.Packetizer.REKEY_BYTES = pow(2, 32) + def check_ssh(method): def wrapper(self, *args, **kwargs): @@ -190,12 +191,13 @@ def _set_ssh_client(self): client = SSHClient() client.set_missing_host_key_policy(IgnoreMissingHostKeyPolicy()) - try: - ip = socket.gethostbyname(self.externally_routable_ip) - except socket.gaierror as e: - ip = None - self._log(logging.WARN, - f"error resolving {self.externally_routable_ip}: {e}") + ip = None + if self.externally_routable_ip: + try: + ip = socket.gethostbyname(self.externally_routable_ip) + except socket.gaierror as e: + self._log(logging.WARN, + f"error resolving {self.externally_routable_ip}: {e}") self._log(logging.DEBUG, f"ssh_config: {self.ssh_config}, external IP: {ip}") @@ -644,7 +646,7 @@ def create_file(self, path, contents): # TODO: what happens if the base part of the path does not exist? node_reachable = False disk_space = "Unknown" - + try: self._log(logging.DEBUG, f"Let's create or overwrite file at: {path}") @@ -670,13 +672,13 @@ def create_file(self, path, contents): except Exception as disk_error: self._log(logging.ERROR, f"Failed to retrieve disk space: {disk_error}") else: - self._log(logging.ERROR, f"Remote directory does not exist: {dir_path}") + self._log(logging.ERROR, f"Remote directory does not exist: {dir_path}") else: self._log(logging.ERROR, "No parent directory to validate") - + except Exception as debug_error: self._log(logging.ERROR, f"Debugging failed: {debug_error}") - + raise Exception( f"Node reachable={node_reachable}" f"Available disk space: {disk_space} bytes" diff --git a/ducktape/command_line/parse_args.py b/ducktape/command_line/parse_args.py index b2006ed35..069a176bb 100644 --- a/ducktape/command_line/parse_args.py +++ b/ducktape/command_line/parse_args.py @@ -215,5 +215,6 @@ def parse_args(args): sys.exit(0) # make list of deflake exclude exceptions if parsed_args_dict["deflake_exclude_exceptions"]: - parsed_args_dict["deflake_exclude_exceptions"] = [d for d in str(parsed_args_dict["deflake_exclude_exceptions"]).split(",") if d] + parsed_args_dict["deflake_exclude_exceptions"] = [ + d for d in str(parsed_args_dict["deflake_exclude_exceptions"]).split(",") if d] return parsed_args_dict diff --git a/ducktape/tests/runner.py b/ducktape/tests/runner.py index f19c8a295..cff678911 100644 --- a/ducktape/tests/runner.py +++ b/ducktape/tests/runner.py @@ -44,6 +44,7 @@ # After such an issues occurs, later test results should be treated with suspicion. CORRUPTING_FAILURE_TAG = "CORRUPTING_FAILURE" + class Receiver(object): def __init__(self, min_port, max_port): assert min_port <= max_port, "Expected min_port <= max_port, but instead: min_port: %s, max_port %s" % \ @@ -265,10 +266,14 @@ def run_all_tests(self): if self._expect_client_requests: try: - event = self.receiver.recv(timeout=int(int(self.session_context.test_runner_timeout) * 1.2)) # test_runner_timeout is handled in the client. adding 20% seconds on top, to guard against client not being able to report to the server + # test_runner_timeout is handled in the client; + # adding 20% on top to guard against client not being able to report to the server + timeout = int(int(self.session_context.test_runner_timeout) * 1.2) + event = self.receiver.recv(timeout=timeout) self._handle(event) except Exception as e: - err_str = "Exception receiving message: %s: %s, active_tests: \n %s \n" % (str(type(e)), str(e), self.active_tests_debug()) + err_str = "Exception receiving message: %s: %s, active_tests: \n %s \n" % ( + str(type(e)), str(e), self.active_tests_debug()) err_str += "\n" + traceback.format_exc(limit=16) self._log(logging.ERROR, err_str) @@ -335,7 +340,6 @@ def _run_single_test(self, test_context): self.client_report[test_key]["name"] = proc.name self.client_report[test_key]["runner_start_time"] = time.time() - def _preallocate_subcluster(self, test_context): """Preallocate the subcluster which will be used to run the test. diff --git a/ducktape/tests/runner_client.py b/ducktape/tests/runner_client.py index 84fe8311c..0b5c7ed9a 100644 --- a/ducktape/tests/runner_client.py +++ b/ducktape/tests/runner_client.py @@ -153,7 +153,7 @@ def __init__( fail_bad_cluster_utilization: bool, deflake_num: int, test_runner_timeout: int, - deflake_exlude_exceptions: List[str]=None + deflake_exlude_exceptions: List[str] = None ): signal.signal(signal.SIGTERM, self._sigterm_handler) # register a SIGTERM handler @@ -279,7 +279,8 @@ def run(self): self.log(logging.INFO, msg) if test_status == FAIL and run_summary: if self.deflake_enabled and self.stop_deflake_retries("\n".join(run_summary)): - self.log(logging.INFO, "exception matches deflake exclude exceptions. stopping deflake retries...") + self.log(logging.INFO, + "exception matches deflake exclude exceptions. stopping deflake retries...") stopped_deflake = True break except BaseException as e: @@ -322,7 +323,10 @@ def run(self): self.test_context = None self.test = None - def process_run_summaries(self, run_summaries: List[List[str]], test_status: TestStatus, stopped_deflake = False, internal_exception: str | None = None) -> List[str]: + def process_run_summaries( + self, run_summaries: List[List[str]], test_status: TestStatus, + stopped_deflake=False, + internal_exception: str | None = None) -> List[str]: """ Converts individual run summaries (there may be multiple if deflake is enabled) into a single run summary diff --git a/tests/tests/check_test.py b/tests/tests/check_test.py index 2b851d0a8..d8761a627 100644 --- a/tests/tests/check_test.py +++ b/tests/tests/check_test.py @@ -94,7 +94,9 @@ def check_from_function_multiline(self): """If the function has a docstring, the description should come from the function""" context = TestContext(session_context=ducktape_mock.session_context(), cls=DummyTest, function=DummyTest.test_multiline_function_description) - assert context.description == "function description\nwith multiple lines, including\nleading and trailing whitespace" + assert context.description == ( + "function description\nwith multiple lines, including\n" + "leading and trailing whitespace") def check_from_class(self): """If the test method has no docstring, description should come from the class docstring"""