From 4b65b58e2638dbfe84b43bde7814495b8dba09f1 Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 18 Oct 2024 07:37:11 +0300 Subject: [PATCH 1/8] refactor tests to explicitly test no regeneration of C files --- testing/cffi0/test_verify.py | 23 +++++++++++++++++++++-- testing/cffi0/test_verify2.py | 16 ---------------- testing/cffi0/test_vgen2.py | 13 ------------- 3 files changed, 21 insertions(+), 31 deletions(-) delete mode 100644 testing/cffi0/test_verify2.py delete mode 100644 testing/cffi0/test_vgen2.py diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py index dfe3f9331..17d7facfc 100644 --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -53,9 +53,11 @@ def test_module_type(): ffi = FFI() lib = ffi.verify() if hasattr(lib, '_cffi_python_module'): - print('verify got a PYTHON module') + pass + # print('verify got a PYTHON module') if hasattr(lib, '_cffi_generic_module'): - print('verify got a GENERIC module') + pass + # print('verify got a GENERIC module') expected_generic = (cffi.verifier._FORCE_GENERIC_ENGINE or '__pypy__' in sys.builtin_module_names) assert hasattr(lib, '_cffi_python_module') == (not expected_generic) @@ -2586,3 +2588,20 @@ def test_passing_large_list(): arg = list(range(20000000)) lib.passing_large_list(arg) # assert did not segfault + +def test_no_regen(): + from cffi.verifier import Verifier, _caller_dir_pycache + import os + ffi = FFI() + modulename = "_cffi_test_no_regen" + ffi.cdef("double sin(double x);") + lib = ffi.verify('#include ', libraries=lib_m, modulename=modulename) + assert lib.sin(1.23) == math.sin(1.23) + # Make sure that recompiling the same code does not rebuild the C file + cfile = os.path.join(ffi.verifier.tmpdir, f"{modulename}.c") + assert os.path.exists(cfile) + os.unlink(cfile) + assert not os.path.exists(cfile) + lib = ffi.verify('#include ', libraries=lib_m, modulename=modulename) + assert lib.sin(1.23) == math.sin(1.23) + assert not os.path.exists(cfile) diff --git a/testing/cffi0/test_verify2.py b/testing/cffi0/test_verify2.py deleted file mode 100644 index cbb7b9dba..000000000 --- a/testing/cffi0/test_verify2.py +++ /dev/null @@ -1,16 +0,0 @@ -import pytest -from .test_verify import * - -pytestmark = [ - pytest.mark.thread_unsafe(reason="FFI verifier is not thread-safe"), - # eliminate warning noise from common test modules that are repeatedly re-imported - pytest.mark.filterwarnings("ignore:reimporting:UserWarning"), -] - -# This test file runs normally after test_verify. We only clean up the .c -# sources, to check that it also works when we have only the .so. The -# tests should run much faster than test_verify. - -def setup_module(): - import cffi.verifier - cffi.verifier.cleanup_tmpdir(keep_so=True) diff --git a/testing/cffi0/test_vgen2.py b/testing/cffi0/test_vgen2.py deleted file mode 100644 index 34147c877..000000000 --- a/testing/cffi0/test_vgen2.py +++ /dev/null @@ -1,13 +0,0 @@ -import cffi.verifier -from .test_vgen import * - -# This test file runs normally after test_vgen. We only clean up the .c -# sources, to check that it also works when we have only the .so. The -# tests should run much faster than test_vgen. - -def setup_module(): - cffi.verifier.cleanup_tmpdir(keep_so=True) - cffi.verifier._FORCE_GENERIC_ENGINE = True - -def teardown_module(): - cffi.verifier._FORCE_GENERIC_ENGINE = False From 28339aa26eccbeb25b842389f57ab5a7bd55e753 Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 18 Oct 2024 07:43:10 +0300 Subject: [PATCH 2/8] when building, cd into tmpdir to shorten builddir name on windows --- src/cffi/verifier.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/cffi/verifier.py b/src/cffi/verifier.py index e392a2b7f..e494d1c8d 100644 --- a/src/cffi/verifier.py +++ b/src/cffi/verifier.py @@ -197,8 +197,20 @@ def _write_source(self, file=None): def _compile_module(self): # compile this C source + # Note: compilation will create artifacts in tmpdir + sourcefilename + # This can exceed the windows MAXPATH quite easily. To make it shorter, + # cd into tmpdir and make the sourcefilename relative to tmdir tmpdir = os.path.dirname(self.sourcefilename) - outputfilename = ffiplatform.compile(tmpdir, self.get_extension()) + olddir = os.getcwd() + os.chdir(tmpdir) + self.sourcefilename_orig = self.sourcefilename + try: + self.sourcefilename = os.path.relpath(self.sourcefilename) + output_rel_filename = ffiplatform.compile(tmpdir, self.get_extension()) + outputfilename = os.path.join(tmpdir, output_rel_filename) + finally: + os.chdir(olddir) + self.sourcefilename = self.sourcefilename_orig try: same = ffiplatform.samefile(outputfilename, self.modulefilename) except OSError: From df46ee1a5beaaabe52e098e908a43906d32f7559 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 29 Oct 2024 08:04:35 +0200 Subject: [PATCH 3/8] also remove '*.o' in cleanup_tmp, enable more windows tests --- .github/workflows/ci.yaml | 4 +--- src/cffi/verifier.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e91e720dc..7acbd4f45 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -353,9 +353,7 @@ jobs: env: CIBW_BUILD: ${{ matrix.spec }} CIBW_TEST_REQUIRES: pytest setuptools - CIBW_TEST_COMMAND: ${{ matrix.test_cmd || 'python -m pytest {package}/src/c' }} - # FIXME: /testing takes ~45min on Windows and has some failures... - # CIBW_TEST_COMMAND='python -m pytest {package}/src/c {package}/testing' + CIBW_TEST_COMMAND: ${{ matrix.test_cmd || 'python -m pytest {package}' }} run: | set -eux diff --git a/src/cffi/verifier.py b/src/cffi/verifier.py index e494d1c8d..8903a9247 100644 --- a/src/cffi/verifier.py +++ b/src/cffi/verifier.py @@ -284,7 +284,9 @@ def cleanup_tmpdir(tmpdir=None, keep_so=False): suffix = _get_so_suffixes()[0].lower() for fn in filelist: if fn.lower().startswith('_cffi_') and ( - fn.lower().endswith(suffix) or fn.lower().endswith('.c')): + fn.lower().endswith(suffix) or + fn.lower().endswith('.c') or + fn.lower().endswith('.o')): try: os.unlink(os.path.join(tmpdir, fn)) except OSError: From 672454d11946166ec67bb61729efb83d070bb35f Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 29 Oct 2024 09:45:27 +0200 Subject: [PATCH 4/8] switch function name in test --- testing/cffi0/test_verify.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py index 17d7facfc..b37ea9ee6 100644 --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -2249,10 +2249,10 @@ def test_implicit_unicode_on_windows(): def test_use_local_dir(): ffi = FFI() - lib = ffi.verify("", modulename="test_use_local_dir") + lib = ffi.verify("", modulename="_cffi_test_use_local_dir") this_dir = os.path.dirname(__file__) pycache_files = os.listdir(os.path.join(this_dir, '__pycache__')) - assert any('test_use_local_dir' in s for s in pycache_files) + assert any('_cffi_test_use_local_dir' in s for s in pycache_files) def test_define_known_value(): ffi = FFI() @@ -2594,14 +2594,14 @@ def test_no_regen(): import os ffi = FFI() modulename = "_cffi_test_no_regen" - ffi.cdef("double sin(double x);") + ffi.cdef("double cos(double x);") lib = ffi.verify('#include ', libraries=lib_m, modulename=modulename) - assert lib.sin(1.23) == math.sin(1.23) + assert lib.cos(1.23) == math.cos(1.23) # Make sure that recompiling the same code does not rebuild the C file cfile = os.path.join(ffi.verifier.tmpdir, f"{modulename}.c") assert os.path.exists(cfile) os.unlink(cfile) assert not os.path.exists(cfile) lib = ffi.verify('#include ', libraries=lib_m, modulename=modulename) - assert lib.sin(1.23) == math.sin(1.23) + assert lib.cos(1.23) == math.cos(1.23) assert not os.path.exists(cfile) From 8278928b889cc8fc83272312970ef43b8835775d Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 29 Oct 2024 10:18:31 +0200 Subject: [PATCH 5/8] make doubly sure the module is recompiled --- testing/cffi0/test_verify.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py index b37ea9ee6..c96c51ca8 100644 --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -2590,10 +2590,11 @@ def test_passing_large_list(): # assert did not segfault def test_no_regen(): - from cffi.verifier import Verifier, _caller_dir_pycache + from cffi.verifier import Verifier, _caller_dir_pycache, _FORCE_GENERIC_ENGINE import os ffi = FFI() - modulename = "_cffi_test_no_regen" + # make sure the module name is unique + modulename = "_cffi_test_no_regen" + str(_FORCE_GENERIC_ENGINE) ffi.cdef("double cos(double x);") lib = ffi.verify('#include ', libraries=lib_m, modulename=modulename) assert lib.cos(1.23) == math.cos(1.23) From ab338ccfd687a48cceb0e19debaf5858b7e4c9d4 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 29 Oct 2024 11:32:36 +0200 Subject: [PATCH 6/8] fix windows tests for distutils from setuptools>73 --- testing/cffi0/test_ownlib.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py index a744a72c2..86efcf41b 100644 --- a/testing/cffi0/test_ownlib.py +++ b/testing/cffi0/test_ownlib.py @@ -142,10 +142,15 @@ def setup_class(cls): return # try (not too hard) to find the version used to compile this python # no mingw - from distutils.msvc9compiler import get_build_version - version = get_build_version() - toolskey = "VS%0.f0COMNTOOLS" % version - toolsdir = os.environ.get(toolskey, None) + toolsdir = None + try: + # This will always fail on setuptools>73 which removes msvc9compiler + from distutils.msvc9compiler import get_build_version + version = get_build_version() + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + except Exception: + pass if toolsdir is None: return productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") From 8be5d8a5c6c671fd4afe32622740bfd2600fb0a1 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 29 Oct 2024 11:36:37 +0200 Subject: [PATCH 7/8] skip embedding tests on windows --- testing/embedding/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testing/embedding/__init__.py b/testing/embedding/__init__.py index e69de29bb..13bb20726 100644 --- a/testing/embedding/__init__.py +++ b/testing/embedding/__init__.py @@ -0,0 +1,5 @@ +import sys +import pytest + +if sys.platform == "win32": + pytest.skip("XXX fixme", allow_module_level=True) From 1e3f086bfc4d2f410938574ae3cc0d11d910c390 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 29 Oct 2024 12:18:39 +0200 Subject: [PATCH 8/8] make filename unique --- testing/cffi0/test_zdistutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/cffi0/test_zdistutils.py b/testing/cffi0/test_zdistutils.py index 0f88a8ae0..fe7dd90dc 100644 --- a/testing/cffi0/test_zdistutils.py +++ b/testing/cffi0/test_zdistutils.py @@ -94,7 +94,7 @@ def test_compile_module_explicit_filename(self): csrc = '/*hi there %s!2*/\n#include \n' % self v = Verifier(ffi, csrc, force_generic_engine=self.generic, libraries=[self.lib_m]) - basename = self.__class__.__name__[:10] + '_test_compile_module' + basename = self.__class__.__name__[:20] + '_test_compile_module' v.modulefilename = filename = str(udir.join(basename + '.so')) v.compile_module() assert filename == v.modulefilename