FastTokenizer/setup.py at master · mingruimingrui/FastTokenizer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import os
import sys
import platform
import subprocess

from setuptools import Extension, setup
from setuptools.command.build_ext import build_ext


class CMakeExtension(Extension):
    """CMake python extension class.

    Adapted from https://github.com/pybind/cmake_example.
    """

    def __init__(self, name, sourcedir=''):
        Extension.__init__(self, name, sources=[])
        self.sourcedir = os.path.abspath(sourcedir)


class CMakeBuild(build_ext):
    """CMake build_ext command class.

    Adapted from https://github.com/pybind/cmake_example.
    """

    def run(self):
        try:
            out = subprocess.check_output(['cmake', '--version'])

        except OSError:
            _msg = (
                "CMake must be installed to build the "
                "following extensions: {}"
            ).format(', '.join(e.name for e in self.extensions))
            raise RuntimeError(_msg)

        if platform.system() == "Windows":
            import re
            from distutils.version import LooseVersion

            cmake_version = re.search(r'version\s*([\d.]+)', out.decode())
            cmake_version = cmake_version.group(1)
            cmake_version = LooseVersion(cmake_version)
            if cmake_version < '3.1.0':
                raise RuntimeError("CMake >= 3.1.0 is required on Windows")

        for ext in self.extensions:
            self.build_extension(ext)

    def build_extension(self, ext):
        extdir = os.path.dirname(self.get_ext_fullpath(ext.name))
        extdir = os.path.abspath(extdir)
        # required for auto-detection of auxiliary "native" libs
        if not extdir.endswith(os.path.sep):
            extdir += os.path.sep

        cfg = 'Debug' if self.debug else 'Release'

        cmake_args = [
            'cmake', ext.sourcedir,
            '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
            '-DPYTHON_EXECUTABLE=' + sys.executable,
            '-DBUILD_SHARED_LIBS:BOOL=OFF',
            '-DBUILD_CLI:BOOL=OFF',
            '-DBUILD_PYTHON:BOOL=ON',
        ]

        if platform.system() == "Windows":
            cmake_args.append('-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(
                cfg.upper(), extdir))
            if sys.maxsize > 2**32:
                cmake_args += ['-A', 'x64']
        else:
            cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]

        env = os.environ.copy()
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp, exist_ok=True)
        subprocess.check_call(cmake_args, cwd=self.build_temp, env=env)
        subprocess.check_call(["make", "-j", "8"], cwd=self.build_temp)


with open('VERSION', 'r') as f:
    VERSION_INFO = f.read().strip()


with open('README.md', 'r', encoding='utf-8') as f:
    LONG_DESCRIPTION = f.read()


setup(
    name='fasttokenizer',
    version=VERSION_INFO,
    author='Wang Ming Rui',
    author_email='mingruimingrui@hotmail.com',
    description='Fast unicode based tokenizer for MT written in C++',
    long_description=LONG_DESCRIPTION,
    long_description_content_type='text/markdown',
    python_requires='>=3.6',

    packages=['fasttokenizer'],
    package_dir={'fasttokenizer': 'bindings/python/fasttokenizer'},
    ext_modules=[CMakeExtension('fasttokenizer._fasttokenizer')],
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'Programming Language :: C++',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3 :: Only',
        'Topic :: Scientific/Engineering',
        'Topic :: Software Development',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Topic :: Text Processing :: Linguistic',
    ],

    cmdclass=dict(build_ext=CMakeBuild),
    zip_safe=False,
)