From cd323abe2e248cdb59770e81229cc58bdda029d1 Mon Sep 17 00:00:00 2001 From: holamgadol Date: Wed, 5 Oct 2022 12:52:20 +0300 Subject: [PATCH 1/4] feat (utils): add move_files_threadpool function --- foliant/contrib/utils.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/foliant/contrib/utils.py b/foliant/contrib/utils.py index 5a5d881..37f0f08 100644 --- a/foliant/contrib/utils.py +++ b/foliant/contrib/utils.py @@ -1,4 +1,8 @@ +from concurrent.futures import ThreadPoolExecutor +from os import makedirs, listdir +from os.path import join from pathlib import Path +from shutil import move from typing import Union @@ -37,3 +41,24 @@ def prepend_file( with open(filepath, 'w', encoding='utf8') as f: f.write(processed_content) + + +def move_files_threadpool(src: Path, dest: Path, n_workers: int = 64): + """Move files from one directory to another with ThreadPoolExecutor + + :param src: path to source directory + :param dest: path to the destination directory + :param n_workers: number of workers in ThreadPool + """ + def move_files(src_paths, dest_dir): + for src_path in src_paths: + move(src_path, dest_dir) + + makedirs(dest, exist_ok=True) + files = [join(src, name) for name in listdir(src)] + chunksize = round(len(files) / n_workers) + chunksize = 1 if chunksize == 0 else chunksize + with ThreadPoolExecutor(n_workers) as exe: + for i in range(0, len(files), chunksize): + filenames = files[i:(i + chunksize)] + _ = exe.submit(move_files, filenames, dest) From f6dc111250969e201acebb2e62af4996aabdd731 Mon Sep 17 00:00:00 2001 From: holamgadol Date: Wed, 5 Oct 2022 12:52:56 +0300 Subject: [PATCH 2/4] release: bump version to 1.0.4 --- changelog.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 0b6577e..903260a 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,8 @@ +# 1.0.4 + +- utils: add `move_files_threadpool` function + + # 1.0.3 - PreprocessorExt: add `debug_msg` param to `_warning` method. diff --git a/setup.py b/setup.py index 868be01..07b3c74 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ description=SHORT_DESCRIPTION, long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', - version='1.0.3', + version='1.0.4', author='Daniil Minukhin', author_email='ddddsa@gmail.com', url='https://github.com/foliant-docs/foliantcontrib.utils', From b6b4fbd912aba55abef8b004eb3d0ef2385c9169 Mon Sep 17 00:00:00 2001 From: Timur Osmanov <54434686+TOsmanov@users.noreply.github.com> Date: Wed, 3 Apr 2024 09:51:05 +0300 Subject: [PATCH 3/4] Update preprocessor_ext.py --- .../preprocessors/utils/preprocessor_ext.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/foliant/preprocessors/utils/preprocessor_ext.py b/foliant/preprocessors/utils/preprocessor_ext.py index b871d2d..d3cf5f5 100644 --- a/foliant/preprocessors/utils/preprocessor_ext.py +++ b/foliant/preprocessors/utils/preprocessor_ext.py @@ -1,5 +1,6 @@ import re import traceback +import threading from pathlib import Path from typing import Callable @@ -10,6 +11,14 @@ from foliant.utils import output +def run_in_thread(fn): + def run(*k, **kw): + t = threading.Thread(target=fn, args=k, kwargs=kw) + t.start() + return t + return run + + def allow_fail(msg: str = 'Failed to process tag. Skipping.') -> Callable: """ If function func fails for some reason, warning is issued but preprocessor @@ -140,7 +149,9 @@ def _process_tags_for_all_files(self, at the end all files will be saved at once. ''' self.logger.info(log_msg) - for markdown_file_path in self.working_dir.rglob('*.md'): + + @run_in_thread + def process(self, markdown_file_path): self.current_filepath = Path(markdown_file_path) self.current_filename = str(self.current_filepath. relative_to(self.working_dir)) @@ -158,6 +169,8 @@ def _process_tags_for_all_files(self, self.buffer[markdown_file_path] = processed_content else: self.save_file(markdown_file_path, processed_content) + for markdown_file_path in self.working_dir.rglob('*.md'): + process(self, markdown_file_path) self.current_filename = '' for path, content in self.buffer.items(): @@ -170,7 +183,9 @@ def _process_all_files(self, buffer: bool = False) -> None: '''Apply function func to all Markdown-files in the working dir''' self.logger.info(log_msg) - for markdown_file_path in self.working_dir.rglob('*.md'): + + @run_in_thread + def process(markdown_file_path): self.current_filepath = Path(markdown_file_path) self.current_filename = str(self.current_filepath. relative_to(self.working_dir)) @@ -185,6 +200,8 @@ def _process_all_files(self, self.buffer[markdown_file_path] = processed_content else: self.save_file(markdown_file_path, processed_content) + for markdown_file_path in self.working_dir.rglob('*.md'): + process(markdown_file_path) self.current_filename = '' for path, content in self.buffer.items(): From 7c052f06b0b4e18e94b3bfc56456ab50ae3630ef Mon Sep 17 00:00:00 2001 From: Timur Osmanov Date: Fri, 29 Aug 2025 12:50:17 +0300 Subject: [PATCH 4/4] add: ci test --- .github/workflows/python-test.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/python-test.yml diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml new file mode 100644 index 0000000..9cd2e56 --- /dev/null +++ b/.github/workflows/python-test.yml @@ -0,0 +1,27 @@ +name: Python package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip3 install . + pip3 install --upgrade foliantcontrib.test_framework + - name: Test with unittest + run: | + python3 -m doctest docs/*.md &&\ + python3 -m unittest discover -v