Update preprocessor_ext.py

.github/workflows/python-test.yml

@@ -0,0 +1,27 @@
+name: Python package
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip3 install .
+          pip3 install --upgrade foliantcontrib.test_framework
+      - name: Test with unittest
+        run: |
+          python3 -m doctest docs/*.md &&\
+            python3 -m unittest discover -v

changelog.md

@@ -1,3 +1,8 @@
+# 1.0.4
+
+-    utils: add `move_files_threadpool` function
+
+
 # 1.0.3
 
 -    PreprocessorExt: add `debug_msg` param to `_warning` method.

foliant/contrib/utils.py

@@ -1,4 +1,8 @@
+from concurrent.futures import ThreadPoolExecutor
+from os import makedirs, listdir
+from os.path import join
 from pathlib import Path
+from shutil import move
 from typing import Union
 
 
@@ -37,3 +41,24 @@ def prepend_file(
 
     with open(filepath, 'w', encoding='utf8') as f:
         f.write(processed_content)
+
+
+def move_files_threadpool(src: Path, dest: Path, n_workers: int = 64):
+    """Move files from one directory to another with ThreadPoolExecutor
+
+    :param src: path to source directory
+    :param dest: path to the destination directory
+    :param n_workers: number of workers in ThreadPool
+    """
+    def move_files(src_paths, dest_dir):
+        for src_path in src_paths:
+            move(src_path, dest_dir)
+
+    makedirs(dest, exist_ok=True)
+    files = [join(src, name) for name in listdir(src)]
+    chunksize = round(len(files) / n_workers)
+    chunksize = 1 if chunksize == 0 else chunksize
+    with ThreadPoolExecutor(n_workers) as exe:
+        for i in range(0, len(files), chunksize):
+            filenames = files[i:(i + chunksize)]
+            _ = exe.submit(move_files, filenames, dest)

foliant/preprocessors/utils/preprocessor_ext.py

@@ -1,5 +1,6 @@
 import re
 import traceback
+import threading
 
 from pathlib import Path
 from typing import Callable
@@ -10,6 +11,14 @@
 from foliant.utils import output
 
 
+def run_in_thread(fn):
+    def run(*k, **kw):
+        t = threading.Thread(target=fn, args=k, kwargs=kw)
+        t.start()
+        return t
+    return run
+
+
 def allow_fail(msg: str = 'Failed to process tag. Skipping.') -> Callable:
     """
     If function func fails for some reason, warning is issued but preprocessor
@@ -140,7 +149,9 @@ def _process_tags_for_all_files(self,
                        at the end all files will be saved at once.
         '''
         self.logger.info(log_msg)
-        for markdown_file_path in self.working_dir.rglob('*.md'):
+
+        @run_in_thread
+        def process(self, markdown_file_path):
             self.current_filepath = Path(markdown_file_path)
             self.current_filename = str(self.current_filepath.
                                         relative_to(self.working_dir))
@@ -158,6 +169,8 @@ def _process_tags_for_all_files(self,
                     self.buffer[markdown_file_path] = processed_content
                 else:
                     self.save_file(markdown_file_path, processed_content)
+        for markdown_file_path in self.working_dir.rglob('*.md'):
+            process(self, markdown_file_path)
         self.current_filename = ''
 
         for path, content in self.buffer.items():
@@ -170,7 +183,9 @@ def _process_all_files(self,
                            buffer: bool = False) -> None:
         '''Apply function func to all Markdown-files in the working dir'''
         self.logger.info(log_msg)
-        for markdown_file_path in self.working_dir.rglob('*.md'):
+
+        @run_in_thread
+        def process(markdown_file_path):
             self.current_filepath = Path(markdown_file_path)
             self.current_filename = str(self.current_filepath.
                                         relative_to(self.working_dir))
@@ -185,6 +200,8 @@ def _process_all_files(self,
                     self.buffer[markdown_file_path] = processed_content
                 else:
                     self.save_file(markdown_file_path, processed_content)
+        for markdown_file_path in self.working_dir.rglob('*.md'):
+            process(markdown_file_path)
         self.current_filename = ''
 
         for path, content in self.buffer.items():

setup.py

@@ -16,7 +16,7 @@
     description=SHORT_DESCRIPTION,
     long_description=LONG_DESCRIPTION,
     long_description_content_type='text/markdown',
-    version='1.0.3',
+    version='1.0.4',
     author='Daniil Minukhin',
     author_email='ddddsa@gmail.com',
     url='https://github.com/foliant-docs/foliantcontrib.utils',