From af16262052063b2e9c34af117dae584bae5a8849 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Tue, 14 Apr 2020 16:24:47 -0700 Subject: [PATCH 001/102] Modified pandoc_reader by adding capabilities to read YAML header --- .gitignore | 1 + README.md | 21 ++++--------- __init__.py | 1 + pandoc_reader.py | 79 +++++++++++++++++++++++++++++++++--------------- 4 files changed, 62 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index c5a9333..892a243 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc .idea *~ +__pycache__ diff --git a/README.md b/README.md index 08055fc..61698ce 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,16 @@ -pandoc_reader -============= +# pandoc_reader A pandoc [markdown] reader plugin for [pelican] +## Requirements -Requirements ------------- +- [pandoc] in $PATH - - [pandoc] in $PATH - - -Installation ------------- +## Installation Instructions for installation of pelican plugins can be obtained from the [pelican plugin manual](https://github.com/getpelican/pelican-plugins/blob/master/Readme.rst). - -Configuration -------------- +## Configuration Additional command line parameters can be passed to pandoc via the PANDOC_ARGS parameter. @@ -37,8 +30,7 @@ PANDOC_EXTENSIONS parameter. '-citations' ] -Contributing ------------- +## Contributing 1. Fork it 2. Create your feature branch (`git checkout -b my-new-feature`) @@ -46,7 +38,6 @@ Contributing 4. Push to the branch (`git push origin my-new-feature`) 5. Create new Pull Request - [markdown]: http://daringfireball.net/projects/markdown/ [pandoc]: http://johnmacfarlane.net/pandoc/ [pelican]: http://getpelican.com diff --git a/__init__.py b/__init__.py index 31d0856..d2d63b3 100644 --- a/__init__.py +++ b/__init__.py @@ -1 +1,2 @@ +"""Importing pandoc_reader package.""" from .pandoc_reader import * diff --git a/pandoc_reader.py b/pandoc_reader.py index 87c7735..b7c907d 100644 --- a/pandoc_reader.py +++ b/pandoc_reader.py @@ -1,48 +1,77 @@ +"""Reader that processes Pandoc Markdown and returns HTML 5.""" import subprocess + from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open + class PandocReader(BaseReader): + """Process files written in Pandoc Markdown.""" + enabled = True file_extensions = ['md', 'markdown', 'mkd', 'mdown'] - def read(self, filename): - with pelican_open(filename) as fp: - text = list(fp.splitlines()) + def read(self, source_path): + """Parse Pandoc Markdown and return HTNL 5 output and metadata.""" + content = "" - metadata = {} - for i, line in enumerate(text): - kv = line.split(':', 1) - if len(kv) == 2: - name, value = kv[0].lower(), kv[1].strip() - metadata[name] = self.process_metadata(name, value) - else: - content = "\n".join(text[i:]) - break + with pelican_open(source_path) as file_content: + content = file_content - extra_args = self.settings.get('PANDOC_ARGS', []) - extensions = self.settings.get('PANDOC_EXTENSIONS', '') - if isinstance(extensions, list): - extensions = ''.join(extensions) + # Parse metadata + metadata = self._process_metadata(list(content.splitlines())) - pandoc_cmd = ["pandoc", "--from=markdown" + extensions, "--to=html5"] - pandoc_cmd.extend(extra_args) + # Get arguments and extensions + extra_args = self.settings.get('PANDOC_ARGS', []) + extensions = self.settings.get('PANDOC_EXTENSIONS', '') + if isinstance(extensions, list): + extensions = ''.join(extensions) - proc = subprocess.Popen(pandoc_cmd, - stdin = subprocess.PIPE, - stdout = subprocess.PIPE) + # Construct Pandoc command + pandoc_cmd = [ + "pandoc", "-f", "markdown" + extensions, "-t", "html5" + ] + pandoc_cmd.extend(extra_args) - output = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') - status = proc.wait() - if status: - raise subprocess.CalledProcessError(status, pandoc_cmd) + # Execute and retrieve HTML 5 output + proc = subprocess.Popen( + pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + output = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') + status = proc.wait() + if status: + raise subprocess.CalledProcessError(status, pandoc_cmd) return output, metadata + def _process_metadata(self, text): + """Process YAML metadata and export.""" + metadata = {} + + # Find the end of the YAML block + lines = text[1:] + yaml_end = "" + for line_num, line in enumerate(lines): + if line in ["---", "..."]: + yaml_end = line_num + break + + # Process the YAML block + for line in lines[:yaml_end]: + metalist = line.split(':', 1) + if len(metalist) == 2: + key, value = metalist[0].lower(), metalist[1].strip() + metadata[key] = self.process_metadata(key, value) + return metadata + + def add_reader(readers): + """Add the PandocReader as the reader for all Pandoc Markdown files.""" for ext in PandocReader.file_extensions: readers.reader_classes[ext] = PandocReader + def register(): + """Register the PandocReader.""" signals.readers_init.connect(add_reader) From 61b221e780cef40082cae5d56c1e9d487a435789 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 15 Apr 2020 12:28:22 -0700 Subject: [PATCH 002/102] Added error checking and the ability to not encode {static} reference in HTML output --- pandoc_reader.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pandoc_reader.py b/pandoc_reader.py index b7c907d..b31bfb9 100644 --- a/pandoc_reader.py +++ b/pandoc_reader.py @@ -5,6 +5,9 @@ from pelican.readers import BaseReader from pelican.utils import pelican_open +STATIC_LINK_ENCODED = "%7Bstatic%7D" +STATIC_LINK_UNENCODED = "{static}" + class PandocReader(BaseReader): """Process files written in Pandoc Markdown.""" @@ -13,7 +16,7 @@ class PandocReader(BaseReader): file_extensions = ['md', 'markdown', 'mkd', 'mdown'] def read(self, source_path): - """Parse Pandoc Markdown and return HTNL 5 output and metadata.""" + """Parse Pandoc Markdown and return HTML 5 output and metadata.""" content = "" with pelican_open(source_path) as file_content: @@ -38,25 +41,39 @@ def read(self, source_path): proc = subprocess.Popen( pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) - output = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') + + html = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') status = proc.wait() if status: raise subprocess.CalledProcessError(status, pandoc_cmd) - return output, metadata + # Replace all occurrences of %7Bfilename%7D to {filename} + # so that static links are resolved by pelican + html = html.replace(STATIC_LINK_ENCODED, STATIC_LINK_UNENCODED) + + return html, metadata def _process_metadata(self, text): """Process YAML metadata and export.""" metadata = {} + # Check that the first line of the file + # starts with a YAML header + if text[0].strip() not in ["---", "..."]: + raise Exception("Could not find metadata header '---' or '...'") + # Find the end of the YAML block lines = text[1:] yaml_end = "" for line_num, line in enumerate(lines): - if line in ["---", "..."]: + if line.strip() in ["---", "..."]: yaml_end = line_num break + # Check if the end of the YAML block was found + if not yaml_end: + raise Exception("Could not find end of metadata block.") + # Process the YAML block for line in lines[:yaml_end]: metalist = line.split(':', 1) From 5a9ec41df61734c7aa094c91e5ced0485625717b Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 15 Apr 2020 13:16:59 -0700 Subject: [PATCH 003/102] Added ability to unencode attach and static link directives --- pandoc_reader.py | 65 ++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/pandoc_reader.py b/pandoc_reader.py index b31bfb9..768242a 100644 --- a/pandoc_reader.py +++ b/pandoc_reader.py @@ -5,8 +5,11 @@ from pelican.readers import BaseReader from pelican.utils import pelican_open -STATIC_LINK_ENCODED = "%7Bstatic%7D" -STATIC_LINK_UNENCODED = "{static}" +ENCODED_LINK_TO_RAW_LINK_MAP = { + "%7Bstatic%7D": "{static}", + "%7Battach%7D": "{attach}", + "%7Bfilename%7D": "{filename}" +} class PandocReader(BaseReader): @@ -22,34 +25,36 @@ def read(self, source_path): with pelican_open(source_path) as file_content: content = file_content - # Parse metadata - metadata = self._process_metadata(list(content.splitlines())) - - # Get arguments and extensions - extra_args = self.settings.get('PANDOC_ARGS', []) - extensions = self.settings.get('PANDOC_EXTENSIONS', '') - if isinstance(extensions, list): - extensions = ''.join(extensions) - - # Construct Pandoc command - pandoc_cmd = [ - "pandoc", "-f", "markdown" + extensions, "-t", "html5" - ] - pandoc_cmd.extend(extra_args) - - # Execute and retrieve HTML 5 output - proc = subprocess.Popen( - pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE - ) - - html = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') - status = proc.wait() - if status: - raise subprocess.CalledProcessError(status, pandoc_cmd) - - # Replace all occurrences of %7Bfilename%7D to {filename} - # so that static links are resolved by pelican - html = html.replace(STATIC_LINK_ENCODED, STATIC_LINK_UNENCODED) + # Parse metadata + metadata = self._process_metadata(list(content.splitlines())) + + # Get arguments and extensions + extra_args = self.settings.get('PANDOC_ARGS', []) + extensions = self.settings.get('PANDOC_EXTENSIONS', '') + if isinstance(extensions, list): + extensions = ''.join(extensions) + + # Construct Pandoc command + pandoc_cmd = [ + "pandoc", "-f", "markdown" + extensions, "-t", "html5" + ] + pandoc_cmd.extend(extra_args) + + # Execute and retrieve HTML 5 output + proc = subprocess.Popen( + pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + + html = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') + status = proc.wait() + if status: + raise subprocess.CalledProcessError(status, pandoc_cmd) + + # Replace all occurrences of %7Bstatic%7D to {static}, + # %7Battach%7D to {attach} and %7Bfilename%7D to {filename} + # so that static links are resolved by pelican. + for encoded_link, raw_link in ENCODED_LINK_TO_RAW_LINK_MAP.items(): + html.replace(encoded_link, raw_link) return html, metadata From 3350703ba246cb05befea162c0fc02d3f76bb960 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 15 Apr 2020 19:26:17 -0700 Subject: [PATCH 004/102] Corrected issue where output was not assigned to post pandoc output --- pandoc_reader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandoc_reader.py b/pandoc_reader.py index 768242a..54ced64 100644 --- a/pandoc_reader.py +++ b/pandoc_reader.py @@ -45,18 +45,18 @@ def read(self, source_path): pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) - html = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') + output = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') status = proc.wait() if status: raise subprocess.CalledProcessError(status, pandoc_cmd) # Replace all occurrences of %7Bstatic%7D to {static}, # %7Battach%7D to {attach} and %7Bfilename%7D to {filename} - # so that static links are resolved by pelican. - for encoded_link, raw_link in ENCODED_LINK_TO_RAW_LINK_MAP.items(): - html.replace(encoded_link, raw_link) + # so that static links are resolved by pelican + for encoded_str, raw_str in ENCODED_LINK_TO_RAW_LINK_MAP.items(): + output = output.replace(encoded_str, raw_str) - return html, metadata + return output, metadata def _process_metadata(self, text): """Process YAML metadata and export.""" From 8cf5526cc4117b6df836376799975bca45cca483 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 15 Apr 2020 20:01:36 -0700 Subject: [PATCH 005/102] Updated wording of README --- README.md | 96 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 61698ce..629d07d 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,89 @@ # pandoc_reader -A pandoc [markdown] reader plugin for [pelican] +A [Pandoc] [Markdown] reader plugin for the [Pelican] static site generator. + +Pandoc's version of Markdown is a flavour of Markdown with extensions and is explained in greater detail [here](https://pandoc.org/MANUAL.html#pandocs-markdown). ## Requirements -- [pandoc] in $PATH +For this plugin to function you must have Pandoc installed on your system. + +Please follow the instructions [here](https://pandoc.org/installing.html) to install Pandoc for your system. ## Installation -Instructions for installation of pelican plugins can be obtained from the [pelican plugin manual](https://github.com/getpelican/pelican-plugins/blob/master/Readme.rst). +For instructions on installing Pelican plugins please see the [Pelican Plugins](https://github.com/getpelican/pelican-plugins/blob/master/Readme.rst) documentation. ## Configuration -Additional command line parameters can be passed to pandoc via the PANDOC_ARGS parameter. +The plugin expects to have metadata at the top of every Markdown file in YAML format like so: + +```yaml +--- +title: +author: +--- +``` + +or + +```yaml +... +title: +author: +... +``` + +Failing to provide this metadata will cause the plugin to fail. + +Additional command line parameters can be passed to Pandoc via the `PANDOC_ARGS` parameter. + +```python +PANDOC_ARGS = [ + '--mathjax', + '--toc', + '--toc-depth=2', + '--number-sections', + '--standalone' +] +``` - PANDOC_ARGS = [ - '--mathjax', - '--smart', - '--toc', - '--toc-depth=2', - '--number-sections', - ] +Pandoc's markdown extensions can be enabled or disabled via the `PANDOC_EXTENSIONS` parameter. -Pandoc's markdown extensions can be enabled or disabled via the -PANDOC_EXTENSIONS parameter. +```python +PANDOC_EXTENSIONS = [ + '+hard_line_breaks', + '-citations' +] +``` - PANDOC_EXTENSIONS = [ - '+hard_line_breaks', - '-citations' - ] +More information about Pandoc's extensions can be found [here](https://pandoc.org/MANUAL.html#extensions). ## Contributing -1. Fork it -2. Create your feature branch (`git checkout -b my-new-feature`) -3. Commit your changes (`git commit -am 'Add some feature'`) -4. Push to the branch (`git push origin my-new-feature`) -5. Create new Pull Request +To contribute to this project follow the steps below: + +1. Fork this project on GitHub. +1. Create your feature branch. + + ```bash + git checkout -b my-new-feature + ``` + +1. Commit your changes. + + ```bash + git commit -am 'Add some feature' + ``` + +1. Push to the branch. + + ```bash + git push origin my-new-feature + ``` + +1. Create new Pull Request on GitHub. -[markdown]: http://daringfireball.net/projects/markdown/ -[pandoc]: http://johnmacfarlane.net/pandoc/ -[pelican]: http://getpelican.com +[Markdown]: http://daringfireball.net/projects/markdown/ +[Pandoc]: https://pandoc.org/ +[Pelican]: http://getpelican.com From 580ec057eb916d28f1475cd40149cac71955e2ac Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 08:11:19 -0700 Subject: [PATCH 006/102] Updated documentation and added a check to see if Pandoc is installed --- .gitignore | 1 + README.md | 18 ++++++++---------- __init__.py | 2 +- pandoc_reader.py | 18 +++++++++++------- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 892a243..8719382 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .idea *~ __pycache__ +venv diff --git a/README.md b/README.md index 629d07d..06e29bf 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,15 @@ Pandoc's version of Markdown is a flavour of Markdown with extensions and is exp For this plugin to function you must have Pandoc installed on your system. -Please follow the instructions [here](https://pandoc.org/installing.html) to install Pandoc for your system. +Please follow the [installation instructions](https://pandoc.org/installing.html) to install Pandoc for your system. ## Installation -For instructions on installing Pelican plugins please see the [Pelican Plugins](https://github.com/getpelican/pelican-plugins/blob/master/Readme.rst) documentation. +For instructions on installing Pelican plugins please see the [Pelican Plugins] documentation. ## Configuration -The plugin expects to have metadata at the top of every Markdown file in YAML format like so: +This plugin expects to have metadata at the top of every Markdown file in YAML as format shown below: ```yaml --- @@ -36,15 +36,12 @@ author: Failing to provide this metadata will cause the plugin to fail. -Additional command line parameters can be passed to Pandoc via the `PANDOC_ARGS` parameter. +Additional command line options can be passed to Pandoc via the `PANDOC_ARGS` parameter. ```python PANDOC_ARGS = [ '--mathjax', - '--toc', - '--toc-depth=2', - '--number-sections', - '--standalone' + '--toc' ] ``` @@ -52,8 +49,8 @@ Pandoc's markdown extensions can be enabled or disabled via the `PANDOC_EXTENSIO ```python PANDOC_EXTENSIONS = [ - '+hard_line_breaks', - '-citations' + '+footnotes', + '-pipe_tables' ] ``` @@ -87,3 +84,4 @@ To contribute to this project follow the steps below: [Markdown]: http://daringfireball.net/projects/markdown/ [Pandoc]: https://pandoc.org/ [Pelican]: http://getpelican.com +[Pelican Plugins]: https://github.com/getpelican/pelican-plugins/blob/master/Readme.rst diff --git a/__init__.py b/__init__.py index d2d63b3..5849527 100644 --- a/__init__.py +++ b/__init__.py @@ -1,2 +1,2 @@ """Importing pandoc_reader package.""" -from .pandoc_reader import * +from pandoc_reader import * diff --git a/pandoc_reader.py b/pandoc_reader.py index 54ced64..a6e2ac2 100644 --- a/pandoc_reader.py +++ b/pandoc_reader.py @@ -1,11 +1,13 @@ """Reader that processes Pandoc Markdown and returns HTML 5.""" +import shutil import subprocess +import sys from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open -ENCODED_LINK_TO_RAW_LINK_MAP = { +ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", "%7Battach%7D": "{attach}", "%7Bfilename%7D": "{filename}" @@ -20,12 +22,15 @@ class PandocReader(BaseReader): def read(self, source_path): """Parse Pandoc Markdown and return HTML 5 output and metadata.""" - content = "" + # Check if pandoc is installed and is executable + if not shutil.which("pandoc"): + sys.exit("Please install Pandoc before using this plugin.") + content = "" with pelican_open(source_path) as file_content: content = file_content - # Parse metadata + # Parse YAML metadata metadata = self._process_metadata(list(content.splitlines())) # Get arguments and extensions @@ -36,7 +41,7 @@ def read(self, source_path): # Construct Pandoc command pandoc_cmd = [ - "pandoc", "-f", "markdown" + extensions, "-t", "html5" + "pandoc", "--from", "markdown" + extensions, "--to", "html5" ] pandoc_cmd.extend(extra_args) @@ -53,7 +58,7 @@ def read(self, source_path): # Replace all occurrences of %7Bstatic%7D to {static}, # %7Battach%7D to {attach} and %7Bfilename%7D to {filename} # so that static links are resolved by pelican - for encoded_str, raw_str in ENCODED_LINK_TO_RAW_LINK_MAP.items(): + for encoded_str, raw_str in ENCODED_LINKS_TO_RAW_LINKS_MAP.items(): output = output.replace(encoded_str, raw_str) return output, metadata @@ -62,8 +67,7 @@ def _process_metadata(self, text): """Process YAML metadata and export.""" metadata = {} - # Check that the first line of the file - # starts with a YAML header + # Check that the first line of the file starts with a YAML header if text[0].strip() not in ["---", "..."]: raise Exception("Could not find metadata header '---' or '...'") From 3c0266d51b0c4347c9027b36d60bd886b5df7adc Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 08:18:11 -0700 Subject: [PATCH 007/102] Updated .gitignore by adding settings from GitHub's python .gitignore --- .gitignore | 59 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 8719382..092b855 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,54 @@ -*.pyc -.idea -*~ -__pycache__ -venv +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# VSCode Project Settings +.vscode From 22520895722f8ead6556dd31892cd6032f617782 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 09:11:57 -0700 Subject: [PATCH 008/102] Restructured code according to new pelican plugin namespaece format --- .editorconfig | 15 +++ .github/workflows/main.yml | 103 ++++++++++++++++++ .pre-commit-config.yaml | 30 +++++ CONTRIBUTING.md | 9 ++ README.md | 53 ++++----- .../plugins/pandoc_reader/__init__.py | 2 +- .../plugins/pandoc_reader/pandoc_reader.py | 0 .../pandoc_reader/test_pandoc_reader.py | 0 pyproject.toml | 70 ++++++++++++ tasks.py | 79 ++++++++++++++ tox.ini | 3 + 11 files changed, 333 insertions(+), 31 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/workflows/main.yml create mode 100644 .pre-commit-config.yaml create mode 100644 CONTRIBUTING.md rename __init__.py => pelican/plugins/pandoc_reader/__init__.py (57%) rename pandoc_reader.py => pelican/plugins/pandoc_reader/pandoc_reader.py (100%) create mode 100644 pelican/plugins/pandoc_reader/test_pandoc_reader.py create mode 100644 pyproject.toml create mode 100644 tasks.py create mode 100644 tox.ini diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..862c1e1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[*.py] +max_line_length = 88 + +[*.yml] +indent_size = 2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..97d6fd2 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,103 @@ +name: build + +on: [push, pull_request] + +env: + PYTEST_ADDOPTS: "--color=yes" + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set up Pip cache + uses: actions/cache@v2 + id: pip-cache + with: + path: ~/.cache/pip + key: pip-${{ hashFiles('**/pyproject.toml') }} + - name: Upgrade Pip + run: python -m pip install -U pip + - name: Install Poetry + run: python -m pip install poetry + - name: Set up Poetry cache + uses: actions/cache@v2 + id: poetry-cache + with: + path: ~/.cache/pypoetry/virtualenvs + key: poetry-${{ hashFiles('**/poetry.lock') }} + - name: Install dependencies + run: | + poetry run pip install -U pip + poetry install + - name: Run tests + run: poetry run invoke tests + + + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Set Poetry cache + uses: actions/cache@v2 + id: poetry-cache + with: + path: ~/.cache/pypoetry/virtualenvs + key: poetry-${{ hashFiles('**/poetry.lock') }} + - name: Upgrade Pip + run: python -m pip install -U pip + - name: Install Poetry + run: python -m pip install poetry + - name: Install dependencies + run: | + poetry run pip install -U pip + poetry install + - name: Run linters + run: poetry run invoke lint + + + deploy: + name: Deploy + needs: [test, lint] + runs-on: ubuntu-latest + if: ${{ github.ref=='refs/heads/main' && github.event_name!='pull_request' }} + + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Check release + id: check_release + run: | + python -m pip install pip --upgrade + python -m pip install poetry githubrelease autopub + echo "##[set-output name=release;]$(autopub check)" + - name: Publish + if: ${{ steps.check_release.outputs.release=='' }} + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + git remote set-url origin https://$GITHUB_TOKEN@github.com/${{ github.repository }} + autopub prepare + poetry build + autopub commit + autopub githubrelease + poetry publish -u __token__ -p $PYPI_PASSWORD diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1cf1dd8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +# See https://pre-commit.com/hooks.html for info on hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black + + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.3 + hooks: + - id: flake8 + args: [--max-line-length=88] + language_version: python3.7 + + - repo: https://github.com/timothycrosley/isort + rev: 5.4.2 + hooks: + - id: isort diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..aee7097 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ +Contributing +============ + +Contributions are welcome and much appreciated. Every little bit helps. You can contribute by improving the documentation, adding missing features, and fixing bugs. You can also help out by reviewing and commenting on [existing issues][]. + +To start contributing to this plugin, review the [Contributing to Pelican][] documentation, beginning with the **Contributing Code** section. + +[existing issues]: https://github.com/pelican-plugins/pandoc-reader/issues +[Contributing to Pelican]: https://docs.getpelican.com/en/latest/contribute.html diff --git a/README.md b/README.md index 06e29bf..7f5c860 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,28 @@ -# pandoc_reader +# Pandoc Reader -A [Pandoc] [Markdown] reader plugin for the [Pelican] static site generator. +Pandoc Reader is a [Pelican][] plugin to render documents written in [Pandoc][] Markdown. -Pandoc's version of Markdown is a flavour of Markdown with extensions and is explained in greater detail [here](https://pandoc.org/MANUAL.html#pandocs-markdown). +Pandoc's version of Markdown is a flavour of [Markdown][] with extensions and is explained in greater detail [here](https://pandoc.org/MANUAL.html#pandocs-markdown). -## Requirements +## Prerequisites For this plugin to function you must have Pandoc installed on your system. -Please follow the [installation instructions](https://pandoc.org/installing.html) to install Pandoc for your system. +Please follow the [installation instructions][] to install Pandoc for your system. ## Installation -For instructions on installing Pelican plugins please see the [Pelican Plugins] documentation. +This plugin can be installed via: -## Configuration +```bash +python -m pip install pelican-pandoc-reader +``` + +## Usage -This plugin expects to have metadata at the top of every Markdown file in YAML as format shown below: +You may use this plugin whenever you wish to write your source files in Pandoc's Markdown and have them rendered as HTML 5. + +This plugin expects to have metadata in YAML format at the top of every Markdown as shown below: ```yaml --- @@ -36,7 +42,7 @@ author: Failing to provide this metadata will cause the plugin to fail. -Additional command line options can be passed to Pandoc via the `PANDOC_ARGS` parameter. +Additional command line options may be passed to Pandoc via the `PANDOC_ARGS` parameter. ```python PANDOC_ARGS = [ @@ -45,7 +51,7 @@ PANDOC_ARGS = [ ] ``` -Pandoc's markdown extensions can be enabled or disabled via the `PANDOC_EXTENSIONS` parameter. +Pandoc's markdown extensions may be enabled or disabled via the `PANDOC_EXTENSIONS` parameter. ```python PANDOC_EXTENSIONS = [ @@ -58,30 +64,17 @@ More information about Pandoc's extensions can be found [here](https://pandoc.or ## Contributing -To contribute to this project follow the steps below: - -1. Fork this project on GitHub. -1. Create your feature branch. - - ```bash - git checkout -b my-new-feature - ``` - -1. Commit your changes. - - ```bash - git commit -am 'Add some feature' - ``` +Contributions are welcome and much appreciated. Every little bit helps. You can contribute by improving the documentation, adding missing features, and fixing bugs. You can also help out by reviewing and commenting on [existing issues][]. -1. Push to the branch. +To start contributing to this plugin, review the [Contributing to Pelican][] documentation, beginning with the **Contributing Code** section. - ```bash - git push origin my-new-feature - ``` +## Credits -1. Create new Pull Request on GitHub. +Originally authored by [Hinrich B. Winther](https://github.com/liob), December 2014, and subsequently enhanced by [Nandakumar Chandrasekhar](https://www.linkedin.com/in/nandakumar-chandrasekhar-a400b45b/). +[installation instructions]: https://pandoc.org/installing.html [Markdown]: http://daringfireball.net/projects/markdown/ [Pandoc]: https://pandoc.org/ [Pelican]: http://getpelican.com -[Pelican Plugins]: https://github.com/getpelican/pelican-plugins/blob/master/Readme.rst +[existing issues]: https://github.com/pelican-plugins/pandoc-reader/issues +[Contributing to Pelican]: https://docs.getpelican.com/en/latest/contribute.html diff --git a/__init__.py b/pelican/plugins/pandoc_reader/__init__.py similarity index 57% rename from __init__.py rename to pelican/plugins/pandoc_reader/__init__.py index 5849527..d2d63b3 100644 --- a/__init__.py +++ b/pelican/plugins/pandoc_reader/__init__.py @@ -1,2 +1,2 @@ """Importing pandoc_reader package.""" -from pandoc_reader import * +from .pandoc_reader import * diff --git a/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py similarity index 100% rename from pandoc_reader.py rename to pelican/plugins/pandoc_reader/pandoc_reader.py diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cf5de60 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,70 @@ +[tool.poetry] +name = "pelican-pandoc-reader" +version = "0.0.1" +description = "Pelican plugin to render Pandoc Markdown files" +authors = ["Nandakumar Chandrasekhar "] +license = "AGPL-3.0" +readme = "README.md" +keywords = ["pelican", "plugin", "pandoc"] +repository = "https://github.com/pelican-plugins/pandoc-reader" +documentation = "https://docs.getpelican.com" +packages = [ + { include = "pelican" }, +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Framework :: Pelican", + "Framework :: Pelican :: Plugins", + "Intended Audience :: End Users/Desktop", + "Operating System :: OS Independent", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.urls] +"Funding" = "https://donate.getpelican.com/" +"Issue Tracker" = "https://github.com/pelican-plugins/pandoc-reader/issues" + +[tool.poetry.dependencies] +python = "^3.6" +pelican = "^4.5" +markdown = {version = "^3.2.2", optional = true} + +[tool.poetry.dev-dependencies] +black = {version = "^19.10b0", allow-prereleases = true} +flake8 = "^3.8" +flake8-black = "^0.1.0" +invoke = "^1.3" +isort = "^5.4" +livereload = "^2.6" +markdown = "^3.2.2" +pytest = "^6.0" +pytest-cov = "^2.7" +pytest-pythonpath = "^0.7.3" +pytest-sugar = "^0.9.4" +Werkzeug = "^1.0" + +[tool.poetry.extras] +markdown = ["markdown"] + +[tool.autopub] +project-name = "Pandoc Reader" +git-username = "botpub" +git-email = "botpub@autopub.rocks" + +[tool.isort] +# Maintain compatibility with Black +combine_as_imports = true +force_grid_wrap = 0 +include_trailing_comma = true +line_length = 88 +multi_line_output = 3 + +# Sort imports within their section independent of the import type +force_sort_within_sections = true + +[build-system] +requires = ["poetry>=1.0"] +build-backend = "poetry.masonry.api" diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..a5dfcb4 --- /dev/null +++ b/tasks.py @@ -0,0 +1,79 @@ +import os +from pathlib import Path +from shutil import which + +from invoke import task + +PKG_NAME = "pandoc_reader" +PKG_PATH = Path(f"pelican/plugins/{PKG_NAME}") +ACTIVE_VENV = os.environ.get("VIRTUAL_ENV", None) +VENV_HOME = Path(os.environ.get("WORKON_HOME", "~/.local/share/virtualenvs")) +VENV_PATH = Path(ACTIVE_VENV) if ACTIVE_VENV else (VENV_HOME / PKG_NAME) +VENV = str(VENV_PATH.expanduser()) + +TOOLS = ["poetry", "pre-commit"] +POETRY = which("poetry") if which("poetry") else (VENV / Path("bin") / "poetry") +PRECOMMIT = ( + which("pre-commit") if which("pre-commit") else (VENV / Path("bin") / "pre-commit") +) + + +@task +def tests(c): + """Run the test suite""" + c.run(f"{VENV}/bin/pytest", pty=True) + + +@task +def black(c, check=False, diff=False): + """Run Black auto-formatter, optionally with --check or --diff""" + check_flag, diff_flag = "", "" + if check: + check_flag = "--check" + if diff: + diff_flag = "--diff" + c.run(f"{VENV}/bin/black {check_flag} {diff_flag} {PKG_PATH} tasks.py") + + +@task +def isort(c, check=False, diff=False): + check_flag, diff_flag = "", "" + if check: + check_flag = "-c" + if diff: + diff_flag = "--diff" + c.run(f"{VENV}/bin/isort {check_flag} {diff_flag} .") + + +@task +def flake8(c): + c.run(f"{VENV}/bin/flake8 {PKG_PATH} tasks.py") + + +@task +def lint(c): + isort(c, check=True) + black(c, check=True) + flake8(c) + + +@task +def tools(c): + """Install tools in the virtual environment if not already on PATH""" + for tool in TOOLS: + if not which(tool): + c.run(f"{VENV}/bin/pip install {tool}") + + +@task +def precommit(c): + """Install pre-commit hooks to .git/hooks/pre-commit""" + c.run(f"{PRECOMMIT} install") + + +@task +def setup(c): + c.run(f"{VENV}/bin/pip install -U pip") + tools(c) + c.run(f"{POETRY} install") + precommit(c) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..91e7c69 --- /dev/null +++ b/tox.ini @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +ignore = E203, E266, E501, W503 From ca5d7af036fc99ec112c1a62dcb083984b7bfcff Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 17:05:10 -0700 Subject: [PATCH 009/102] Used isort to sort imports correctly and fix linting issue on GitHub --- pelican/plugins/pandoc_reader/pandoc_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index a6e2ac2..b7af085 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -3,10 +3,11 @@ import subprocess import sys -from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open +from pelican import signals + ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", "%7Battach%7D": "{attach}", From 39ef3f90d995b6a9733684b81206541f2dce2901 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 17:13:09 -0700 Subject: [PATCH 010/102] Used black package to reformat code to fix linting issues --- pelican/plugins/pandoc_reader/pandoc_reader.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index b7af085..3f6ba04 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -11,7 +11,7 @@ ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", "%7Battach%7D": "{attach}", - "%7Bfilename%7D": "{filename}" + "%7Bfilename%7D": "{filename}", } @@ -19,7 +19,7 @@ class PandocReader(BaseReader): """Process files written in Pandoc Markdown.""" enabled = True - file_extensions = ['md', 'markdown', 'mkd', 'mdown'] + file_extensions = ["md", "markdown", "mkd", "mdown"] def read(self, source_path): """Parse Pandoc Markdown and return HTML 5 output and metadata.""" @@ -35,15 +35,13 @@ def read(self, source_path): metadata = self._process_metadata(list(content.splitlines())) # Get arguments and extensions - extra_args = self.settings.get('PANDOC_ARGS', []) - extensions = self.settings.get('PANDOC_EXTENSIONS', '') + extra_args = self.settings.get("PANDOC_ARGS", []) + extensions = self.settings.get("PANDOC_EXTENSIONS", "") if isinstance(extensions, list): - extensions = ''.join(extensions) + extensions = "".join(extensions) # Construct Pandoc command - pandoc_cmd = [ - "pandoc", "--from", "markdown" + extensions, "--to", "html5" - ] + pandoc_cmd = ["pandoc", "--from", "markdown" + extensions, "--to", "html5"] pandoc_cmd.extend(extra_args) # Execute and retrieve HTML 5 output @@ -51,7 +49,7 @@ def read(self, source_path): pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) - output = proc.communicate(content.encode('utf-8'))[0].decode('utf-8') + output = proc.communicate(content.encode("utf-8"))[0].decode("utf-8") status = proc.wait() if status: raise subprocess.CalledProcessError(status, pandoc_cmd) @@ -86,7 +84,7 @@ def _process_metadata(self, text): # Process the YAML block for line in lines[:yaml_end]: - metalist = line.split(':', 1) + metalist = line.split(":", 1) if len(metalist) == 2: key, value = metalist[0].lower(), metalist[1].strip() metadata[key] = self.process_metadata(key, value) From cf890fff89a02fb19927c234753a7078f254ca0a Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 17:16:27 -0700 Subject: [PATCH 011/102] Added docstring to empty tests file --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index e69de29..e707fae 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -0,0 +1 @@ +"""Empty tests file.""" From 92aec38c05e5ef01d88f942ef6110bfd0c56e9ab Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 17:24:34 -0700 Subject: [PATCH 012/102] Ran flake8 linter on file to fix issues --- pelican/plugins/pandoc_reader/__init__.py | 2 +- pelican/plugins/pandoc_reader/pandoc_reader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pelican/plugins/pandoc_reader/__init__.py b/pelican/plugins/pandoc_reader/__init__.py index d2d63b3..124ec0b 100644 --- a/pelican/plugins/pandoc_reader/__init__.py +++ b/pelican/plugins/pandoc_reader/__init__.py @@ -1,2 +1,2 @@ """Importing pandoc_reader package.""" -from .pandoc_reader import * +from .pandoc_reader import * # NOQA diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 3f6ba04..8f2cd5a 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -41,7 +41,7 @@ def read(self, source_path): extensions = "".join(extensions) # Construct Pandoc command - pandoc_cmd = ["pandoc", "--from", "markdown" + extensions, "--to", "html5"] + pandoc_cmd = ["pandoc", "-f", "markdown" + extensions, "-t", "html5"] pandoc_cmd.extend(extra_args) # Execute and retrieve HTML 5 output From 8f2496fbe22e2a720dc40ad1583b1600b83b9bad Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 18:11:22 -0700 Subject: [PATCH 013/102] Added tests to see if all checks pass --- .../plugins/pandoc_reader/pandoc_reader.py | 3 +-- .../pandoc_reader/test_pandoc_reader.py | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 8f2cd5a..05a2f70 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -1,7 +1,6 @@ """Reader that processes Pandoc Markdown and returns HTML 5.""" import shutil import subprocess -import sys from pelican.readers import BaseReader from pelican.utils import pelican_open @@ -25,7 +24,7 @@ def read(self, source_path): """Parse Pandoc Markdown and return HTML 5 output and metadata.""" # Check if pandoc is installed and is executable if not shutil.which("pandoc"): - sys.exit("Please install Pandoc before using this plugin.") + raise Exception("Could not find Pandoc. Please install.") content = "" with pelican_open(source_path) as file_content: diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index e707fae..3079f46 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -1 +1,19 @@ -"""Empty tests file.""" +"""Tests for pandoc-reader plugin.""" +import os +import unittest + +from pandoc_reader import PandocReader + + +class TestPandocReader(unittest.TestCase): + """Test class for pandoc-reader plugin.""" + + def test_pandoc_installed(self): + """Check if Pandoc is installed.""" + with self.assertRaises(Exception) as context: + PandocReader.read(os.getcwd()) + self.assertTrue("Could not find Pandoc. Please install." in context.exception) + + +if __name__ == "__main__": + unittest.main() From 9c4a14d97d91ded476a64227d6af6d637961c69e Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 18:23:59 -0700 Subject: [PATCH 014/102] Fixed issue with test --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 3079f46..8f5b38e 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -10,9 +10,12 @@ class TestPandocReader(unittest.TestCase): def test_pandoc_installed(self): """Check if Pandoc is installed.""" - with self.assertRaises(Exception) as context: + self.assertRaises(Exception, PandocReader.read, os.getcwd()) + with self.assertRaises(TypeError) as context: PandocReader.read(os.getcwd()) - self.assertTrue("Could not find Pandoc. Please install." in context.exception) + self.assertEqual( + "Could not find Pandoc. Please install.", str(context.exception) + ) if __name__ == "__main__": From f0cee8e3bd35991eed332024ab0f0a771cc758cd Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Thu, 15 Oct 2020 18:27:20 -0700 Subject: [PATCH 015/102] Fixing raising of exception test --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 8f5b38e..9753d93 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -10,7 +10,7 @@ class TestPandocReader(unittest.TestCase): def test_pandoc_installed(self): """Check if Pandoc is installed.""" - self.assertRaises(Exception, PandocReader.read, os.getcwd()) + self.assertRaises(Exception, PandocReader.read(), os.getcwd()) with self.assertRaises(TypeError) as context: PandocReader.read(os.getcwd()) self.assertEqual( From 3bb09f42442817ab1607a2846fb3652017ff646e Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 18:52:33 -0700 Subject: [PATCH 016/102] Added invalid path unit tests for pandoc-reader --- .editorconfig | 2 +- .../plugins/pandoc_reader/fixtures/empty.md | 0 .../pandoc_reader/fixtures/no_metadata.md | 3 + .../pandoc_reader/fixtures/no_metadata_end.md | 4 ++ .../fixtures/wrong_metadata_end.md | 5 ++ .../plugins/pandoc_reader/pandoc_reader.py | 4 ++ .../pandoc_reader/test_pandoc_reader.py | 67 +++++++++++++++++-- pyproject.toml | 2 +- tasks.py | 2 +- 9 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 pelican/plugins/pandoc_reader/fixtures/empty.md create mode 100644 pelican/plugins/pandoc_reader/fixtures/no_metadata.md create mode 100644 pelican/plugins/pandoc_reader/fixtures/no_metadata_end.md create mode 100644 pelican/plugins/pandoc_reader/fixtures/wrong_metadata_end.md diff --git a/.editorconfig b/.editorconfig index 862c1e1..b42ca8c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,7 +9,7 @@ insert_final_newline = true trim_trailing_whitespace = true [*.py] -max_line_length = 88 +max_line_length = 79 [*.yml] indent_size = 2 diff --git a/pelican/plugins/pandoc_reader/fixtures/empty.md b/pelican/plugins/pandoc_reader/fixtures/empty.md new file mode 100644 index 0000000..e69de29 diff --git a/pelican/plugins/pandoc_reader/fixtures/no_metadata.md b/pelican/plugins/pandoc_reader/fixtures/no_metadata.md new file mode 100644 index 0000000..da70023 --- /dev/null +++ b/pelican/plugins/pandoc_reader/fixtures/no_metadata.md @@ -0,0 +1,3 @@ +# File that does not contain any metadata + +This is a file that does not have any metadata diff --git a/pelican/plugins/pandoc_reader/fixtures/no_metadata_end.md b/pelican/plugins/pandoc_reader/fixtures/no_metadata_end.md new file mode 100644 index 0000000..089ebda --- /dev/null +++ b/pelican/plugins/pandoc_reader/fixtures/no_metadata_end.md @@ -0,0 +1,4 @@ +--- +title: "No Metadata End" +author: "Nandakumar Chandrasekhar" +date: "2020-10-16" diff --git a/pelican/plugins/pandoc_reader/fixtures/wrong_metadata_end.md b/pelican/plugins/pandoc_reader/fixtures/wrong_metadata_end.md new file mode 100644 index 0000000..7979bc1 --- /dev/null +++ b/pelican/plugins/pandoc_reader/fixtures/wrong_metadata_end.md @@ -0,0 +1,5 @@ +--- +title: "No Metadata End" +author: "Nandakumar Chandrasekhar" +date: "2020-10-16" +~~~ diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 05a2f70..0d04d74 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -65,6 +65,10 @@ def _process_metadata(self, text): """Process YAML metadata and export.""" metadata = {} + # Check that the given text is not empty + if not text: + raise Exception("Could not find metadata. File is empty") + # Check that the first line of the file starts with a YAML header if text[0].strip() not in ["---", "..."]: raise Exception("Could not find metadata header '---' or '...'") diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 9753d93..c8fdf5d 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -1,22 +1,81 @@ """Tests for pandoc-reader plugin.""" import os +import shutil import unittest from pandoc_reader import PandocReader +from pelican.tests.support import get_settings + +FILE_PATH = os.path.join(os.getcwd(), "fixtures") class TestPandocReader(unittest.TestCase): """Test class for pandoc-reader plugin.""" + settings = get_settings() + pandoc_reader = PandocReader(settings) + def test_pandoc_installed(self): """Check if Pandoc is installed.""" - self.assertRaises(Exception, PandocReader.read(), os.getcwd()) - with self.assertRaises(TypeError) as context: - PandocReader.read(os.getcwd()) + source_path = os.path.join(FILE_PATH, "empty.md") + + if not shutil.which("pandoc"): + # Case where pandoc is not installed + with self.assertRaises(Exception) as context_manager: + self.pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Could not find Pandoc. Please install.", message) + else: + # Case where pandoc is installed + self.assertTrue("Pandoc is installed.") + + def test_empty_file(self): + """Check if a file is empty.""" + source_path = os.path.join(FILE_PATH, "empty.md") + + # If the file is empty retrieval of metadata should fail + with self.assertRaises(Exception) as context_manager: + self.pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Could not find metadata. File is empty", message) + + def test_non_empty_file_no_metadata(self): + """Check if a file has no metadata.""" + source_path = os.path.join(FILE_PATH, "no_metadata.md") + + # If the file is not empty but has no metadata it should fail + with self.assertRaises(Exception) as context_manager: + self.pandoc_reader.read(source_path) + + message = str(context_manager.exception) self.assertEqual( - "Could not find Pandoc. Please install.", str(context.exception) + "Could not find metadata header '---' or '...'", message ) + def test_metadata_block_end(self): + """Check if the metadata block ends.""" + source_path = os.path.join(FILE_PATH, "no_metadata_end.md") + + # Metadata blocks should end with '___' or '...' if not it should fail + with self.assertRaises(Exception) as context_manager: + self.pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Could not find end of metadata block.", message) + + def test_invalid_metadata_block_end(self): + """Check if the metadata block end is wrong.""" + source_path = os.path.join(FILE_PATH, "no_metadata_end.md") + + # Metadata blocks should end with '___' or '...' if not it should fail + with self.assertRaises(Exception) as context_manager: + self.pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Could not find end of metadata block.", message) + if __name__ == "__main__": unittest.main() diff --git a/pyproject.toml b/pyproject.toml index cf5de60..527bb32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ packages = [ ] classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 3 - Alpha", "Environment :: Console", "Framework :: Pelican", "Framework :: Pelican :: Plugins", diff --git a/tasks.py b/tasks.py index a5dfcb4..c5f21fc 100644 --- a/tasks.py +++ b/tasks.py @@ -32,7 +32,7 @@ def black(c, check=False, diff=False): check_flag = "--check" if diff: diff_flag = "--diff" - c.run(f"{VENV}/bin/black {check_flag} {diff_flag} {PKG_PATH} tasks.py") + c.run(f"{VENV}/bin/black --line-length 79 {check_flag} {diff_flag} {PKG_PATH} tasks.py") @task From 6c1fdb696c132415b4c68c50790410567c4789f0 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 18:56:37 -0700 Subject: [PATCH 017/102] Reformatted tasks.py so that we can pass liny checks on GitHub --- tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index c5f21fc..f6b3650 100644 --- a/tasks.py +++ b/tasks.py @@ -32,7 +32,9 @@ def black(c, check=False, diff=False): check_flag = "--check" if diff: diff_flag = "--diff" - c.run(f"{VENV}/bin/black --line-length 79 {check_flag} {diff_flag} {PKG_PATH} tasks.py") + c.run( + f"{VENV}/bin/black --line-length 79 {check_flag} {diff_flag} {PKG_PATH} tasks.py" + ) @task From db1c019ee5f96a1f7014b1bcd002f8ec3e4f038d Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 19:06:41 -0700 Subject: [PATCH 018/102] Added pandoc installation step as a test dependency --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 97d6fd2..15e5bfc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,6 +39,10 @@ jobs: run: | poetry run pip install -U pip poetry install + - name: Install pandoc + run: | + wget https://github.com/jgm/pandoc/releases/download/2.11.0.2/pandoc-2.11.0.2-1-amd64.deb + dpkg -i pandoc-2.11.0.2-1-amd64.deb - name: Run tests run: poetry run invoke tests From a5464d136938f4011f86aff9b59b25415e13cbd8 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 19:09:04 -0700 Subject: [PATCH 019/102] Adding sudo to pandoc installation step --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 15e5bfc..f49b137 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,7 +42,7 @@ jobs: - name: Install pandoc run: | wget https://github.com/jgm/pandoc/releases/download/2.11.0.2/pandoc-2.11.0.2-1-amd64.deb - dpkg -i pandoc-2.11.0.2-1-amd64.deb + sudo dpkg -i pandoc-2.11.0.2-1-amd64.deb - name: Run tests run: poetry run invoke tests From ae2ba83871fcdc42f9e504c7940c766edcf388bf Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 19:15:03 -0700 Subject: [PATCH 020/102] Removed line-length check on black command to see if we can pass lint checks --- tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tasks.py b/tasks.py index f6b3650..a5dfcb4 100644 --- a/tasks.py +++ b/tasks.py @@ -32,9 +32,7 @@ def black(c, check=False, diff=False): check_flag = "--check" if diff: diff_flag = "--diff" - c.run( - f"{VENV}/bin/black --line-length 79 {check_flag} {diff_flag} {PKG_PATH} tasks.py" - ) + c.run(f"{VENV}/bin/black {check_flag} {diff_flag} {PKG_PATH} tasks.py") @task From 0fb62fcc7fe229d9ef941dcf3c42168c3d8adc77 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 19:17:07 -0700 Subject: [PATCH 021/102] Reformatted test_pandoc_reader.py --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index c8fdf5d..040789b 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -50,9 +50,7 @@ def test_non_empty_file_no_metadata(self): self.pandoc_reader.read(source_path) message = str(context_manager.exception) - self.assertEqual( - "Could not find metadata header '---' or '...'", message - ) + self.assertEqual("Could not find metadata header '---' or '...'", message) def test_metadata_block_end(self): """Check if the metadata block ends.""" From a52aa86d3d3ac3a526e2d2f78870b6d2c1062106 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 19:54:04 -0700 Subject: [PATCH 022/102] Using absolute paths to fixtures so that tests pass --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 040789b..e4c412b 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -6,7 +6,8 @@ from pandoc_reader import PandocReader from pelican.tests.support import get_settings -FILE_PATH = os.path.join(os.getcwd(), "fixtures") +ABS_PATH = os.path.abspath(__file__) +FILE_PATH = os.path.join(ABS_PATH, "fixtures") class TestPandocReader(unittest.TestCase): From 86d28b8d34527f1951760e91b6fd672efe8129af Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 19:56:59 -0700 Subject: [PATCH 023/102] Fixing path to fixtures --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index e4c412b..d8eaa24 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -6,8 +6,8 @@ from pandoc_reader import PandocReader from pelican.tests.support import get_settings -ABS_PATH = os.path.abspath(__file__) -FILE_PATH = os.path.join(ABS_PATH, "fixtures") +DIR_PATH = os.path.dirname(__file__) +FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "fixtures") class TestPandocReader(unittest.TestCase): From b8161158f311c7c478d4f0878180ed7bb11cabca Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 16 Oct 2020 20:00:58 -0700 Subject: [PATCH 024/102] Fixed unmatched bracket syntax issue --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index d8eaa24..56b18da 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -7,7 +7,7 @@ from pelican.tests.support import get_settings DIR_PATH = os.path.dirname(__file__) -FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "fixtures") +FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "fixtures")) class TestPandocReader(unittest.TestCase): From 32d56698d90ac79b1f4de9e4bdd26a2095d7516a Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 14:33:40 -0700 Subject: [PATCH 025/102] Added more tests and set line length to 79 characters --- .pre-commit-config.yaml | 3 +- .../{fixtures => content}/empty.md | 0 .../pandoc_reader/content/mathjax_content.md | 8 +++ .../{fixtures => content}/no_metadata.md | 0 .../{fixtures => content}/no_metadata_end.md | 0 .../pandoc_reader/content/valid_content.md | 9 +++ .../wrong_metadata_end.md | 0 .../plugins/pandoc_reader/pandoc_reader.py | 39 ++++++++++--- .../pandoc_reader/test_pandoc_reader.py | 55 +++++++++++++++++-- pyproject.toml | 2 +- 10 files changed, 101 insertions(+), 15 deletions(-) rename pelican/plugins/pandoc_reader/{fixtures => content}/empty.md (100%) create mode 100644 pelican/plugins/pandoc_reader/content/mathjax_content.md rename pelican/plugins/pandoc_reader/{fixtures => content}/no_metadata.md (100%) rename pelican/plugins/pandoc_reader/{fixtures => content}/no_metadata_end.md (100%) create mode 100644 pelican/plugins/pandoc_reader/content/valid_content.md rename pelican/plugins/pandoc_reader/{fixtures => content}/wrong_metadata_end.md (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1cf1dd8..d4fc363 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,12 +16,13 @@ repos: rev: 19.10b0 hooks: - id: black + args: [--line-length 79] - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: - id: flake8 - args: [--max-line-length=88] + args: [--max-line-length=79] language_version: python3.7 - repo: https://github.com/timothycrosley/isort diff --git a/pelican/plugins/pandoc_reader/fixtures/empty.md b/pelican/plugins/pandoc_reader/content/empty.md similarity index 100% rename from pelican/plugins/pandoc_reader/fixtures/empty.md rename to pelican/plugins/pandoc_reader/content/empty.md diff --git a/pelican/plugins/pandoc_reader/content/mathjax_content.md b/pelican/plugins/pandoc_reader/content/mathjax_content.md new file mode 100644 index 0000000..10cf098 --- /dev/null +++ b/pelican/plugins/pandoc_reader/content/mathjax_content.md @@ -0,0 +1,8 @@ +--- +title: MathJax Content +author: Nandakumar Chandrasekhar +date: 2020-10-16 +--- +$$ +e^{i\theta} = \cos\theta + i \sin\theta. +$$ diff --git a/pelican/plugins/pandoc_reader/fixtures/no_metadata.md b/pelican/plugins/pandoc_reader/content/no_metadata.md similarity index 100% rename from pelican/plugins/pandoc_reader/fixtures/no_metadata.md rename to pelican/plugins/pandoc_reader/content/no_metadata.md diff --git a/pelican/plugins/pandoc_reader/fixtures/no_metadata_end.md b/pelican/plugins/pandoc_reader/content/no_metadata_end.md similarity index 100% rename from pelican/plugins/pandoc_reader/fixtures/no_metadata_end.md rename to pelican/plugins/pandoc_reader/content/no_metadata_end.md diff --git a/pelican/plugins/pandoc_reader/content/valid_content.md b/pelican/plugins/pandoc_reader/content/valid_content.md new file mode 100644 index 0000000..10b8a05 --- /dev/null +++ b/pelican/plugins/pandoc_reader/content/valid_content.md @@ -0,0 +1,9 @@ +--- +title: Valid Content +author: Nandakumar Chandrasekhar +date: 2020-10-16 +--- + +# Valid Content + +This is some valid content that should pass. If it does not pass we will know something is wrong. diff --git a/pelican/plugins/pandoc_reader/fixtures/wrong_metadata_end.md b/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md similarity index 100% rename from pelican/plugins/pandoc_reader/fixtures/wrong_metadata_end.md rename to pelican/plugins/pandoc_reader/content/wrong_metadata_end.md diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 0d04d74..9aa2671 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -34,21 +34,42 @@ def read(self, source_path): metadata = self._process_metadata(list(content.splitlines())) # Get arguments and extensions - extra_args = self.settings.get("PANDOC_ARGS", []) - extensions = self.settings.get("PANDOC_EXTENSIONS", "") - if isinstance(extensions, list): - extensions = "".join(extensions) - - # Construct Pandoc command - pandoc_cmd = ["pandoc", "-f", "markdown" + extensions, "-t", "html5"] - pandoc_cmd.extend(extra_args) + if not self.settings.get("PANDOC_DEFAULT_FILES"): + extra_args = self.settings.get("PANDOC_ARGS", []) + extensions = self.settings.get("PANDOC_EXTENSIONS", "") + if isinstance(extensions, list): + extensions = "".join(extensions) + + # Construct Pandoc command + pandoc_cmd = [ + "pandoc", + "--from", + "markdown" + extensions, + "--to", + "html5", + ] + pandoc_cmd.extend(extra_args) + else: + defaults_cmd_str = "" + for file in self.settings.get("PANDOC_DEFAULT_FILES"): + defaults_cmd_str += " --defaults=file" + + # Construct Pandoc command + pandoc_cmd = [ + "pandoc", + defaults_cmd_str, + " --from", + "markdown", + "--to", + "html5", + ] # Execute and retrieve HTML 5 output proc = subprocess.Popen( pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) - output = proc.communicate(content.encode("utf-8"))[0].decode("utf-8") + output = proc.communicate(content.encode("UTF-8"))[0].decode("UTF-8") status = proc.wait() if status: raise subprocess.CalledProcessError(status, pandoc_cmd) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 56b18da..276c9f8 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -4,16 +4,24 @@ import unittest from pandoc_reader import PandocReader + from pelican.tests.support import get_settings DIR_PATH = os.path.dirname(__file__) -FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "fixtures")) +FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "content")) + +# Test settings that will be set in pelicanconf.py by plugin users +PANDOC_ARGS = ["--mathjax"] +PANDOC_EXTENSIONS = ["+smart", "+citations", "+implicit_figures"] class TestPandocReader(unittest.TestCase): """Test class for pandoc-reader plugin.""" - settings = get_settings() + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + pandoc_reader = PandocReader(settings) def test_pandoc_installed(self): @@ -29,7 +37,8 @@ def test_pandoc_installed(self): self.assertEqual("Could not find Pandoc. Please install.", message) else: # Case where pandoc is installed - self.assertTrue("Pandoc is installed.") + message = "Pandoc is installed." + self.assertEqual("Pandoc is installed.", message) def test_empty_file(self): """Check if a file is empty.""" @@ -51,7 +60,9 @@ def test_non_empty_file_no_metadata(self): self.pandoc_reader.read(source_path) message = str(context_manager.exception) - self.assertEqual("Could not find metadata header '---' or '...'", message) + self.assertEqual( + "Could not find metadata header '---' or '...'", message + ) def test_metadata_block_end(self): """Check if the metadata block ends.""" @@ -75,6 +86,42 @@ def test_invalid_metadata_block_end(self): message = str(context_manager.exception) self.assertEqual("Could not find end of metadata block.", message) + def test_valid_file(self): + """Check if we get the appropriate output for valid input.""" + source_path = os.path.join(FILE_PATH, "valid_content.md") + output, metadata = self.pandoc_reader.read(source_path) + + self.assertEqual( + ( + '

Valid Content

\n

This' + " is some valid content that should pass." + " If it does not pass we" + " will know something is wrong.

\n" + ), + output, + ) + + self.assertEqual("Valid Content", str(metadata["title"])) + self.assertEqual("Nandakumar Chandrasekhar", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + + def test_mathjax_content(self): + """Check if mathematics is rendered correctly.""" + source_path = os.path.join(FILE_PATH, "mathjax_content.md") + output, metadata = self.pandoc_reader.read(source_path) + + self.assertEqual( + ( + '

\\[\ne^{i\\theta} = ' + "\\cos\\theta + i \\sin\\theta.\n\\]

\n" + ), + output, + ) + + self.assertEqual("MathJax Content", str(metadata["title"])) + self.assertEqual("Nandakumar Chandrasekhar", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + if __name__ == "__main__": unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 527bb32..012453d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ git-email = "botpub@autopub.rocks" combine_as_imports = true force_grid_wrap = 0 include_trailing_comma = true -line_length = 88 +line_length = 79 multi_line_output = 3 # Sort imports within their section independent of the import type From ddaeafd7c99ad3a8e5d5aee7ded512a988d95da8 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 14:36:00 -0700 Subject: [PATCH 026/102] Fixing import order --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 276c9f8..ec18f77 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -3,9 +3,10 @@ import shutil import unittest +from pelican.tests.support import get_settings + from pandoc_reader import PandocReader -from pelican.tests.support import get_settings DIR_PATH = os.path.dirname(__file__) FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "content")) From 7182afcbd0110e23f83543f3df4eeff6b2da07a3 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 14:39:34 -0700 Subject: [PATCH 027/102] Fixing import order --- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index ec18f77..9f3d41b 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -3,10 +3,8 @@ import shutil import unittest -from pelican.tests.support import get_settings - from pandoc_reader import PandocReader - +from pelican.tests.support import get_settings DIR_PATH = os.path.dirname(__file__) FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "content")) From de9fb7d67b8c4b40b4dc6bf8deab4b13a60e2a95 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 14:45:20 -0700 Subject: [PATCH 028/102] Setting line-legth argument for black --- .pre-commit-config.yaml | 1 - pelican/plugins/pandoc_reader/test_pandoc_reader.py | 3 ++- tasks.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d4fc363..7503e21 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,6 @@ repos: rev: 19.10b0 hooks: - id: black - args: [--line-length 79] - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 9f3d41b..663cb5e 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -60,7 +60,8 @@ def test_non_empty_file_no_metadata(self): message = str(context_manager.exception) self.assertEqual( - "Could not find metadata header '---' or '...'", message + "Could not find metadata header '---' or '...'", + message ) def test_metadata_block_end(self): diff --git a/tasks.py b/tasks.py index a5dfcb4..742d340 100644 --- a/tasks.py +++ b/tasks.py @@ -32,7 +32,7 @@ def black(c, check=False, diff=False): check_flag = "--check" if diff: diff_flag = "--diff" - c.run(f"{VENV}/bin/black {check_flag} {diff_flag} {PKG_PATH} tasks.py") + c.run(f"{VENV}/bin/black {check_flag} {diff_flag} --line-length 79 {PKG_PATH} tasks.py") @task From 80dbce89b32a911d4c5dd2fa7818d450a65688c5 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 14:50:33 -0700 Subject: [PATCH 029/102] Reformatted files as per linting issues --- .editorconfig | 2 +- .pre-commit-config.yaml | 2 +- pelican/plugins/pandoc_reader/test_pandoc_reader.py | 5 +---- pyproject.toml | 2 +- tasks.py | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.editorconfig b/.editorconfig index b42ca8c..862c1e1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,7 +9,7 @@ insert_final_newline = true trim_trailing_whitespace = true [*.py] -max_line_length = 79 +max_line_length = 88 [*.yml] indent_size = 2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7503e21..1cf1dd8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: rev: 3.8.3 hooks: - id: flake8 - args: [--max-line-length=79] + args: [--max-line-length=88] language_version: python3.7 - repo: https://github.com/timothycrosley/isort diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 663cb5e..45779fa 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -59,10 +59,7 @@ def test_non_empty_file_no_metadata(self): self.pandoc_reader.read(source_path) message = str(context_manager.exception) - self.assertEqual( - "Could not find metadata header '---' or '...'", - message - ) + self.assertEqual("Could not find metadata header '---' or '...'", message) def test_metadata_block_end(self): """Check if the metadata block ends.""" diff --git a/pyproject.toml b/pyproject.toml index 012453d..527bb32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ git-email = "botpub@autopub.rocks" combine_as_imports = true force_grid_wrap = 0 include_trailing_comma = true -line_length = 79 +line_length = 88 multi_line_output = 3 # Sort imports within their section independent of the import type diff --git a/tasks.py b/tasks.py index 742d340..a5dfcb4 100644 --- a/tasks.py +++ b/tasks.py @@ -32,7 +32,7 @@ def black(c, check=False, diff=False): check_flag = "--check" if diff: diff_flag = "--diff" - c.run(f"{VENV}/bin/black {check_flag} {diff_flag} --line-length 79 {PKG_PATH} tasks.py") + c.run(f"{VENV}/bin/black {check_flag} {diff_flag} {PKG_PATH} tasks.py") @task From a6dee23c5d7dbc52ff3422e2b5b1100c4c495a44 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 15:39:05 -0700 Subject: [PATCH 030/102] Updated and improved README with links to Pandoc's documentation --- README.md | 93 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 7f5c860..321b1ce 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ # Pandoc Reader -Pandoc Reader is a [Pelican][] plugin to render documents written in [Pandoc][] Markdown. +Pandoc Reader is a [Pelican](http://getpelican.com) plugin to convert documents written in [Pandoc](https://pandoc.org/) Markdown to HTML 5. -Pandoc's version of Markdown is a flavour of [Markdown][] with extensions and is explained in greater detail [here](https://pandoc.org/MANUAL.html#pandocs-markdown). +[Pandoc's Markdown](https://pandoc.org/MANUAL.html#pandocs-markdown) is a flavour of [Markdown](http://daringfireball.net/projects/markdown/) with extensions. ## Prerequisites -For this plugin to function you must have Pandoc installed on your system. +For this plugin to function you must have Pandoc installed on your local machine. -Please follow the [installation instructions][] to install Pandoc for your system. +Please follow the [installation instructions](https://pandoc.org/installing.html) to install Pandoc. ## Installation -This plugin can be installed via: +The plugin can be installed the command: ```bash python -m pip install pelican-pandoc-reader @@ -20,14 +20,59 @@ python -m pip install pelican-pandoc-reader ## Usage -You may use this plugin whenever you wish to write your source files in Pandoc's Markdown and have them rendered as HTML 5. +This plugin converts Pandoc's Markdown into HTML 5. Formats like [CommonMark](https://commonmark.org/) may be supported at some future time. -This plugin expects to have metadata in YAML format at the top of every Markdown as shown below: +Conversion to formats other than HTML 5 will not be supported. + +### Specifying Pandoc Options + +There are two ways to specify options to Pandoc: + +1. You may configure two constants in your `pelicanconf.py` file namely: + * `PANDOC_ARGS` + * `PANDOC_EXTENSIONS` + + In the `PANDOC_ARGS` parameter you may specify any arguments supported by Pandoc. + + ```python + PANDOC_ARGS = [ + '--mathjax', + '--toc' + ] + ``` + + **Note: We do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** + + In the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). + + ```python + PANDOC_EXTENSIONS = [ + '+footnotes', # Enabled extension + '-pipe_tables' # Disabled extension + ] + ``` + +1. The second method is to specify a path to a YAML file with all your preferences by making use of the `PANDOC_DEFAULT_FILES` constant in your `pelicanconf.py` file. + + ```python + PANDOC_DEFAULT_FILES = [ + '' + ] + ``` + + The format of this file is described [here](https://pandoc.org/MANUAL.html#default-files). + + **Note: Again we do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** + +### Specifying File Metadata + +The plugin expects all markdown files to start with a YAML block as shown below. ```yaml --- title: author: +data: --- ``` @@ -37,44 +82,22 @@ or ... title: author: +date: ... ``` -Failing to provide this metadata will cause the plugin to fail. +**Note: Specifying the file metadata in the format above does not comply with Pelican's format. If you wish to stop using this plugin and switch back to Pelican's native Markdown you may have to change the metadata format.** -Additional command line options may be passed to Pandoc via the `PANDOC_ARGS` parameter. +More information about specifying file metadata is available [here](https://docs.getpelican.com/en/stable/content.html#file-metadata). -```python -PANDOC_ARGS = [ - '--mathjax', - '--toc' -] -``` -Pandoc's markdown extensions may be enabled or disabled via the `PANDOC_EXTENSIONS` parameter. - -```python -PANDOC_EXTENSIONS = [ - '+footnotes', - '-pipe_tables' -] -``` - -More information about Pandoc's extensions can be found [here](https://pandoc.org/MANUAL.html#extensions). ## Contributing -Contributions are welcome and much appreciated. Every little bit helps. You can contribute by improving the documentation, adding missing features, and fixing bugs. You can also help out by reviewing and commenting on [existing issues][]. +Contributions are welcome and much appreciated. Every little bit helps. You can contribute by improving the documentation, adding missing features, and fixing bugs. You can also help out by reviewing and commenting on [existing issues](https://github.com/pelican-plugins/pandoc-reader/issues). -To start contributing to this plugin, review the [Contributing to Pelican][] documentation, beginning with the **Contributing Code** section. +To start contributing to this plugin, review the [Contributing to Pelican](https://docs.getpelican.com/en/latest/contribute.html) documentation, beginning with the **Contributing Code** section. ## Credits -Originally authored by [Hinrich B. Winther](https://github.com/liob), December 2014, and subsequently enhanced by [Nandakumar Chandrasekhar](https://www.linkedin.com/in/nandakumar-chandrasekhar-a400b45b/). - -[installation instructions]: https://pandoc.org/installing.html -[Markdown]: http://daringfireball.net/projects/markdown/ -[Pandoc]: https://pandoc.org/ -[Pelican]: http://getpelican.com -[existing issues]: https://github.com/pelican-plugins/pandoc-reader/issues -[Contributing to Pelican]: https://docs.getpelican.com/en/latest/contribute.html +Originally authored by [Hinrich B. Winther](https://github.com/liob), December 2014, and subsequently enhanced by [Nandakumar Chandrasekhar](https://www.linkedin.com/in/nandakumar-chandrasekhar-a400b45b/) with additional features. From dea0bf3479254c3cde2d03f5385545ac7695a645 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 15:40:19 -0700 Subject: [PATCH 031/102] Fixed README formatting issues --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 321b1ce..a877a51 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,6 @@ date: More information about specifying file metadata is available [here](https://docs.getpelican.com/en/stable/content.html#file-metadata). - - ## Contributing Contributions are welcome and much appreciated. Every little bit helps. You can contribute by improving the documentation, adding missing features, and fixing bugs. You can also help out by reviewing and commenting on [existing issues](https://github.com/pelican-plugins/pandoc-reader/issues). From f8fe46a9eb642d9e7dec13c4c5280fb4865f47f6 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 15:43:47 -0700 Subject: [PATCH 032/102] More tweaks to the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a877a51..0762dab 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Please follow the [installation instructions](https://pandoc.org/installing.html ## Installation -The plugin can be installed the command: +The plugin may be installed using [pip](https://pip.pypa.io/en/stable/installing/): ```bash python -m pip install pelican-pandoc-reader From 88f9610d5820936c3015f83fd4f6d13a6eff79ad Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 18 Oct 2020 15:52:45 -0700 Subject: [PATCH 033/102] Added more information about the two ways to specify options to Pandoc --- README.md | 61 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 0762dab..47db445 100644 --- a/README.md +++ b/README.md @@ -20,49 +20,54 @@ python -m pip install pelican-pandoc-reader ## Usage -This plugin converts Pandoc's Markdown into HTML 5. Formats like [CommonMark](https://commonmark.org/) may be supported at some future time. +This plugin converts Pandoc's Markdown into HTML 5. Input formats like [CommonMark](https://commonmark.org/) may be supported at some future time. Conversion to formats other than HTML 5 will not be supported. ### Specifying Pandoc Options -There are two ways to specify options to Pandoc: +The plugin supports two methods to pass options to Pandoc and are **mutually exclusive**. These methods are described in the sections below. -1. You may configure two constants in your `pelicanconf.py` file namely: - * `PANDOC_ARGS` - * `PANDOC_EXTENSIONS` +#### Method One - In the `PANDOC_ARGS` parameter you may specify any arguments supported by Pandoc. +You may configure two constants in your `pelicanconf.py` file namely: - ```python - PANDOC_ARGS = [ - '--mathjax', - '--toc' - ] - ``` +* `PANDOC_ARGS` +* `PANDOC_EXTENSIONS` - **Note: We do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** +In the `PANDOC_ARGS` parameter you may specify any arguments supported by Pandoc. - In the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). +```python +PANDOC_ARGS = [ + '--mathjax', + '--toc' +] +``` + +**Note: We do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** + +In the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). - ```python - PANDOC_EXTENSIONS = [ - '+footnotes', # Enabled extension - '-pipe_tables' # Disabled extension - ] - ``` +```python +PANDOC_EXTENSIONS = [ + '+footnotes', # Enabled extension + '-pipe_tables' # Disabled extension +] +``` + +#### Method Two -1. The second method is to specify a path to a YAML file with all your preferences by making use of the `PANDOC_DEFAULT_FILES` constant in your `pelicanconf.py` file. +The second method is to specify a path to a YAML file, with all your preferences, by setting the `PANDOC_DEFAULT_FILES` constant in your `pelicanconf.py` file. - ```python - PANDOC_DEFAULT_FILES = [ - '' - ] - ``` +```python +PANDOC_DEFAULT_FILES = [ + '' +] +``` - The format of this file is described [here](https://pandoc.org/MANUAL.html#default-files). +The format of this file is described [here](https://pandoc.org/MANUAL.html#default-files). - **Note: Again we do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** +**Note: Again we do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** ### Specifying File Metadata From f9a76e356b62ee9cf49c551d7fa930117d0865ec Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Mon, 19 Oct 2020 20:05:39 -0700 Subject: [PATCH 034/102] Added code and test for default file support and added PyYAML package --- README.md | 18 ++-- .../pandoc_reader/content/mathjax_content.md | 2 +- .../pandoc_reader/content/no_metadata.md | 2 +- .../pandoc_reader/content/no_metadata_end.md | 6 +- .../pandoc_reader/content/valid_content.md | 2 +- .../content/wrong_metadata_end.md | 6 +- .../defaults/selfcontained_true.yaml | 12 +++ .../defaults/standalone_true.yaml | 12 +++ .../plugins/pandoc_reader/pandoc_reader.py | 94 +++++++++++++++++-- .../pandoc_reader/test_pandoc_reader.py | 63 ++++++++++--- pyproject.toml | 2 + requirements.txt | 1 + 12 files changed, 186 insertions(+), 34 deletions(-) create mode 100644 pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/standalone_true.yaml create mode 100644 requirements.txt diff --git a/README.md b/README.md index 47db445..e413fb5 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,19 @@ Pandoc Reader is a [Pelican](http://getpelican.com) plugin to convert documents ## Prerequisites -For this plugin to function you must have Pandoc installed on your local machine. +For this plugin to function you must have Pandoc and the [PyYAML](https://pypi.org/project/PyYAML/) python package installed on your system. Please follow the [installation instructions](https://pandoc.org/installing.html) to install Pandoc. +To install PyYAML execute the following command using [pip](https://pip.pypa.io/en/stable/installing/): + +```bash +pip install PyYAML==5.3.1 +``` + ## Installation -The plugin may be installed using [pip](https://pip.pypa.io/en/stable/installing/): +The plugin may be installed using pip: ```bash python -m pip install pelican-pandoc-reader @@ -20,9 +26,7 @@ python -m pip install pelican-pandoc-reader ## Usage -This plugin converts Pandoc's Markdown into HTML 5. Input formats like [CommonMark](https://commonmark.org/) may be supported at some future time. - -Conversion to formats other than HTML 5 will not be supported. +This plugin converts Pandoc's Markdown into HTML 5. Conversion to formats other than HTML 5 will not be supported. ### Specifying Pandoc Options @@ -65,9 +69,11 @@ PANDOC_DEFAULT_FILES = [ ] ``` +Using a default file has the added benefit of allowing you to use other markdown flavors supported by Pandoc such as [CommonMark](https://commonmark.org/) and [GitHub-Flavored Markdown](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github). + The format of this file is described [here](https://pandoc.org/MANUAL.html#default-files). -**Note: Again we do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** +**Note: If `--standalone` or `--self-contained` are set to `true` you will get an error message.** ### Specifying File Metadata diff --git a/pelican/plugins/pandoc_reader/content/mathjax_content.md b/pelican/plugins/pandoc_reader/content/mathjax_content.md index 10cf098..62eacbd 100644 --- a/pelican/plugins/pandoc_reader/content/mathjax_content.md +++ b/pelican/plugins/pandoc_reader/content/mathjax_content.md @@ -1,6 +1,6 @@ --- title: MathJax Content -author: Nandakumar Chandrasekhar +author: My Author date: 2020-10-16 --- $$ diff --git a/pelican/plugins/pandoc_reader/content/no_metadata.md b/pelican/plugins/pandoc_reader/content/no_metadata.md index da70023..72aa6f1 100644 --- a/pelican/plugins/pandoc_reader/content/no_metadata.md +++ b/pelican/plugins/pandoc_reader/content/no_metadata.md @@ -1,3 +1,3 @@ # File that does not contain any metadata -This is a file that does not have any metadata +This is a file that does not have any metadata. diff --git a/pelican/plugins/pandoc_reader/content/no_metadata_end.md b/pelican/plugins/pandoc_reader/content/no_metadata_end.md index 089ebda..80736c1 100644 --- a/pelican/plugins/pandoc_reader/content/no_metadata_end.md +++ b/pelican/plugins/pandoc_reader/content/no_metadata_end.md @@ -1,4 +1,4 @@ --- -title: "No Metadata End" -author: "Nandakumar Chandrasekhar" -date: "2020-10-16" +title: No Metadata End +author: My Author +date: 2020-10-16 diff --git a/pelican/plugins/pandoc_reader/content/valid_content.md b/pelican/plugins/pandoc_reader/content/valid_content.md index 10b8a05..6047c02 100644 --- a/pelican/plugins/pandoc_reader/content/valid_content.md +++ b/pelican/plugins/pandoc_reader/content/valid_content.md @@ -1,6 +1,6 @@ --- title: Valid Content -author: Nandakumar Chandrasekhar +author: My Author date: 2020-10-16 --- diff --git a/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md b/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md index 7979bc1..8c13a0c 100644 --- a/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md +++ b/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md @@ -1,5 +1,5 @@ --- -title: "No Metadata End" -author: "Nandakumar Chandrasekhar" -date: "2020-10-16" +title: No Metadata End +author: My Author +date: 2020-10-16 ~~~ diff --git a/pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml b/pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml new file mode 100644 index 0000000..664acb6 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml @@ -0,0 +1,12 @@ +# selfcontained_true.yaml +# +# Defaults for generating HTML5 from Markdown using Pandoc +# using the Pelican Pandoc Reader plugin +# +reader: markdown+smart+citations+implicit_figures +writer: html5 +self-contained: true + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/standalone_true.yaml b/pelican/plugins/pandoc_reader/defaults/standalone_true.yaml new file mode 100644 index 0000000..3ac5db2 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/standalone_true.yaml @@ -0,0 +1,12 @@ +# standalone_true.yaml +# +# Defaults for generating HTML5 from Markdown using Pandoc +# using the Pelican Pandoc Reader plugin +# +reader: markdown+smart+citations+implicit_figures +writer: html5 +standalone: true + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 9aa2671..2a5b2b6 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -1,6 +1,7 @@ """Reader that processes Pandoc Markdown and returns HTML 5.""" import shutil import subprocess +import yaml from pelican.readers import BaseReader from pelican.utils import pelican_open @@ -13,6 +14,16 @@ "%7Bfilename%7D": "{filename}", } +VALID_INPUT_FORMATS = ( + "markdown", + "commonmark", + "gfm" +) + +VALID_OUTPUT_FORMATS = ( + "html", + "html5" +) class PandocReader(BaseReader): """Process files written in Pandoc Markdown.""" @@ -51,24 +62,22 @@ def read(self, source_path): pandoc_cmd.extend(extra_args) else: defaults_cmd_str = "" - for file in self.settings.get("PANDOC_DEFAULT_FILES"): - defaults_cmd_str += " --defaults=file" + for filepath in self.settings.get("PANDOC_DEFAULT_FILES"): + self._check_defaults(filepath) + defaults_cmd_str += " --defaults={0}".format(filepath) # Construct Pandoc command pandoc_cmd = [ "pandoc", defaults_cmd_str, - " --from", - "markdown", - "--to", - "html5", ] - # Execute and retrieve HTML 5 output + # Execute Pandoc command proc = subprocess.Popen( pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) + # retrieve HTML 5 output output = proc.communicate(content.encode("UTF-8"))[0].decode("UTF-8") status = proc.wait() if status: @@ -76,12 +85,81 @@ def read(self, source_path): # Replace all occurrences of %7Bstatic%7D to {static}, # %7Battach%7D to {attach} and %7Bfilename%7D to {filename} - # so that static links are resolved by pelican + # so that static links are resolvable by pelican for encoded_str, raw_str in ENCODED_LINKS_TO_RAW_LINKS_MAP.items(): output = output.replace(encoded_str, raw_str) return output, metadata + def _check_defaults(self, filepath): + """Check if the given Pandoc defaults file has valid values.""" + defaults = {} + + # Convert YAML data to a Python dictionary + with open(filepath) as defaults_file: + defaults = yaml.safe_load(defaults_file) + + standalone = defaults.get("standalone", "") + self_contained = defaults.get("self-contained", "") + + # Raise an exception if standalone is true + if standalone: + raise ValueError( + "The default standalone should be set to false." + ) + + # Raise an exception if self-contained is true + if self_contained: + raise ValueError( + "The default self-contained should be set to false." + ) + + reader_input = defaults.get("reader", "") + from_input = defaults.get("from", "") + + # Case where no input format is specified + if not reader_input and not from_input: + raise ValueError("No input format specified.") + + # Case where reader is specified + elif reader_input and not from_input: + reader_prefix = reader_input.replace("+", "-").split("-")[0] + + # Check to see if the reader_prefix matches a valid input type + if not reader_prefix.startswith(VALID_INPUT_FORMATS): + raise ValueError("Input type has to be a markdown variant.") + + # Case where from is specified + elif not reader_input and from_input: + reader_prefix = from_input.replace("+", "-").split("-")[0] + + # Check to see if the reader_prefix matches a valid input type + if not reader_prefix.startswith(VALID_INPUT_FORMATS): + raise ValueError("Input type has to be a markdown variant.") + + # Case where both reader and from are specified which is not supported + elif reader_input and from_input: + raise ValueError(( + "Specifying both from and reader values is not supported." + " Please specify just one." + )) + + writer_output = defaults.get("writer", "") + to_output = defaults.get("to", "") + + # Case where both writer and to are specified which is not supported + if writer_output and to_output: + raise ValueError(( + "Specifying both to and writer values is not supported." + " Please specify just one." + )) + + # Case where neither writer not to value is set to html + if (writer_output not in VALID_OUTPUT_FORMATS and + to_output not in VALID_OUTPUT_FORMATS): + raise ValueError("Output format type must be html or html5") + + def _process_metadata(self, text): """Process YAML metadata and export.""" metadata = {} diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 45779fa..36a4cf8 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -7,13 +7,13 @@ from pelican.tests.support import get_settings DIR_PATH = os.path.dirname(__file__) -FILE_PATH = os.path.abspath(os.path.join(DIR_PATH, "content")) +CONTENT_PATH = os.path.abspath(os.path.join(DIR_PATH, "content")) +DEFAULTS_PATH = os.path.abspath(os.path.join(DIR_PATH, "defaults")) # Test settings that will be set in pelicanconf.py by plugin users PANDOC_ARGS = ["--mathjax"] PANDOC_EXTENSIONS = ["+smart", "+citations", "+implicit_figures"] - class TestPandocReader(unittest.TestCase): """Test class for pandoc-reader plugin.""" @@ -25,7 +25,7 @@ class TestPandocReader(unittest.TestCase): def test_pandoc_installed(self): """Check if Pandoc is installed.""" - source_path = os.path.join(FILE_PATH, "empty.md") + source_path = os.path.join(CONTENT_PATH, "empty.md") if not shutil.which("pandoc"): # Case where pandoc is not installed @@ -41,7 +41,7 @@ def test_pandoc_installed(self): def test_empty_file(self): """Check if a file is empty.""" - source_path = os.path.join(FILE_PATH, "empty.md") + source_path = os.path.join(CONTENT_PATH, "empty.md") # If the file is empty retrieval of metadata should fail with self.assertRaises(Exception) as context_manager: @@ -52,7 +52,7 @@ def test_empty_file(self): def test_non_empty_file_no_metadata(self): """Check if a file has no metadata.""" - source_path = os.path.join(FILE_PATH, "no_metadata.md") + source_path = os.path.join(CONTENT_PATH, "no_metadata.md") # If the file is not empty but has no metadata it should fail with self.assertRaises(Exception) as context_manager: @@ -63,7 +63,7 @@ def test_non_empty_file_no_metadata(self): def test_metadata_block_end(self): """Check if the metadata block ends.""" - source_path = os.path.join(FILE_PATH, "no_metadata_end.md") + source_path = os.path.join(CONTENT_PATH, "no_metadata_end.md") # Metadata blocks should end with '___' or '...' if not it should fail with self.assertRaises(Exception) as context_manager: @@ -74,7 +74,7 @@ def test_metadata_block_end(self): def test_invalid_metadata_block_end(self): """Check if the metadata block end is wrong.""" - source_path = os.path.join(FILE_PATH, "no_metadata_end.md") + source_path = os.path.join(CONTENT_PATH, "no_metadata_end.md") # Metadata blocks should end with '___' or '...' if not it should fail with self.assertRaises(Exception) as context_manager: @@ -85,7 +85,7 @@ def test_invalid_metadata_block_end(self): def test_valid_file(self): """Check if we get the appropriate output for valid input.""" - source_path = os.path.join(FILE_PATH, "valid_content.md") + source_path = os.path.join(CONTENT_PATH, "valid_content.md") output, metadata = self.pandoc_reader.read(source_path) self.assertEqual( @@ -99,12 +99,12 @@ def test_valid_file(self): ) self.assertEqual("Valid Content", str(metadata["title"])) - self.assertEqual("Nandakumar Chandrasekhar", str(metadata["author"])) + self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) def test_mathjax_content(self): """Check if mathematics is rendered correctly.""" - source_path = os.path.join(FILE_PATH, "mathjax_content.md") + source_path = os.path.join(CONTENT_PATH, "mathjax_content.md") output, metadata = self.pandoc_reader.read(source_path) self.assertEqual( @@ -116,9 +116,50 @@ def test_mathjax_content(self): ) self.assertEqual("MathJax Content", str(metadata["title"])) - self.assertEqual("Nandakumar Chandrasekhar", str(metadata["author"])) + self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + def test_invalid_standalone(self): + """Check if exception is raised if standalone is true.""" + pandoc_default_files = [ + os.path.join(DEFAULTS_PATH, "standalone_true.yaml") + ] + + settings = get_settings( + PANDOC_DEFAULT_FILES=pandoc_default_files + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual( + "The default standalone should be set to false.", message + ) + + def test_invalid_self_contained(self): + """Check if exception is raised if self-contained is true.""" + pandoc_default_files = [ + os.path.join(DEFAULTS_PATH, "selfcontained_true.yaml") + ] + + settings = get_settings( + PANDOC_DEFAULT_FILES=pandoc_default_files + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual( + "The default self-contained should be set to false.", message + ) if __name__ == "__main__": unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 527bb32..6925f56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ classifiers = [ python = "^3.6" pelican = "^4.5" markdown = {version = "^3.2.2", optional = true} +pyyaml = "^5.3.1" [tool.poetry.dev-dependencies] black = {version = "^19.10b0", allow-prereleases = true} @@ -45,6 +46,7 @@ pytest-cov = "^2.7" pytest-pythonpath = "^0.7.3" pytest-sugar = "^0.9.4" Werkzeug = "^1.0" +pyyaml = "^5.3.1" [tool.poetry.extras] markdown = ["markdown"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7a997b5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +PyYAML==5.3.1 From c14e5f5a8bc502d9098077418028ad0501760b08 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Mon, 19 Oct 2020 20:21:14 -0700 Subject: [PATCH 035/102] Fixed formatting and warnings to pass lint checks --- .../plugins/pandoc_reader/pandoc_reader.py | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 2a5b2b6..32c40ad 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -1,29 +1,22 @@ """Reader that processes Pandoc Markdown and returns HTML 5.""" import shutil import subprocess + import yaml +from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open -from pelican import signals - ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", "%7Battach%7D": "{attach}", "%7Bfilename%7D": "{filename}", } -VALID_INPUT_FORMATS = ( - "markdown", - "commonmark", - "gfm" -) +VALID_INPUT_FORMATS = ("markdown", "commonmark", "gfm") +VALID_OUTPUT_FORMATS = ("html", "html5") -VALID_OUTPUT_FORMATS = ( - "html", - "html5" -) class PandocReader(BaseReader): """Process files written in Pandoc Markdown.""" @@ -63,7 +56,7 @@ def read(self, source_path): else: defaults_cmd_str = "" for filepath in self.settings.get("PANDOC_DEFAULT_FILES"): - self._check_defaults(filepath) + self.check_defaults(filepath) defaults_cmd_str += " --defaults={0}".format(filepath) # Construct Pandoc command @@ -91,7 +84,8 @@ def read(self, source_path): return output, metadata - def _check_defaults(self, filepath): + @staticmethod + def check_defaults(filepath): """Check if the given Pandoc defaults file has valid values.""" defaults = {} @@ -104,15 +98,11 @@ def _check_defaults(self, filepath): # Raise an exception if standalone is true if standalone: - raise ValueError( - "The default standalone should be set to false." - ) + raise ValueError("The default standalone should be set to false.") # Raise an exception if self-contained is true if self_contained: - raise ValueError( - "The default self-contained should be set to false." - ) + raise ValueError("The default self-contained should be set to false.") reader_input = defaults.get("reader", "") from_input = defaults.get("from", "") @@ -122,7 +112,7 @@ def _check_defaults(self, filepath): raise ValueError("No input format specified.") # Case where reader is specified - elif reader_input and not from_input: + if reader_input and not from_input: reader_prefix = reader_input.replace("+", "-").split("-")[0] # Check to see if the reader_prefix matches a valid input type @@ -130,7 +120,7 @@ def _check_defaults(self, filepath): raise ValueError("Input type has to be a markdown variant.") # Case where from is specified - elif not reader_input and from_input: + if not reader_input and from_input: reader_prefix = from_input.replace("+", "-").split("-")[0] # Check to see if the reader_prefix matches a valid input type @@ -138,28 +128,33 @@ def _check_defaults(self, filepath): raise ValueError("Input type has to be a markdown variant.") # Case where both reader and from are specified which is not supported - elif reader_input and from_input: - raise ValueError(( - "Specifying both from and reader values is not supported." - " Please specify just one." - )) + if reader_input and from_input: + raise ValueError( + ( + "Specifying both from and reader values is not supported." + " Please specify just one." + ) + ) writer_output = defaults.get("writer", "") to_output = defaults.get("to", "") # Case where both writer and to are specified which is not supported if writer_output and to_output: - raise ValueError(( - "Specifying both to and writer values is not supported." - " Please specify just one." - )) + raise ValueError( + ( + "Specifying both to and writer values is not supported." + " Please specify just one." + ) + ) # Case where neither writer not to value is set to html - if (writer_output not in VALID_OUTPUT_FORMATS and - to_output not in VALID_OUTPUT_FORMATS): + if ( + writer_output not in VALID_OUTPUT_FORMATS + and to_output not in VALID_OUTPUT_FORMATS + ): raise ValueError("Output format type must be html or html5") - def _process_metadata(self, text): """Process YAML metadata and export.""" metadata = {} From 991f665f2e34af2388f4dda402f7364de6871878 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Mon, 19 Oct 2020 20:26:44 -0700 Subject: [PATCH 036/102] Fixing import sorting error --- pelican/plugins/pandoc_reader/pandoc_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 32c40ad..652d506 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -2,12 +2,12 @@ import shutil import subprocess -import yaml - from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open +import yaml + ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", "%7Battach%7D": "{attach}", From 97b97ea82f739f373011923f38b292aee5d3145d Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Mon, 19 Oct 2020 20:30:16 -0700 Subject: [PATCH 037/102] Fixing import error --- pelican/plugins/pandoc_reader/pandoc_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 652d506..32c40ad 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -2,12 +2,12 @@ import shutil import subprocess +import yaml + from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open -import yaml - ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", "%7Battach%7D": "{attach}", From 606b2a90beaea7fbfe7475247fdfe0cc51cf18ca Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Mon, 19 Oct 2020 20:42:53 -0700 Subject: [PATCH 038/102] Fixing import error --- pelican/plugins/pandoc_reader/pandoc_reader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 32c40ad..f0c41ee 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -1,7 +1,6 @@ """Reader that processes Pandoc Markdown and returns HTML 5.""" import shutil import subprocess - import yaml from pelican import signals From 5860380a1f4c3b92f5ec2b2278178e38d0a7e984 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 21 Oct 2020 15:30:05 -0700 Subject: [PATCH 039/102] Added tests using default files --- README.md | 2 +- .../content/valid_content_with_raw_paths.md | 15 + .../defaults/from_reader_both_given.yaml | 11 + .../defaults/invalid_from_input_format.yaml | 10 + .../defaults/invalid_reader_input_format.yaml | 10 + .../defaults/invalid_to_output_format.yaml | 10 + .../invalid_writer_output_format.yaml | 10 + .../defaults/no_input_format.yaml | 9 + .../defaults/no_output_format.yaml | 9 + .../defaults/selfcontained_true.yaml | 3 +- .../defaults/standalone_true.yaml | 3 +- .../defaults/to_writer_both_given.yaml | 11 + .../defaults/valid_defaults.yaml | 10 + .../plugins/pandoc_reader/pandoc_reader.py | 40 ++- .../pandoc_reader/test_pandoc_reader.py | 333 ++++++++++++++++-- pyproject.toml | 2 +- 16 files changed, 440 insertions(+), 48 deletions(-) create mode 100644 pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md create mode 100644 pelican/plugins/pandoc_reader/defaults/from_reader_both_given.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/invalid_from_input_format.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/invalid_reader_input_format.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/invalid_to_output_format.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/invalid_writer_output_format.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/no_input_format.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/no_output_format.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/to_writer_both_given.yaml create mode 100644 pelican/plugins/pandoc_reader/defaults/valid_defaults.yaml diff --git a/README.md b/README.md index e413fb5..7b8bdff 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ PANDOC_ARGS = [ ] ``` -**Note: We do not recommend specifying `--standalone` or `--self-contained` as this would conflict with you theme's template files.** +**Note: Specifying the arguments `--standalone` or `--self-contained` are not supported and will throw an error.** In the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). diff --git a/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md b/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md new file mode 100644 index 0000000..2c3f27b --- /dev/null +++ b/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md @@ -0,0 +1,15 @@ +--- +title: Valid Content with Fictitious Raw Paths +author: My Author +date: 2020-10-16 +--- + +# Valid Content with Fictitious Raw Paths + +This is some valid content that should pass. If it does not pass we will know something is wrong. + +Our fictitious internal files are available [at]({filename}/path/to/file): + +Our fictitious static files are available [at]({static}/path/to/file): + +Our fictitious attachments are available [at]({attach}path/to/file): diff --git a/pelican/plugins/pandoc_reader/defaults/from_reader_both_given.yaml b/pelican/plugins/pandoc_reader/defaults/from_reader_both_given.yaml new file mode 100644 index 0000000..de711a9 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/from_reader_both_given.yaml @@ -0,0 +1,11 @@ +# from_reader_both_given.yaml +# +# A test default file that sets both reader and from +# +reader: markdown+smart+citations+implicit_figures +from: markdown+smart+citations+implicit_figures +writer: html5 + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/invalid_from_input_format.yaml b/pelican/plugins/pandoc_reader/defaults/invalid_from_input_format.yaml new file mode 100644 index 0000000..90b74a6 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/invalid_from_input_format.yaml @@ -0,0 +1,10 @@ +# invalid_from_input_format.yaml +# +# A test default file that sets from to an invalid input format +# +from: myinputformat +to: html5 + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/invalid_reader_input_format.yaml b/pelican/plugins/pandoc_reader/defaults/invalid_reader_input_format.yaml new file mode 100644 index 0000000..0cde43d --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/invalid_reader_input_format.yaml @@ -0,0 +1,10 @@ +# invalid_reader_input_format.yaml +# +# A test default file that sets reader to an invalid input format +# +reader: myinputformat +writer: html5 + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/invalid_to_output_format.yaml b/pelican/plugins/pandoc_reader/defaults/invalid_to_output_format.yaml new file mode 100644 index 0000000..95e5be2 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/invalid_to_output_format.yaml @@ -0,0 +1,10 @@ +# invalid_to_output_format.yaml +# +# A test default file that sets to an invalid output format +# +from: markdown+smart+citations+implicit_figures +to: myoutputformat + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/invalid_writer_output_format.yaml b/pelican/plugins/pandoc_reader/defaults/invalid_writer_output_format.yaml new file mode 100644 index 0000000..2e1bc08 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/invalid_writer_output_format.yaml @@ -0,0 +1,10 @@ +# invalid_writer_output_format.yaml +# +# A test default file that sets writer to an invalid output format +# +reader: markdown+smart+citations+implicit_figures +writer: myoutputformat + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/no_input_format.yaml b/pelican/plugins/pandoc_reader/defaults/no_input_format.yaml new file mode 100644 index 0000000..ae4d890 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/no_input_format.yaml @@ -0,0 +1,9 @@ +# no_input_format.yaml +# +# A test default file that specifies no input format +# +writer: html5 + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/no_output_format.yaml b/pelican/plugins/pandoc_reader/defaults/no_output_format.yaml new file mode 100644 index 0000000..cc65a82 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/no_output_format.yaml @@ -0,0 +1,9 @@ +# no_output_format.yaml +# +# A test default file that specifies no output format +# +reader: markdown + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml b/pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml index 664acb6..5b98857 100644 --- a/pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml +++ b/pelican/plugins/pandoc_reader/defaults/selfcontained_true.yaml @@ -1,7 +1,6 @@ # selfcontained_true.yaml # -# Defaults for generating HTML5 from Markdown using Pandoc -# using the Pelican Pandoc Reader plugin +# A test default file that sets self-contained to true # reader: markdown+smart+citations+implicit_figures writer: html5 diff --git a/pelican/plugins/pandoc_reader/defaults/standalone_true.yaml b/pelican/plugins/pandoc_reader/defaults/standalone_true.yaml index 3ac5db2..5a9fda3 100644 --- a/pelican/plugins/pandoc_reader/defaults/standalone_true.yaml +++ b/pelican/plugins/pandoc_reader/defaults/standalone_true.yaml @@ -1,7 +1,6 @@ # standalone_true.yaml # -# Defaults for generating HTML5 from Markdown using Pandoc -# using the Pelican Pandoc Reader plugin +# A test default file that sets standalone to true # reader: markdown+smart+citations+implicit_figures writer: html5 diff --git a/pelican/plugins/pandoc_reader/defaults/to_writer_both_given.yaml b/pelican/plugins/pandoc_reader/defaults/to_writer_both_given.yaml new file mode 100644 index 0000000..81989b4 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/to_writer_both_given.yaml @@ -0,0 +1,11 @@ +# to_writer_both_given.yaml +# +# A test default file that sets both reader and from +# +reader: markdown+smart+citations+implicit_figures +to: html5 +writer: html5 + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/defaults/valid_defaults.yaml b/pelican/plugins/pandoc_reader/defaults/valid_defaults.yaml new file mode 100644 index 0000000..0ae12ce --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/valid_defaults.yaml @@ -0,0 +1,10 @@ +# valid_defaults.yml +# +# A test default file that is valid +# +reader: markdown+smart+citations+implicit_figures +writer: html5 + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index f0c41ee..9f3ecb7 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -1,7 +1,8 @@ """Reader that processes Pandoc Markdown and returns HTML 5.""" import shutil import subprocess -import yaml + +from yaml import safe_load from pelican import signals from pelican.readers import BaseReader @@ -15,6 +16,7 @@ VALID_INPUT_FORMATS = ("markdown", "commonmark", "gfm") VALID_OUTPUT_FORMATS = ("html", "html5") +UNSUPPORTED_ARGUMENTS = ("--standalone", "--self-contained") class PandocReader(BaseReader): @@ -38,7 +40,7 @@ def read(self, source_path): # Get arguments and extensions if not self.settings.get("PANDOC_DEFAULT_FILES"): - extra_args = self.settings.get("PANDOC_ARGS", []) + arguments = self.settings.get("PANDOC_ARGS", []) extensions = self.settings.get("PANDOC_EXTENSIONS", "") if isinstance(extensions, list): extensions = "".join(extensions) @@ -51,18 +53,17 @@ def read(self, source_path): "--to", "html5", ] - pandoc_cmd.extend(extra_args) + + self.check_arguments(arguments) + pandoc_cmd.extend(arguments) else: - defaults_cmd_str = "" + default_files_cmd = [] for filepath in self.settings.get("PANDOC_DEFAULT_FILES"): self.check_defaults(filepath) - defaults_cmd_str += " --defaults={0}".format(filepath) + default_files_cmd.append("--defaults={0}".format(filepath)) # Construct Pandoc command - pandoc_cmd = [ - "pandoc", - defaults_cmd_str, - ] + pandoc_cmd = ["pandoc"] + default_files_cmd # Execute Pandoc command proc = subprocess.Popen( @@ -83,6 +84,13 @@ def read(self, source_path): return output, metadata + @staticmethod + def check_arguments(arguments): + """Check to see that only supported arguments have been passed.""" + for argument in arguments: + if argument in UNSUPPORTED_ARGUMENTS: + raise ValueError("Argument {0} is not supported.".format(argument)) + @staticmethod def check_defaults(filepath): """Check if the given Pandoc defaults file has valid values.""" @@ -90,7 +98,7 @@ def check_defaults(filepath): # Convert YAML data to a Python dictionary with open(filepath) as defaults_file: - defaults = yaml.safe_load(defaults_file) + defaults = safe_load(defaults_file) standalone = defaults.get("standalone", "") self_contained = defaults.get("self-contained", "") @@ -130,7 +138,7 @@ def check_defaults(filepath): if reader_input and from_input: raise ValueError( ( - "Specifying both from and reader values is not supported." + "Specifying both from and reader is not supported." " Please specify just one." ) ) @@ -142,17 +150,17 @@ def check_defaults(filepath): if writer_output and to_output: raise ValueError( ( - "Specifying both to and writer values is not supported." + "Specifying both to and writer is not supported." " Please specify just one." ) ) - # Case where neither writer not to value is set to html + # Case where neither writer nor to value is set to html if ( writer_output not in VALID_OUTPUT_FORMATS and to_output not in VALID_OUTPUT_FORMATS ): - raise ValueError("Output format type must be html or html5") + raise ValueError("Output format type must be html or html5.") def _process_metadata(self, text): """Process YAML metadata and export.""" @@ -160,11 +168,11 @@ def _process_metadata(self, text): # Check that the given text is not empty if not text: - raise Exception("Could not find metadata. File is empty") + raise Exception("Could not find metadata. File is empty.") # Check that the first line of the file starts with a YAML header if text[0].strip() not in ["---", "..."]: - raise Exception("Could not find metadata header '---' or '...'") + raise Exception("Could not find metadata header '...' or '---'.") # Find the end of the YAML block lines = text[1:] diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 36a4cf8..cb6056e 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -4,6 +4,7 @@ import unittest from pandoc_reader import PandocReader + from pelican.tests.support import get_settings DIR_PATH = os.path.dirname(__file__) @@ -14,23 +15,24 @@ PANDOC_ARGS = ["--mathjax"] PANDOC_EXTENSIONS = ["+smart", "+citations", "+implicit_figures"] + class TestPandocReader(unittest.TestCase): """Test class for pandoc-reader plugin.""" - settings = get_settings( - PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS - ) - - pandoc_reader = PandocReader(settings) - + # Test using pelicanconf settings variables def test_pandoc_installed(self): """Check if Pandoc is installed.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "empty.md") if not shutil.which("pandoc"): # Case where pandoc is not installed with self.assertRaises(Exception) as context_manager: - self.pandoc_reader.read(source_path) + pandoc_reader.read(source_path) message = str(context_manager.exception) self.assertEqual("Could not find Pandoc. Please install.", message) @@ -41,52 +43,109 @@ def test_pandoc_installed(self): def test_empty_file(self): """Check if a file is empty.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "empty.md") # If the file is empty retrieval of metadata should fail with self.assertRaises(Exception) as context_manager: - self.pandoc_reader.read(source_path) + pandoc_reader.read(source_path) message = str(context_manager.exception) - self.assertEqual("Could not find metadata. File is empty", message) + self.assertEqual("Could not find metadata. File is empty.", message) def test_non_empty_file_no_metadata(self): """Check if a file has no metadata.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "no_metadata.md") # If the file is not empty but has no metadata it should fail with self.assertRaises(Exception) as context_manager: - self.pandoc_reader.read(source_path) + pandoc_reader.read(source_path) message = str(context_manager.exception) - self.assertEqual("Could not find metadata header '---' or '...'", message) + self.assertEqual("Could not find metadata header '...' or '---'.", message) def test_metadata_block_end(self): """Check if the metadata block ends.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "no_metadata_end.md") # Metadata blocks should end with '___' or '...' if not it should fail with self.assertRaises(Exception) as context_manager: - self.pandoc_reader.read(source_path) + pandoc_reader.read(source_path) message = str(context_manager.exception) self.assertEqual("Could not find end of metadata block.", message) def test_invalid_metadata_block_end(self): """Check if the metadata block end is wrong.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "no_metadata_end.md") # Metadata blocks should end with '___' or '...' if not it should fail with self.assertRaises(Exception) as context_manager: - self.pandoc_reader.read(source_path) + pandoc_reader.read(source_path) message = str(context_manager.exception) self.assertEqual("Could not find end of metadata block.", message) + def test_invalid_standalone_argument(self): + """Check that specifying --standalone raises an exception.""" + pandoc_arguments = ["--standalone"] + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=pandoc_arguments + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Argument --standalone is not supported.", message) + + def test_invalid_self_contained_argument(self): + """Check that specifying --self-contained raises an exception.""" + pandoc_arguments = ["--self-contained"] + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=pandoc_arguments + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Argument --self-contained is not supported.", message) + def test_valid_file(self): """Check if we get the appropriate output for valid input.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "valid_content.md") - output, metadata = self.pandoc_reader.read(source_path) + output, metadata = pandoc_reader.read(source_path) self.assertEqual( ( @@ -104,8 +163,13 @@ def test_valid_file(self): def test_mathjax_content(self): """Check if mathematics is rendered correctly.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "mathjax_content.md") - output, metadata = self.pandoc_reader.read(source_path) + output, metadata = pandoc_reader.read(source_path) self.assertEqual( ( @@ -119,15 +183,128 @@ def test_mathjax_content(self): self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + def test_encoded_to_raw_conversion(self): + """Check if raw paths are left untouched in output returned""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content_with_raw_paths.md") + output, metadata = pandoc_reader.read(source_path) + + # Setting this so that assert is able to execute the difference + self.maxDiff = None + + self.assertEqual( + ( + '

' + "Valid Content with Fictitious Raw Paths

\n" + "

This is some valid content that should pass." + " If it does not pass we will know something is wrong.

\n" + "

Our fictitious internal files are available" + ' at:

\n' + "

Our fictitious static files are available" + ' at:

\n' + "

Our fictitious attachments are available" + ' at:

\n' + ), + output, + ) + + self.assertEqual( + "Valid Content with Fictitious Raw Paths", str(metadata["title"]) + ) + self.assertEqual("My Author", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + + # Tests using default files def test_invalid_standalone(self): """Check if exception is raised if standalone is true.""" + pandoc_default_files = [os.path.join(DEFAULTS_PATH, "standalone_true.yaml")] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("The default standalone should be set to false.", message) + + def test_invalid_self_contained(self): + """Check if exception is raised if self-contained is true.""" + pandoc_default_files = [os.path.join(DEFAULTS_PATH, "selfcontained_true.yaml")] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("The default self-contained should be set to false.", message) + + def test_no_input_format(self): + """Check if exception is raised if no input format is specified.""" + pandoc_default_files = [os.path.join(DEFAULTS_PATH, "no_input_format.yaml")] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("No input format specified.", message) + + def test_invalid_reader_input_format(self): + """Check if exception is raised if reader input format is invalid.""" pandoc_default_files = [ - os.path.join(DEFAULTS_PATH, "standalone_true.yaml") + os.path.join(DEFAULTS_PATH, "invalid_reader_input_format.yaml") ] - settings = get_settings( - PANDOC_DEFAULT_FILES=pandoc_default_files - ) + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Input type has to be a markdown variant.", message) + + def test_invalid_from_input_format(self): + """Check if exception is raised if from input format is invalid.""" + pandoc_default_files = [ + os.path.join(DEFAULTS_PATH, "invalid_from_input_format.yaml") + ] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Input type has to be a markdown variant.", message) + + def test_from_reader_both_given(self): + """Check if exception is raised if from and reader are both given.""" + pandoc_default_files = [ + os.path.join(DEFAULTS_PATH, "from_reader_both_given.yaml") + ] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "valid_content.md") @@ -137,19 +314,59 @@ def test_invalid_standalone(self): message = str(context_manager.exception) self.assertEqual( - "The default standalone should be set to false.", message + ( + "Specifying both from and reader is not supported." + " Please specify just one." + ), + message, ) - def test_invalid_self_contained(self): - """Check if exception is raised if self-contained is true.""" + def test_to_writer_both_given(self): + """Check if exception is raised if to and writer are both given.""" pandoc_default_files = [ - os.path.join(DEFAULTS_PATH, "selfcontained_true.yaml") + os.path.join(DEFAULTS_PATH, "to_writer_both_given.yaml") ] - settings = get_settings( - PANDOC_DEFAULT_FILES=pandoc_default_files + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual( + ( + "Specifying both to and writer is not supported." + " Please specify just one." + ), + message, ) + def test_no_output_format(self): + """Check if exception is raised if no output format is specified.""" + pandoc_default_files = [os.path.join(DEFAULTS_PATH, "no_output_format.yaml")] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Output format type must be html or html5.", message) + + def test_invalid_writer_output_format(self): + """Check if exception is raised if writer output format is invalid.""" + pandoc_default_files = [ + os.path.join(DEFAULTS_PATH, "invalid_writer_output_format.yaml") + ] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + pandoc_reader = PandocReader(settings) source_path = os.path.join(CONTENT_PATH, "valid_content.md") @@ -157,9 +374,73 @@ def test_invalid_self_contained(self): pandoc_reader.read(source_path) message = str(context_manager.exception) + self.assertEqual("Output format type must be html or html5.", message) + + def test_invalid_to_output_format(self): + """Check if exception is raised if to output format is invalid.""" + pandoc_default_files = [ + os.path.join(DEFAULTS_PATH, "invalid_to_output_format.yaml") + ] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + + with self.assertRaises(ValueError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("Output format type must be html or html5.", message) + + def test_valid_file_with_valid_defaults(self): + """Check if we get the appropriate output specifying defaults.""" + pandoc_default_files = [os.path.join(DEFAULTS_PATH, "valid_defaults.yaml")] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + + source_path = os.path.join(CONTENT_PATH, "valid_content.md") + output, metadata = pandoc_reader.read(source_path) + self.assertEqual( - "The default self-contained should be set to false.", message + ( + '

Valid Content

\n

This' + " is some valid content that should pass." + " If it does not pass we" + " will know something is wrong.

\n" + ), + output, + ) + + self.assertEqual("Valid Content", str(metadata["title"])) + self.assertEqual("My Author", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + + def test_mathjax_with_valid_defaults(self): + """Check if mathematics is rendered correctly with defaults.""" + pandoc_default_files = [os.path.join(DEFAULTS_PATH, "valid_defaults.yaml")] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + + pandoc_reader = PandocReader(settings) + + source_path = os.path.join(CONTENT_PATH, "mathjax_content.md") + output, metadata = pandoc_reader.read(source_path) + + self.assertEqual( + ( + '

\\[\ne^{i\\theta} = ' + "\\cos\\theta + i \\sin\\theta.\n\\]

\n" + ), + output, ) + self.assertEqual("MathJax Content", str(metadata["title"])) + self.assertEqual("My Author", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + + if __name__ == "__main__": unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 6925f56..4e451a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "pelican-pandoc-reader" version = "0.0.1" -description = "Pelican plugin to render Pandoc Markdown files" +description = "Pelican plugin to convert Pandoc Markdown files to HTML5." authors = ["Nandakumar Chandrasekhar "] license = "AGPL-3.0" readme = "README.md" From 4428a9169572e6c80aa89b572ff6383d98530431 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 21 Oct 2020 15:33:58 -0700 Subject: [PATCH 040/102] Trying to fix linting errors with imports --- pelican/plugins/pandoc_reader/pandoc_reader.py | 1 - pelican/plugins/pandoc_reader/test_pandoc_reader.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 9f3ecb7..5567740 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -3,7 +3,6 @@ import subprocess from yaml import safe_load - from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index cb6056e..92b9af9 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -4,7 +4,6 @@ import unittest from pandoc_reader import PandocReader - from pelican.tests.support import get_settings DIR_PATH = os.path.dirname(__file__) From 90d24b605e81245ea64662be2fed0349ba3fbaeb Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 21 Oct 2020 15:39:59 -0700 Subject: [PATCH 041/102] Output isort diff to get insight into import formatting error --- tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks.py b/tasks.py index a5dfcb4..82f52cf 100644 --- a/tasks.py +++ b/tasks.py @@ -52,6 +52,7 @@ def flake8(c): @task def lint(c): + isort(c, diff=True) isort(c, check=True) black(c, check=True) flake8(c) From ffd94e480218d30d585756170b16472704225c66 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 21 Oct 2020 15:42:55 -0700 Subject: [PATCH 042/102] Fixing import formatting errors --- pelican/plugins/pandoc_reader/pandoc_reader.py | 5 +++-- tasks.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 5567740..f8e70b2 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -2,10 +2,11 @@ import shutil import subprocess -from yaml import safe_load -from pelican import signals from pelican.readers import BaseReader from pelican.utils import pelican_open +from yaml import safe_load + +from pelican import signals ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", diff --git a/tasks.py b/tasks.py index 82f52cf..a5dfcb4 100644 --- a/tasks.py +++ b/tasks.py @@ -52,7 +52,6 @@ def flake8(c): @task def lint(c): - isort(c, diff=True) isort(c, check=True) black(c, check=True) flake8(c) From 12fefeb1714b4481d221ab53a279ba19379b8e06 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Wed, 21 Oct 2020 16:11:23 -0700 Subject: [PATCH 043/102] Updated README with more information about usage and dependencies --- README.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7b8bdff..8bbc09b 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,14 @@ Please follow the [installation instructions](https://pandoc.org/installing.html To install PyYAML execute the following command using [pip](https://pip.pypa.io/en/stable/installing/): ```bash -pip install PyYAML==5.3.1 +pip install PyYAML ``` +This package has been tested using the following versions of the above dependencies: + +* Pandoc 2.11.0 +* PyYAML 5.3.1 + ## Installation The plugin may be installed using pip: @@ -32,14 +37,14 @@ This plugin converts Pandoc's Markdown into HTML 5. Conversion to formats other The plugin supports two methods to pass options to Pandoc and are **mutually exclusive**. These methods are described in the sections below. -#### Method One +#### Method One: Using Settings in `pelicanconf.py` -You may configure two constants in your `pelicanconf.py` file namely: +The first method involves configuring two settings in your `pelicanconf.py` file: * `PANDOC_ARGS` * `PANDOC_EXTENSIONS` -In the `PANDOC_ARGS` parameter you may specify any arguments supported by Pandoc. +In the `PANDOC_ARGS` parameter you may specify any argument supported by Pandoc ah shown below: ```python PANDOC_ARGS = [ @@ -50,7 +55,7 @@ PANDOC_ARGS = [ **Note: Specifying the arguments `--standalone` or `--self-contained` are not supported and will throw an error.** -In the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). +Then in the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of the supported [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). ```python PANDOC_EXTENSIONS = [ @@ -59,9 +64,11 @@ PANDOC_EXTENSIONS = [ ] ``` -#### Method Two +#### Method Two: Using Pandoc Defaults Files + +The second method involves specifying the path(s) to one or more YAML file(s), with all your preferences. -The second method is to specify a path to a YAML file, with all your preferences, by setting the `PANDOC_DEFAULT_FILES` constant in your `pelicanconf.py` file. +These paths should be set in your `pelicanconf.py` file by using the setting `PANDOC_DEFAULT_FILES`. The paths maybe absolute or relative but we recommend using relative paths as they are more portable. ```python PANDOC_DEFAULT_FILES = [ @@ -97,9 +104,9 @@ date: ... ``` -**Note: Specifying the file metadata in the format above does not comply with Pelican's format. If you wish to stop using this plugin and switch back to Pelican's native Markdown you may have to change the metadata format.** +**Note: Specifying the file metadata in the format above is a requirement of Pandoc. Pelican's recommended format is different and may require you to rewrite the metadata in your files, if you stop using this plugin.** -More information about specifying file metadata is available [here](https://docs.getpelican.com/en/stable/content.html#file-metadata). +More information about Pelican's predefined metadata is available [here](https://docs.getpelican.com/en/stable/content.html#file-metadata). ## Contributing @@ -109,4 +116,4 @@ To start contributing to this plugin, review the [Contributing to Pelican](https ## Credits -Originally authored by [Hinrich B. Winther](https://github.com/liob), December 2014, and subsequently enhanced by [Nandakumar Chandrasekhar](https://www.linkedin.com/in/nandakumar-chandrasekhar-a400b45b/) with additional features. +Originally authored by [Hinrich B. Winther](https://github.com/liob), December 2014, and subsequently forked and enhanced by [Nandakumar Chandrasekhar](https://www.linkedin.com/in/nandakumar-chandrasekhar-a400b45b/) with additional features in October 2020. From 10fcc926dcae65e55ca465f9bb12c1763c4e479f Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Tue, 27 Oct 2020 04:32:29 -0700 Subject: [PATCH 044/102] Added capability to add title metadata as first heading --- README.md | 4 +++ .../content/no_title_in_metadata.md | 5 ++++ .../pandoc_reader/content/valid_content.md | 3 --- .../content/valid_content_with_raw_paths.md | 3 --- .../plugins/pandoc_reader/pandoc_reader.py | 26 +++++++++++++++++-- .../pandoc_reader/test_pandoc_reader.py | 22 ++++++++++++++-- pyproject.toml | 2 +- 7 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 pelican/plugins/pandoc_reader/content/no_title_in_metadata.md diff --git a/README.md b/README.md index 8bbc09b..447e9ed 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ date: ... ``` +The plugin automatically insert title metadata as the main heading of your post or page. Therefore, there is no need to specify the first heading in your content. + +Failing to provide a title metadata field will result in an error. + **Note: Specifying the file metadata in the format above is a requirement of Pandoc. Pelican's recommended format is different and may require you to rewrite the metadata in your files, if you stop using this plugin.** More information about Pelican's predefined metadata is available [here](https://docs.getpelican.com/en/stable/content.html#file-metadata). diff --git a/pelican/plugins/pandoc_reader/content/no_title_in_metadata.md b/pelican/plugins/pandoc_reader/content/no_title_in_metadata.md new file mode 100644 index 0000000..62306fc --- /dev/null +++ b/pelican/plugins/pandoc_reader/content/no_title_in_metadata.md @@ -0,0 +1,5 @@ +--- +author: My Author +date: 2020-10-16 +--- +This is some valid content that should pass. If it does not pass we will know something is wrong. diff --git a/pelican/plugins/pandoc_reader/content/valid_content.md b/pelican/plugins/pandoc_reader/content/valid_content.md index 6047c02..da14010 100644 --- a/pelican/plugins/pandoc_reader/content/valid_content.md +++ b/pelican/plugins/pandoc_reader/content/valid_content.md @@ -3,7 +3,4 @@ title: Valid Content author: My Author date: 2020-10-16 --- - -# Valid Content - This is some valid content that should pass. If it does not pass we will know something is wrong. diff --git a/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md b/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md index 2c3f27b..0b16a71 100644 --- a/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md +++ b/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md @@ -3,9 +3,6 @@ title: Valid Content with Fictitious Raw Paths author: My Author date: 2020-10-16 --- - -# Valid Content with Fictitious Raw Paths - This is some valid content that should pass. If it does not pass we will know something is wrong. Our fictitious internal files are available [at]({filename}/path/to/file): diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index f8e70b2..72f728a 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -35,8 +35,14 @@ def read(self, source_path): with pelican_open(source_path) as file_content: content = file_content + # Evaluate the list immediately to stop it from becoming a generator + content_line_by_line = list(content.splitlines()) + # Parse YAML metadata - metadata = self._process_metadata(list(content.splitlines())) + metadata, yaml_end = self._process_metadata(content_line_by_line) + + # Insert title metadata argument in content after metadata block + content = self.insert_title(content_line_by_line, metadata, yaml_end) # Get arguments and extensions if not self.settings.get("PANDOC_DEFAULT_FILES"): @@ -162,6 +168,22 @@ def check_defaults(filepath): ): raise ValueError("Output format type must be html or html5.") + @staticmethod + def insert_title(content, metadata, yaml_end): + # Construct page title retrieved from metadata + try: + title = "# {}".format(str(metadata["title"])) + except KeyError: + raise KeyError("No title field found in metadata.") + + # Insert title to the start of the text + content.insert(yaml_end + 2, title) + + # Reconstruct the content back into string delimited by new lines + content = "\n".join(list(content)) + + return content + def _process_metadata(self, text): """Process YAML metadata and export.""" metadata = {} @@ -192,7 +214,7 @@ def _process_metadata(self, text): if len(metalist) == 2: key, value = metalist[0].lower(), metalist[1].strip() metadata[key] = self.process_metadata(key, value) - return metadata + return metadata, yaml_end def add_reader(readers): diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 92b9af9..80b6798 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -104,6 +104,22 @@ def test_invalid_metadata_block_end(self): message = str(context_manager.exception) self.assertEqual("Could not find end of metadata block.", message) + def test_no_title_field_in_metadata(self): + """Check if an exception is raised if no title is found in metadata.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "no_title_in_metadata.md") + + # Metadata blocks should end with '___' or '...' if not it should fail + with self.assertRaises(KeyError) as context_manager: + pandoc_reader.read(source_path) + + message = str(context_manager.exception) + self.assertEqual("'No title field found in metadata.'", message) + def test_invalid_standalone_argument(self): """Check that specifying --standalone raises an exception.""" pandoc_arguments = ["--standalone"] @@ -148,8 +164,8 @@ def test_valid_file(self): self.assertEqual( ( - '

Valid Content

\n

This' - " is some valid content that should pass." + '

Valid Content

\n' + "

This is some valid content that should pass." " If it does not pass we" " will know something is wrong.

\n" ), @@ -172,6 +188,7 @@ def test_mathjax_content(self): self.assertEqual( ( + '

MathJax Content

\n' '

\\[\ne^{i\\theta} = ' "\\cos\\theta + i \\sin\\theta.\n\\]

\n" ), @@ -430,6 +447,7 @@ def test_mathjax_with_valid_defaults(self): self.assertEqual( ( + '

MathJax Content

\n' '

\\[\ne^{i\\theta} = ' "\\cos\\theta + i \\sin\\theta.\n\\]

\n" ), diff --git a/pyproject.toml b/pyproject.toml index 4e451a8..20bc6bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ packages = [ ] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Environment :: Console", "Framework :: Pelican", "Framework :: Pelican :: Plugins", From 04a9f4f75e7755b60ed47bd8a1a6b34f3390346f Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 30 Oct 2020 04:36:39 -0700 Subject: [PATCH 045/102] Removal of leading and trailing quotes before saving as metadata --- .../plugins/pandoc_reader/content/mathjax_content.md | 6 +++--- .../plugins/pandoc_reader/content/no_metadata_end.md | 6 +++--- .../pandoc_reader/content/no_title_in_metadata.md | 4 ++-- .../plugins/pandoc_reader/content/valid_content.md | 6 +++--- .../content/valid_content_with_raw_paths.md | 6 +++--- .../pandoc_reader/content/wrong_metadata_end.md | 6 +++--- pelican/plugins/pandoc_reader/pandoc_reader.py | 2 +- requirements.txt | 11 +++++++++++ 8 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pelican/plugins/pandoc_reader/content/mathjax_content.md b/pelican/plugins/pandoc_reader/content/mathjax_content.md index 62eacbd..f982084 100644 --- a/pelican/plugins/pandoc_reader/content/mathjax_content.md +++ b/pelican/plugins/pandoc_reader/content/mathjax_content.md @@ -1,7 +1,7 @@ --- -title: MathJax Content -author: My Author -date: 2020-10-16 +title: "MathJax Content" +author: "My Author" +date: "2020-10-16" --- $$ e^{i\theta} = \cos\theta + i \sin\theta. diff --git a/pelican/plugins/pandoc_reader/content/no_metadata_end.md b/pelican/plugins/pandoc_reader/content/no_metadata_end.md index 80736c1..195442c 100644 --- a/pelican/plugins/pandoc_reader/content/no_metadata_end.md +++ b/pelican/plugins/pandoc_reader/content/no_metadata_end.md @@ -1,4 +1,4 @@ --- -title: No Metadata End -author: My Author -date: 2020-10-16 +title: "No Metadata End" +author: "My Author" +date: "2020-10-16" diff --git a/pelican/plugins/pandoc_reader/content/no_title_in_metadata.md b/pelican/plugins/pandoc_reader/content/no_title_in_metadata.md index 62306fc..31da45b 100644 --- a/pelican/plugins/pandoc_reader/content/no_title_in_metadata.md +++ b/pelican/plugins/pandoc_reader/content/no_title_in_metadata.md @@ -1,5 +1,5 @@ --- -author: My Author -date: 2020-10-16 +author: "My Author" +date: "2020-10-16" --- This is some valid content that should pass. If it does not pass we will know something is wrong. diff --git a/pelican/plugins/pandoc_reader/content/valid_content.md b/pelican/plugins/pandoc_reader/content/valid_content.md index da14010..7cfa55b 100644 --- a/pelican/plugins/pandoc_reader/content/valid_content.md +++ b/pelican/plugins/pandoc_reader/content/valid_content.md @@ -1,6 +1,6 @@ --- -title: Valid Content -author: My Author -date: 2020-10-16 +title: "Valid Content" +author: "My Author" +date: "2020-10-16" --- This is some valid content that should pass. If it does not pass we will know something is wrong. diff --git a/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md b/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md index 0b16a71..95e701f 100644 --- a/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md +++ b/pelican/plugins/pandoc_reader/content/valid_content_with_raw_paths.md @@ -1,7 +1,7 @@ --- -title: Valid Content with Fictitious Raw Paths -author: My Author -date: 2020-10-16 +title: "Valid Content with Fictitious Raw Paths" +author: "My Author" +date: "2020-10-16" --- This is some valid content that should pass. If it does not pass we will know something is wrong. diff --git a/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md b/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md index 8c13a0c..1e46a70 100644 --- a/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md +++ b/pelican/plugins/pandoc_reader/content/wrong_metadata_end.md @@ -1,5 +1,5 @@ --- -title: No Metadata End -author: My Author -date: 2020-10-16 +title: "No Metadata End" +author: "My Author" +date: "2020-10-16" ~~~ diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 72f728a..2e673e3 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -212,7 +212,7 @@ def _process_metadata(self, text): for line in lines[:yaml_end]: metalist = line.split(":", 1) if len(metalist) == 2: - key, value = metalist[0].lower(), metalist[1].strip() + key, value = metalist[0].lower(), metalist[1].strip().strip('"') metadata[key] = self.process_metadata(key, value) return metadata, yaml_end diff --git a/requirements.txt b/requirements.txt index 7a997b5..397d447 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,12 @@ +blinker==1.4 +docutils==0.16 +feedgenerator==1.9.1 +Jinja2==2.11.2 +MarkupSafe==1.1.1 +pelican==4.5.0 +Pygments==2.7.2 +python-dateutil==2.8.1 +pytz==2020.1 PyYAML==5.3.1 +six==1.15.0 +Unidecode==1.1.1 From a39d597ee228da0d236c4c222c58afe1aa70e553 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 30 Oct 2020 04:41:18 -0700 Subject: [PATCH 046/102] Updated README with examples showing quotation marks for YAML files --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 447e9ed..7ab58fe 100644 --- a/README.md +++ b/README.md @@ -98,13 +98,13 @@ or ```yaml ... -title: -author: -date: +title: "" +author: "" +date: "" ... ``` -The plugin automatically insert title metadata as the main heading of your post or page. Therefore, there is no need to specify the first heading in your content. +The plugin automatically inserts title metadata field as the main heading of your post or page. Therefore, there is no need to specify the first heading in your content. Failing to provide a title metadata field will result in an error. From 1514474be98e68f04d30fc9be89a3ebae2f33892 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 30 Oct 2020 04:41:43 -0700 Subject: [PATCH 047/102] Updated README with examples showing quotation marks for YAML files --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7ab58fe..9fe909c 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,9 @@ The plugin expects all markdown files to start with a YAML block as shown below. ```yaml --- -title: -author: -data: +title: "" +author: "" +data: "" --- ``` From 32c9bb5a013efb03b8b0ad7a3a10aad11da5d467 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 1 Nov 2020 06:11:23 -0800 Subject: [PATCH 048/102] Removed code to automatically insert title specified in metadata --- README.md | 6 +---- .../plugins/pandoc_reader/pandoc_reader.py | 26 ++---------------- .../pandoc_reader/test_pandoc_reader.py | 27 ++----------------- 3 files changed, 5 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 9fe909c..8b5e1bf 100644 --- a/README.md +++ b/README.md @@ -104,11 +104,7 @@ date: "" ... ``` -The plugin automatically inserts title metadata field as the main heading of your post or page. Therefore, there is no need to specify the first heading in your content. - -Failing to provide a title metadata field will result in an error. - -**Note: Specifying the file metadata in the format above is a requirement of Pandoc. Pelican's recommended format is different and may require you to rewrite the metadata in your files, if you stop using this plugin.** +**Note: Pelican's recommended format for metadata is different to what is specified here, and may require you to rewrite the metadata in your files, if you stop using this plugin.** More information about Pelican's predefined metadata is available [here](https://docs.getpelican.com/en/stable/content.html#file-metadata). diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 2e673e3..7256c56 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -35,14 +35,8 @@ def read(self, source_path): with pelican_open(source_path) as file_content: content = file_content - # Evaluate the list immediately to stop it from becoming a generator - content_line_by_line = list(content.splitlines()) - # Parse YAML metadata - metadata, yaml_end = self._process_metadata(content_line_by_line) - - # Insert title metadata argument in content after metadata block - content = self.insert_title(content_line_by_line, metadata, yaml_end) + metadata = self._process_metadata(list(content.splitlines())) # Get arguments and extensions if not self.settings.get("PANDOC_DEFAULT_FILES"): @@ -168,22 +162,6 @@ def check_defaults(filepath): ): raise ValueError("Output format type must be html or html5.") - @staticmethod - def insert_title(content, metadata, yaml_end): - # Construct page title retrieved from metadata - try: - title = "# {}".format(str(metadata["title"])) - except KeyError: - raise KeyError("No title field found in metadata.") - - # Insert title to the start of the text - content.insert(yaml_end + 2, title) - - # Reconstruct the content back into string delimited by new lines - content = "\n".join(list(content)) - - return content - def _process_metadata(self, text): """Process YAML metadata and export.""" metadata = {} @@ -214,7 +192,7 @@ def _process_metadata(self, text): if len(metalist) == 2: key, value = metalist[0].lower(), metalist[1].strip().strip('"') metadata[key] = self.process_metadata(key, value) - return metadata, yaml_end + return metadata def add_reader(readers): diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index 80b6798..d015401 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -104,22 +104,6 @@ def test_invalid_metadata_block_end(self): message = str(context_manager.exception) self.assertEqual("Could not find end of metadata block.", message) - def test_no_title_field_in_metadata(self): - """Check if an exception is raised if no title is found in metadata.""" - settings = get_settings( - PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS - ) - - pandoc_reader = PandocReader(settings) - source_path = os.path.join(CONTENT_PATH, "no_title_in_metadata.md") - - # Metadata blocks should end with '___' or '...' if not it should fail - with self.assertRaises(KeyError) as context_manager: - pandoc_reader.read(source_path) - - message = str(context_manager.exception) - self.assertEqual("'No title field found in metadata.'", message) - def test_invalid_standalone_argument(self): """Check that specifying --standalone raises an exception.""" pandoc_arguments = ["--standalone"] @@ -164,7 +148,6 @@ def test_valid_file(self): self.assertEqual( ( - '

Valid Content

\n' "

This is some valid content that should pass." " If it does not pass we" " will know something is wrong.

\n" @@ -188,7 +171,6 @@ def test_mathjax_content(self): self.assertEqual( ( - '

MathJax Content

\n' '

\\[\ne^{i\\theta} = ' "\\cos\\theta + i \\sin\\theta.\n\\]

\n" ), @@ -214,8 +196,6 @@ def test_encoded_to_raw_conversion(self): self.assertEqual( ( - '

' - "Valid Content with Fictitious Raw Paths

\n" "

This is some valid content that should pass." " If it does not pass we will know something is wrong.

\n" "

Our fictitious internal files are available" @@ -422,10 +402,8 @@ def test_valid_file_with_valid_defaults(self): self.assertEqual( ( - '

Valid Content

\n

This' - " is some valid content that should pass." - " If it does not pass we" - " will know something is wrong.

\n" + "

This is some valid content that should pass." + " If it does not pass we will know something is wrong.

\n" ), output, ) @@ -447,7 +425,6 @@ def test_mathjax_with_valid_defaults(self): self.assertEqual( ( - '

MathJax Content

\n' '

\\[\ne^{i\\theta} = ' "\\cos\\theta + i \\sin\\theta.\n\\]

\n" ), From 8135ca6bbddaf39220c2e272de6b936acca50ba9 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sun, 1 Nov 2020 06:31:35 -0800 Subject: [PATCH 049/102] Updated README by removing information about title insertion and added example defaults file. --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8b5e1bf..e3dfef3 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,12 @@ In the `PANDOC_ARGS` parameter you may specify any argument supported by Pandoc ```python PANDOC_ARGS = [ - '--mathjax', - '--toc' + '--mathjax' + '--citeproc' ] ``` -**Note: Specifying the arguments `--standalone` or `--self-contained` are not supported and will throw an error.** +**Note: Specifying the arguments `--standalone` or `--self-contained` are not supported and will throw an error. Arguments that only are only relevant in standalone mode such as `--highlight-style` will not take effect and will not output an error.** Then in the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of the supported [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). @@ -72,15 +72,23 @@ These paths should be set in your `pelicanconf.py` file by using the setting `PA ```python PANDOC_DEFAULT_FILES = [ - '' + '', + '' ] ``` -Using a default file has the added benefit of allowing you to use other markdown flavors supported by Pandoc such as [CommonMark](https://commonmark.org/) and [GitHub-Flavored Markdown](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github). +Using default files has the added benefit of allowing you to use other markdown flavors supported by Pandoc such as [CommonMark](https://commonmark.org/) and [GitHub-Flavored Markdown](https://docs.github.com/en/free-pro-team@latest/github/writing-on-github). -The format of this file is described [here](https://pandoc.org/MANUAL.html#default-files). +Here is a simple example of a content that should be available in a Pandoc defaults file: -**Note: If `--standalone` or `--self-contained` are set to `true` you will get an error message.** +```yaml +reader: markdown +writer: html5 +``` + +Please see [Pandoc Default files](https://pandoc.org/MANUAL.html#default-files) for a more complete example of the options available for this file. + +**Note: If `standalone` or `self-contained` are set to `true` you will get an error message. Specifying fields that are only relevant in standalone mode such as `highlight-style` will not take effect and will not output an error.** ### Specifying File Metadata From a4aa126da47d9b367d5b7f3d625b3f6b450e35a7 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 6 Nov 2020 20:32:07 -0800 Subject: [PATCH 050/102] Added capability to generate a table of contents --- .../content/valid_content_with_toc.md | 22 +++ .../defaults/valid_defaults_with_toc.yaml | 12 ++ .../plugins/pandoc_reader/pandoc_reader.py | 77 +++++++-- .../pandoc_reader/templates/toc-template.html | 8 + .../pandoc_reader/test_pandoc_reader.py | 155 ++++++++++++++++++ 5 files changed, 261 insertions(+), 13 deletions(-) create mode 100644 pelican/plugins/pandoc_reader/content/valid_content_with_toc.md create mode 100644 pelican/plugins/pandoc_reader/defaults/valid_defaults_with_toc.yaml create mode 100644 pelican/plugins/pandoc_reader/templates/toc-template.html diff --git a/pelican/plugins/pandoc_reader/content/valid_content_with_toc.md b/pelican/plugins/pandoc_reader/content/valid_content_with_toc.md new file mode 100644 index 0000000..03aaa04 --- /dev/null +++ b/pelican/plugins/pandoc_reader/content/valid_content_with_toc.md @@ -0,0 +1,22 @@ +--- +title: "Valid Content with Table of Contents" +author: "My Author" +date: "2020-10-16" +--- +This is some valid content that should pass. If it does not pass we will know something is wrong. + +## First Heading + +This should be the first heading in my table of contents. + +## Second Heading + +This should be the second heading in my table of contents. + +### First Subheading + +This is a subsection that should be shown as such in the table of contents. + +### Second Subheading + +This is another subsection that should be shown as such in the table of contents. diff --git a/pelican/plugins/pandoc_reader/defaults/valid_defaults_with_toc.yaml b/pelican/plugins/pandoc_reader/defaults/valid_defaults_with_toc.yaml new file mode 100644 index 0000000..8db4ba4 --- /dev/null +++ b/pelican/plugins/pandoc_reader/defaults/valid_defaults_with_toc.yaml @@ -0,0 +1,12 @@ +# valid_defaults_with_toc.yml +# +# A test default file that is valid and sets table_of_contents to true +# +reader: markdown+smart+citations+implicit_figures +writer: html5 + +table-of-contents: true + +html-math-method: + method: mathjax + url: "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index 7256c56..a6cd65a 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -1,4 +1,5 @@ """Reader that processes Pandoc Markdown and returns HTML 5.""" +import os import shutil import subprocess @@ -8,6 +9,10 @@ from pelican import signals +DIR_PATH = os.path.dirname(__file__) +TEMPLATES_PATH = os.path.abspath(os.path.join(DIR_PATH, "templates")) +TOC_TEMPLATE = "toc-template.html" + ENCODED_LINKS_TO_RAW_LINKS_MAP = { "%7Bstatic%7D": "{static}", "%7Battach%7D": "{attach}", @@ -35,8 +40,9 @@ def read(self, source_path): with pelican_open(source_path) as file_content: content = file_content - # Parse YAML metadata - metadata = self._process_metadata(list(content.splitlines())) + generate_table_of_contents = False + metadata = "" + pandoc_cmd = [] # Get arguments and extensions if not self.settings.get("PANDOC_DEFAULT_FILES"): @@ -55,16 +61,63 @@ def read(self, source_path): ] self.check_arguments(arguments) + + # Check if we should generate a table of contents + if "--toc" in arguments: + generate_table_of_contents = True + elif "--table-of-contents" in arguments: + generate_table_of_contents = True + pandoc_cmd.extend(arguments) else: default_files_cmd = [] for filepath in self.settings.get("PANDOC_DEFAULT_FILES"): - self.check_defaults(filepath) - default_files_cmd.append("--defaults={0}".format(filepath)) + defaults = self.check_defaults(filepath) + # Check if we need to generate a table of contents + if not generate_table_of_contents: + if defaults.get("table-of-contents", ""): + generate_table_of_contents = True + + default_files_cmd.append("--defaults={0}".format(filepath)) # Construct Pandoc command pandoc_cmd = ["pandoc"] + default_files_cmd + # Generate HTML + output = self.run_pandoc(pandoc_cmd, content) + + # Replace all occurrences of %7Bstatic%7D to {static}, + # %7Battach%7D to {attach} and %7Bfilename%7D to {filename} + # so that static links are resolvable by pelican + for encoded_str, raw_str in ENCODED_LINKS_TO_RAW_LINKS_MAP.items(): + output = output.replace(encoded_str, raw_str) + + # Generate a table of contents if the variable below was set to true + if generate_table_of_contents: + # The table of contents is generated using a template which + # takes effect only in standalone mode + gen_toc_extra_args = [ + "--standalone", + "--template", + os.path.join(TEMPLATES_PATH, TOC_TEMPLATE) + ] + + # Generate the table of contents + pandoc_gen_toc_cmd = pandoc_cmd + gen_toc_extra_args + table_of_contents = self.run_pandoc(pandoc_gen_toc_cmd, content) + + # Parse YAML metadata and add the table of contents to the metadata + metadata = self._process_metadata( + list(content.splitlines()), table_of_contents + ) + else: + # Parse YAML metadata + metadata = self._process_metadata(list(content.splitlines())) + + return output, metadata + + @staticmethod + def run_pandoc(pandoc_cmd, content): # Execute Pandoc command proc = subprocess.Popen( pandoc_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE @@ -75,14 +128,7 @@ def read(self, source_path): status = proc.wait() if status: raise subprocess.CalledProcessError(status, pandoc_cmd) - - # Replace all occurrences of %7Bstatic%7D to {static}, - # %7Battach%7D to {attach} and %7Bfilename%7D to {filename} - # so that static links are resolvable by pelican - for encoded_str, raw_str in ENCODED_LINKS_TO_RAW_LINKS_MAP.items(): - output = output.replace(encoded_str, raw_str) - - return output, metadata + return output @staticmethod def check_arguments(arguments): @@ -161,8 +207,9 @@ def check_defaults(filepath): and to_output not in VALID_OUTPUT_FORMATS ): raise ValueError("Output format type must be html or html5.") + return defaults - def _process_metadata(self, text): + def _process_metadata(self, text, table_of_contents=""): """Process YAML metadata and export.""" metadata = {} @@ -192,6 +239,10 @@ def _process_metadata(self, text): if len(metalist) == 2: key, value = metalist[0].lower(), metalist[1].strip().strip('"') metadata[key] = self.process_metadata(key, value) + + # Add the table of contents to the metadata + if table_of_contents: + metadata["toc"] = table_of_contents return metadata diff --git a/pelican/plugins/pandoc_reader/templates/toc-template.html b/pelican/plugins/pandoc_reader/templates/toc-template.html new file mode 100644 index 0000000..b3b0108 --- /dev/null +++ b/pelican/plugins/pandoc_reader/templates/toc-template.html @@ -0,0 +1,8 @@ +$if(toc)$ + +$endif$ diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index d015401..acbad97 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -214,6 +214,110 @@ def test_encoded_to_raw_conversion(self): self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + def test_valid_content_with_toc(self): + """Check if output returned is valid and table of contents is valid.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, + PANDOC_ARGS=PANDOC_ARGS + ["--toc"] + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content_with_toc.md") + output, metadata = pandoc_reader.read(source_path) + + # Setting this so that assert is able to execute the difference + self.maxDiff = None + + self.assertEqual( + ( + '

This is some valid content that should pass.' + ' If it does not pass we will know something is wrong.

\n' + '

First Heading

\n' + '

This should be the first heading in my' + ' table of contents.

\n' + '

Second Heading

\n' + '

This should be the second heading in my' + ' table of contents.

\n' + '

First Subheading

\n' + '

This is a subsection that should be shown as such' + ' in the table of contents.

\n' + '

Second Subheading

\n' + '

This is another subsection that should be shown as' + ' such in the table of contents.

\n' + ), + output + ) + self.assertEqual( + "Valid Content with Table of Contents", str(metadata["title"]) + ) + self.assertEqual("My Author", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + self.assertEqual( + '\n', + str(metadata["toc"]) + ) + + def test_valid_content_with_toc_2(self): + """Check if output returned is valid and table of contents is valid.""" + settings = get_settings( + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, + PANDOC_ARGS=PANDOC_ARGS + ["--table-of-contents"] + ) + + pandoc_reader = PandocReader(settings) + source_path = os.path.join(CONTENT_PATH, "valid_content_with_toc.md") + output, metadata = pandoc_reader.read(source_path) + + # Setting this so that assert is able to execute the difference + self.maxDiff = None + + self.assertEqual( + ( + '

This is some valid content that should pass.' + ' If it does not pass we will know something is wrong.

\n' + '

First Heading

\n' + '

This should be the first heading in my' + ' table of contents.

\n' + '

Second Heading

\n' + '

This should be the second heading in my' + ' table of contents.

\n' + '

First Subheading

\n' + '

This is a subsection that should be shown as such' + ' in the table of contents.

\n' + '

Second Subheading

\n' + '

This is another subsection that should be shown as' + ' such in the table of contents.

\n' + ), + output + ) + self.assertEqual( + "Valid Content with Table of Contents", str(metadata["title"]) + ) + self.assertEqual("My Author", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + self.assertEqual( + '\n', + str(metadata["toc"]) + ) + # Tests using default files def test_invalid_standalone(self): """Check if exception is raised if standalone is true.""" @@ -435,6 +539,57 @@ def test_mathjax_with_valid_defaults(self): self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + def test_toc_with_valid_defaults(self): + """Check if output and table of contents are valid with defaults.""" + pandoc_default_files = [ + os.path.join(DEFAULTS_PATH, "valid_defaults_with_toc.yaml") + ] + + settings = get_settings(PANDOC_DEFAULT_FILES=pandoc_default_files) + pandoc_reader = PandocReader(settings) + + source_path = os.path.join(CONTENT_PATH, "valid_content_with_toc.md") + output, metadata = pandoc_reader.read(source_path) + + self.assertEqual( + ( + '

This is some valid content that should pass.' + ' If it does not pass we will know something is wrong.

\n' + '

First Heading

\n' + '

This should be the first heading in my' + ' table of contents.

\n' + '

Second Heading

\n' + '

This should be the second heading in my' + ' table of contents.

\n' + '

First Subheading

\n' + '

This is a subsection that should be shown as such' + ' in the table of contents.

\n' + '

Second Subheading

\n' + '

This is another subsection that should be shown as' + ' such in the table of contents.

\n' + ), + output + ) + self.assertEqual( + "Valid Content with Table of Contents", + str(metadata["title"]) + ) + self.assertEqual("My Author", str(metadata["author"])) + self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) + self.assertEqual( + '\n', + str(metadata["toc"]) + ) + if __name__ == "__main__": unittest.main() From 5cc3d1d6691bf023b564fc332f56de078ec3cde0 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 6 Nov 2020 20:36:56 -0800 Subject: [PATCH 051/102] Reformatted files using black --- .../plugins/pandoc_reader/pandoc_reader.py | 2 +- .../pandoc_reader/test_pandoc_reader.py | 120 ++++++++---------- 2 files changed, 57 insertions(+), 65 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index a6cd65a..d192b43 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -99,7 +99,7 @@ def read(self, source_path): gen_toc_extra_args = [ "--standalone", "--template", - os.path.join(TEMPLATES_PATH, TOC_TEMPLATE) + os.path.join(TEMPLATES_PATH, TOC_TEMPLATE), ] # Generate the table of contents diff --git a/pelican/plugins/pandoc_reader/test_pandoc_reader.py b/pelican/plugins/pandoc_reader/test_pandoc_reader.py index acbad97..d4c00cf 100644 --- a/pelican/plugins/pandoc_reader/test_pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/test_pandoc_reader.py @@ -217,8 +217,7 @@ def test_encoded_to_raw_conversion(self): def test_valid_content_with_toc(self): """Check if output returned is valid and table of contents is valid.""" settings = get_settings( - PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, - PANDOC_ARGS=PANDOC_ARGS + ["--toc"] + PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, PANDOC_ARGS=PANDOC_ARGS + ["--toc"] ) pandoc_reader = PandocReader(settings) @@ -230,47 +229,45 @@ def test_valid_content_with_toc(self): self.assertEqual( ( - '

This is some valid content that should pass.' - ' If it does not pass we will know something is wrong.

\n' + "

This is some valid content that should pass." + " If it does not pass we will know something is wrong.

\n" '

First Heading

\n' - '

This should be the first heading in my' - ' table of contents.

\n' + "

This should be the first heading in my" + " table of contents.

\n" '

Second Heading

\n' - '

This should be the second heading in my' - ' table of contents.

\n' + "

This should be the second heading in my" + " table of contents.

\n" '

First Subheading

\n' - '

This is a subsection that should be shown as such' - ' in the table of contents.

\n' + "

This is a subsection that should be shown as such" + " in the table of contents.

\n" '

Second Subheading

\n' - '

This is another subsection that should be shown as' - ' such in the table of contents.

\n' + "

This is another subsection that should be shown as" + " such in the table of contents.

\n" ), - output - ) - self.assertEqual( - "Valid Content with Table of Contents", str(metadata["title"]) + output, ) + self.assertEqual("Valid Content with Table of Contents", str(metadata["title"])) self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) self.assertEqual( '\n', - str(metadata["toc"]) + "\n" + "\n" + "\n", + str(metadata["toc"]), ) def test_valid_content_with_toc_2(self): """Check if output returned is valid and table of contents is valid.""" settings = get_settings( PANDOC_EXTENSIONS=PANDOC_EXTENSIONS, - PANDOC_ARGS=PANDOC_ARGS + ["--table-of-contents"] + PANDOC_ARGS=PANDOC_ARGS + ["--table-of-contents"], ) pandoc_reader = PandocReader(settings) @@ -282,40 +279,38 @@ def test_valid_content_with_toc_2(self): self.assertEqual( ( - '

This is some valid content that should pass.' - ' If it does not pass we will know something is wrong.

\n' + "

This is some valid content that should pass." + " If it does not pass we will know something is wrong.

\n" '

First Heading

\n' - '

This should be the first heading in my' - ' table of contents.

\n' + "

This should be the first heading in my" + " table of contents.

\n" '

Second Heading

\n' - '

This should be the second heading in my' - ' table of contents.

\n' + "

This should be the second heading in my" + " table of contents.

\n" '

First Subheading

\n' - '

This is a subsection that should be shown as such' - ' in the table of contents.

\n' + "

This is a subsection that should be shown as such" + " in the table of contents.

\n" '

Second Subheading

\n' - '

This is another subsection that should be shown as' - ' such in the table of contents.

\n' + "

This is another subsection that should be shown as" + " such in the table of contents.

\n" ), - output - ) - self.assertEqual( - "Valid Content with Table of Contents", str(metadata["title"]) + output, ) + self.assertEqual("Valid Content with Table of Contents", str(metadata["title"])) self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) self.assertEqual( '\n', - str(metadata["toc"]) + "\n" + "\n" + "\n", + str(metadata["toc"]), ) # Tests using default files @@ -553,41 +548,38 @@ def test_toc_with_valid_defaults(self): self.assertEqual( ( - '

This is some valid content that should pass.' - ' If it does not pass we will know something is wrong.

\n' + "

This is some valid content that should pass." + " If it does not pass we will know something is wrong.

\n" '

First Heading

\n' - '

This should be the first heading in my' - ' table of contents.

\n' + "

This should be the first heading in my" + " table of contents.

\n" '

Second Heading

\n' - '

This should be the second heading in my' - ' table of contents.

\n' + "

This should be the second heading in my" + " table of contents.

\n" '

First Subheading

\n' - '

This is a subsection that should be shown as such' - ' in the table of contents.

\n' + "

This is a subsection that should be shown as such" + " in the table of contents.

\n" '

Second Subheading

\n' - '

This is another subsection that should be shown as' - ' such in the table of contents.

\n' + "

This is another subsection that should be shown as" + " such in the table of contents.

\n" ), - output - ) - self.assertEqual( - "Valid Content with Table of Contents", - str(metadata["title"]) + output, ) + self.assertEqual("Valid Content with Table of Contents", str(metadata["title"])) self.assertEqual("My Author", str(metadata["author"])) self.assertEqual("2020-10-16 00:00:00", str(metadata["date"])) self.assertEqual( '\n', - str(metadata["toc"]) + "\n" + "\n" + "\n", + str(metadata["toc"]), ) From 3930c208dee6ed881b1907768e2536b48a2f5bcf Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 6 Nov 2020 20:55:18 -0800 Subject: [PATCH 052/102] Updated README with information about generating a table of contents for articles --- README.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e3dfef3..1dfa4dc 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,25 @@ PANDOC_ARGS = [ ] ``` -**Note: Specifying the arguments `--standalone` or `--self-contained` are not supported and will throw an error. Arguments that only are only relevant in standalone mode such as `--highlight-style` will not take effect and will not output an error.** +Generation of a table of contents is supported by this plugin by specifying the `--toc` or `--table-of-contents` argument as shown below: + +```python +PANDOC_ARGS = [ + '--toc' +] +``` + +or + +```python +PANDOC_ARGS = [ + '--table-of-contents' +] +``` + +The table of contents will be available as an HTML snippet in the metadata of an article and can be referenced as `{{ article.toc }}` in templates. + +**Note: Specifying the arguments `--standalone` or `--self-contained` are not supported and will throw an error. Arguments that are relevant only in standalone mode such as `--highlight-style` will not take effect except for `--toc` of `table-of-contents`.** Then in the `PANDOC_EXTENSIONS` parameter you may enable/disable any number of the supported [Pandoc extensions](https://pandoc.org/MANUAL.html#extensions). @@ -86,9 +104,17 @@ reader: markdown writer: html5 ``` +Generation of a table of contents is supported by this plugin by setting the `table-of-contents` to `true` as shown below: + +```yaml +table-of-contents: true +``` + +The table of contents will be available as an HTML snippet in the metadata of an article and can be referenced as `{{ article.toc }}` in templates. + Please see [Pandoc Default files](https://pandoc.org/MANUAL.html#default-files) for a more complete example of the options available for this file. -**Note: If `standalone` or `self-contained` are set to `true` you will get an error message. Specifying fields that are only relevant in standalone mode such as `highlight-style` will not take effect and will not output an error.** +**Note: If `standalone` or `self-contained` are set to `true` you will get an error message. Specifying fields that are only relevant in standalone mode such as `highlight-style` will not take effect except for `table-of-contents`.** ### Specifying File Metadata From 8964041ab5840f5af04fe6c308b05a99829d386f Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Fri, 6 Nov 2020 23:49:58 -0800 Subject: [PATCH 053/102] Added missing call to process metadata for table of contents --- pelican/plugins/pandoc_reader/pandoc_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pelican/plugins/pandoc_reader/pandoc_reader.py b/pelican/plugins/pandoc_reader/pandoc_reader.py index d192b43..78a3cf8 100644 --- a/pelican/plugins/pandoc_reader/pandoc_reader.py +++ b/pelican/plugins/pandoc_reader/pandoc_reader.py @@ -240,9 +240,9 @@ def _process_metadata(self, text, table_of_contents=""): key, value = metalist[0].lower(), metalist[1].strip().strip('"') metadata[key] = self.process_metadata(key, value) - # Add the table of contents to the metadata + # Add the table of contents to the content's metadata if table_of_contents: - metadata["toc"] = table_of_contents + metadata["toc"] = self.process_metadata("toc", table_of_contents) return metadata From 6fb3efbbd4acb6a7aee34f78405488c2c3c77d62 Mon Sep 17 00:00:00 2001 From: Nandakumar Chandrasekhar Date: Sat, 7 Nov 2020 06:17:38 -0800 Subject: [PATCH 054/102] Updated tanle of comtents template to have a class instead of an id --- pelican/plugins/pandoc_reader/templates/toc-template.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pelican/plugins/pandoc_reader/templates/toc-template.html b/pelican/plugins/pandoc_reader/templates/toc-template.html index b3b0108..ceb3efe 100644 --- a/pelican/plugins/pandoc_reader/templates/toc-template.html +++ b/pelican/plugins/pandoc_reader/templates/toc-template.html @@ -1,5 +1,5 @@ $if(toc)$ -