Skip to content

Commit 26b2152

Browse files
wxtimwxtim
committed
Basic Metadata Writing extension
1 parent 3eacddf commit 26b2152

File tree

5 files changed

+374
-0
lines changed

5 files changed

+374
-0
lines changed

‎conf.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
'cylc.sphinx_ext.diff_selection',
3535
'cylc.sphinx_ext.grid_table',
3636
'cylc.sphinx_ext.hieroglyph_addons',
37+
'cylc.sphinx_ext.metadata',
3738
'cylc.sphinx_ext.minicylc',
3839
'cylc.sphinx_ext.practical',
3940
'cylc.sphinx_ext.rtd_theme_addons',

‎cylc/sphinx_ext/metadata/__init__.py‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
# -----------------------------------------------------------------------------
2+
# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
3+
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
# -----------------------------------------------------------------------------
18+
from docutils import nodes
19+
from docutils.statemachine import ViewList
20+
import json
21+
import os
22+
from pathlib import Path
23+
from sphinx.util.docutils import SphinxDirective
24+
from sphinx.util.nodes import nested_parse_with_titles
25+
from subprocess import run
26+
27+
28+
rawdoc = """An extension for grabbing Cylc Config Metadata.
29+
30+
.. rst-example::
31+
32+
.. cylc_metadata::
33+
:global: {workflow_path}
34+
35+
.. cylc_metadata::
36+
:source: {workflow_path}
37+
38+
39+
Directives
40+
----------
41+
42+
.. rst:directive:: cylc_metadata
43+
44+
Get a Cylc Configuration and render it.
45+
46+
.. rst:directive:option:: source
47+
:type: string
48+
49+
If set, renders the metadata of a workflow, otherwise the global
50+
config.
51+
52+
.. rst:directive:option:: global
53+
:type: string
54+
55+
Set CYLC_SITE_CONF_PATH to this value.
56+
57+
"""
58+
59+
workflow_path = Path(__file__).parent.parent.parent.parent / 'etc'
60+
__doc__ = rawdoc.format(workflow_path=workflow_path)
61+
__all__ = ['CylcMetadata', 'setup']
62+
__version__ = '1.0.0'
63+
64+
65+
def setup(app):
66+
"""Sphinx plugin setup function."""
67+
app.add_directive('cylc_metadata', CylcMetadata)
68+
69+
return {'version': __version__, 'parallel_read_safe': True}
70+
71+
72+
class Doc(ViewList):
73+
"""Convenience wrapper for ViewList to allow us to use it to
74+
collect lines of RST, with options to underline."""
75+
def append(self, text, underline=None):
76+
super().append(text, '', 1)
77+
if underline:
78+
super().append(underline * len(text), '', 1)
79+
super().append('', '', 1)
80+
81+
82+
class DependencyError(Exception):
83+
...
84+
85+
86+
class CylcMetadata(SphinxDirective):
87+
"""Represent a Cylc Config.
88+
"""
89+
optional_arguments = 3
90+
91+
def run(self):
92+
# Parse input options:
93+
for key, value in zip(
94+
[i.strip(':') for i in self.arguments[::2]],
95+
list(self.arguments[1::2])
96+
):
97+
self.options.update({key: value})
98+
99+
# Get global or workflow metadara
100+
if 'source' in self.options:
101+
config = self.load_workflow_cfg(self.options['source'])
102+
metadata = self.get_workflow_metadata(
103+
config, self.options['source'])
104+
rst = self.convert_workflow_to_rst(metadata)
105+
else:
106+
config = self.load_global_cfg(self.options['global'])
107+
metadata = self.get_global_metadata(config)
108+
rst = self.convert_global_to_rst(metadata)
109+
110+
container = nodes.Element()
111+
nested_parse_with_titles(self.state, rst, container)
112+
return container.children
113+
114+
@staticmethod
115+
def load_global_cfg(conf_path=None):
116+
"""Get Global Configuration metadata:
117+
118+
Args:
119+
Path: Global conf path.
120+
121+
"""
122+
# Load Global Config:
123+
if conf_path:
124+
env = os.environ
125+
sub = run(
126+
['cylc', 'config', '--json'],
127+
capture_output=True,
128+
env=env.update({'CYLC_SITE_CONF_PATH': conf_path})
129+
)
130+
else:
131+
sub = run(['cylc', 'config', '--json'], capture_output=True)
132+
133+
CylcMetadata.check_subproc_output(sub)
134+
135+
return json.loads(sub.stdout)
136+
137+
@staticmethod
138+
def get_global_metadata(config):
139+
"""
140+
Additional Processing:
141+
* Get lists of hosts/platforms and job runner from the config.
142+
* If no title is provided, use the platform/group regex as the title.
143+
* If title != regex then insert text saying which regex
144+
needs matching to select this platform.
145+
146+
Returns:
147+
A dictionary in the form:
148+
'platforms': {'platform regex': {..metadata..}},
149+
'platform groups': {'platform regex': {..metadata..}}
150+
"""
151+
metadata = {}
152+
for section, select_from in zip(
153+
['platforms', 'platform groups'],
154+
['hosts', 'platforms']
155+
):
156+
metadata[section] = config.get(section)
157+
if not metadata[section]:
158+
continue
159+
for key in config.get(section).keys():
160+
# Grab a list of hosts or platforms that this
161+
# platform or group will select from:
162+
select_from = (
163+
config.get(section).get(key).get('hosts')
164+
or config.get(section).get(key).get('platforms'))
165+
select_from = select_from or [key]
166+
metadata[section][key]['select_from'] = select_from
167+
168+
# Grab the job runner this platform uses:
169+
if section == 'platforms':
170+
metadata[section][key]['job_runner'] = config.get(
171+
section).get(key).get('job runner', 'background')
172+
return metadata
173+
174+
@staticmethod
175+
def convert_global_to_rst(meta):
176+
"""Convert the global metadata into rst format."""
177+
rst = Doc()
178+
rst.append('Global Config', '#')
179+
rst.append('.. note::')
180+
rst.append(
181+
' platforms and platform groups are listed in the order in which'
182+
' Cylc will check for matches to the'
183+
' ``[runtime][NAMESPACE]platform`` setting.')
184+
for settings, selects in zip(
185+
['platform groups', 'platforms'], ['platforms', 'hosts']
186+
):
187+
if meta.get(settings, {}):
188+
rst.append(settings, '=')
189+
for regex, info in reversed(meta[settings].items()):
190+
title = info.get('title', '')
191+
if not title:
192+
title = regex
193+
rst.append(title, '^')
194+
if title != regex:
195+
rst.append(
196+
f'match ``{regex}`` to select these {settings}.')
197+
rst.append(info.get('description', 'No description'))
198+
199+
if info.get('job_runner'):
200+
rst.append(
201+
'This platform uses job runner'
202+
f' ``{info.get("job_runner")}``')
203+
204+
rst.append(f'Selects {selects} from:')
205+
for selectable in info['select_from']:
206+
rst.append(f'* ``{selectable}``')
207+
return rst
208+
209+
@staticmethod
210+
def load_workflow_cfg(conf_path):
211+
"""Get Workflow Configuration metadata:
212+
213+
Args:
214+
conf_path: workflow conf path.
215+
"""
216+
# Load Global Config:
217+
sub = run(
218+
['cylc', 'config', '--json', conf_path],
219+
capture_output=True,
220+
)
221+
CylcMetadata.check_subproc_output(sub)
222+
return json.loads(sub.stdout)
223+
224+
@staticmethod
225+
def get_workflow_metadata(config, conf_path):
226+
"""Get workflow metadata.
227+
228+
Additional processing:
229+
* If no title field is provided use either the workflow folder
230+
or the task/family name.
231+
* Don't return the root family if there is no metadata.
232+
233+
Returns:
234+
'workflow': {.. top level metadata ..},
235+
'runtime': {'namespace': '.. task or family metadata ..'}
236+
237+
"""
238+
# Extract Data
239+
meta = {}
240+
241+
# Copy metadata to the two top level sections:
242+
meta['workflow'] = config.get('meta')
243+
meta['runtime'] = {
244+
k: v.get('meta', {})
245+
for k, v in config.get('runtime', {}).items()}
246+
247+
# Title is parent directory if otherwise unset:
248+
if not meta.get('workflow', {}).get('title', ''):
249+
meta['workflow']['title'] = Path(conf_path).name
250+
251+
# Title of namespace is parent if otherwise unset:
252+
poproot = False
253+
for namespace, info in meta['runtime'].items():
254+
# don't display root unless it's actually had some
255+
# metadata added, but save a flag rather than modifying
256+
# the iterable being looped over:
257+
if (
258+
namespace == 'root'
259+
and not any(meta['runtime'].get('root').values())
260+
):
261+
poproot = True
262+
263+
# If metadata doesn't have a title set title to the namespace name:
264+
if not info.get('title', ''):
265+
meta['runtime'][namespace]['title'] = namespace
266+
267+
if poproot:
268+
meta['runtime'].pop('root')
269+
270+
return meta
271+
272+
@staticmethod
273+
def convert_workflow_to_rst(meta):
274+
"""Convert workflow metadata to RST.
275+
276+
Returns
277+
"""
278+
rst = Doc()
279+
280+
# Handle the workflow config metadata:
281+
CylcMetadata.write_section(rst, meta.get('workflow', {}), '#')
282+
283+
# Handle the runtime config metadata:
284+
rst.append('Runtime', '=')
285+
for taskmeta in meta['runtime'].values():
286+
CylcMetadata.write_section(rst, taskmeta)
287+
288+
return rst
289+
290+
@staticmethod
291+
def write_section(rst, section, title_level='^'):
292+
# Title
293+
title = section.get('title', '')
294+
if not title:
295+
return
296+
rst.append(title, title_level)
297+
298+
# Url
299+
url = section.get('url', '')
300+
if url:
301+
rst.append(url)
302+
303+
# Description
304+
rst.append(section.get('description', ''))
305+
306+
@staticmethod
307+
def check_subproc_output(sub):
308+
"""Check subprocess outputs - catch failure.
309+
"""
310+
if sub.returncode:
311+
# Very specifically handle the case where the correct
312+
# version of Cylc isn't installed:
313+
if 'no such option: --json' in sub.stderr.decode():
314+
msg = (
315+
'Requires cylc config --json, not available'
316+
' for this version of Cylc')
317+
raise DependencyError(msg)
318+
# Handle any and all other errors in the subprocess:
319+
else:
320+
msg = 'Cylc config metadata failed with: \n'
321+
msg += '\n'.join(
322+
i.strip("\n") for i in sub.stderr.decode().split('\n'))
323+
raise Exception(msg)

‎etc/flow.cylc‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
[meta]
2+
title = 'Hello World'
3+
description = """
4+
This flow.cylc file is placed here to allow the
5+
testing of the metadata config extension.
6+
"""
7+
url = 'https://www.myproject.com'
8+
custom key yan = "Perhaps it's relevent?"
9+
custom key tan = "Perhaps it's also relevent?"
10+
custom key tethera = "Or perhaps not?"
11+
12+
13+
[scheduling]
14+
[[graph]]
15+
R1 = foo & bar
16+
17+
[runtime]
18+
[[foo]]
19+
[[[meta]]]
20+
title = 'task title'
21+
description = """
22+
All about my task
23+
I will document my working
24+
All about my task
25+
for a P12M + P1D
26+
"""
27+
url = 'https://www.myproject.com/tasks/foo'
28+
29+
[[bar]]
30+
[[[meta]]]
31+
description = """
32+
What should happen if I forget the title?
33+
Should I process RST?
34+
"""
35+
url = 'https://www.myproject.com/tasks/bar'
36+
see also = Bar task docs.

‎etc/flow/global.cylc‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[platforms]
2+
[[.*cross]]
3+
[[[meta]]]
4+
title = "Kings/Charing/Bounds Cross"
5+
description = """
6+
If platform name is a regex you might want
7+
an explicit title
8+
"""
9+
[[mornington_crescent]]
10+
[[[meta]]]
11+
description = """
12+
I win!
13+
"""

‎index.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ Available Extensions
99

1010
cylc.sphinx_ext.cylc_lang
1111
cylc.sphinx_ext.diff_selection
12+
cylc.sphinx_ext.metadata
1213
cylc.sphinx_ext.grid_table
1314
cylc.sphinx_ext.hieroglyph_addons
1415
cylc.sphinx_ext.literal_sub_include

0 commit comments

Comments
 (0)