diff --git a/README.md b/README.md index dd889b4..0dfb4ad 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,10 @@ - [crossplane minify](#crossplane-minify) - [Python Module](#python-module) - [crossplane.parse()](#crossplaneparse) + - [crossplane.parse_string()](#crossplaneparse_string) - [crossplane.build()](#crossplanebuild) - [crossplane.lex()](#crossplanelex) + - [crossplane.lex_string()](#crossplanelex_string) - [Other Languages](#other-languages) ## Install @@ -469,8 +471,8 @@ optional arguments: ## Python Module In addition to the command line tool, you can import `crossplane` as a -python module. There are two basic functions that the module will -provide you: `parse` and `lex`. +python module. There are four basic functions that the module will +provide you: `parse`, `parse_string`, `lex`, and `lex_string`. ### crossplane.parse() @@ -483,6 +485,23 @@ This will return the same payload as described in the [crossplane parse](#crossplane-parse) section, except it will be Python dicts and not one giant JSON string. +### crossplane.parse_string() + +```python +import crossplane +text = """ +events {} +http { + include conf.d/*.conf; +} +""" +payload = crossplane.parse_string(text, filename='/etc/nginx/nginx.conf') +``` + +Parses configuration provided as a string. If you pass `filename`, relative include +patterns are resolved against its directory, and error messages reference it. Options +mirror `crossplane.parse`. + ### crossplane.build() ```python @@ -515,6 +534,17 @@ will result in a long list similar to what you can see in the is used, except it will obviously be a Python list of tuples and not one giant JSON string. +### crossplane.lex_string() + +```python +import crossplane +text = "events { worker_connections 1024; }" +tokens = list(crossplane.lex_string(text, filename='')) +``` + +Lexes tokens from a configuration string. The optional `filename` is used for error +reporting when brace-balance errors are detected. + ## Other Languages - Go port by [@aluttik](https://github.com/aluttik): diff --git a/crossplane/__init__.py b/crossplane/__init__.py index 6a69217..b9aff51 100644 --- a/crossplane/__init__.py +++ b/crossplane/__init__.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -from .parser import parse -from .lexer import lex +from .parser import parse, parse_string +from .lexer import lex, lex_string from .builder import build from .formatter import format from .ext.lua import LuaBlockPlugin -__all__ = ['parse', 'lex', 'build', 'format'] +__all__ = ['parse', 'parse_string', 'lex', 'lex_string', 'build', 'format'] __title__ = 'crossplane' __summary__ = 'Reliable and fast NGINX configuration file parser.' diff --git a/crossplane/lexer.py b/crossplane/lexer.py index 2db9c6e..dd0abb0 100644 --- a/crossplane/lexer.py +++ b/crossplane/lexer.py @@ -155,3 +155,16 @@ def lex(filename): def register_external_lexer(directives, lexer): for directive in directives: EXTERNAL_LEXERS[directive] = lexer + + +def lex_string(text, filename=None): + """Generates tokens from an nginx config string. + + :param text: configuration text to lex + :param filename: optional filename to use in error reporting + """ + f = io.StringIO(text) + it = _lex_file_object(f) + it = _balance_braces(it, filename) + for token, line, quoted in it: + yield (token, line, quoted) diff --git a/crossplane/parser.py b/crossplane/parser.py index 09268c8..b6b3771 100644 --- a/crossplane/parser.py +++ b/crossplane/parser.py @@ -2,7 +2,7 @@ import glob import os -from .lexer import lex +from .lexer import lex, lex_string from .analyzer import analyze, enter_block_ctx from .errors import NgxParserDirectiveError @@ -22,35 +22,19 @@ def _prepare_if_args(stmt): args[:] = args[start:end] -def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False, +def _parse_with_initial_tokens(initial_tokens, initial_file, config_dir, + onerror=None, catch_errors=True, ignore=(), single=False, comments=False, strict=False, combine=False, check_ctx=True, check_args=True): - """ - Parses an nginx config file and returns a nested dict payload - - :param filename: string contianing the name of the config file to parse - :param onerror: function that determines what's saved in "callback" - :param catch_errors: bool; if False, parse stops after first error - :param ignore: list or tuple of directives to exclude from the payload - :param combine: bool; if True, use includes to create a single config obj - :param single: bool; if True, including from other files doesn't happen - :param comments: bool; if True, including comments to json payload - :param strict: bool; if True, unrecognized directives raise errors - :param check_ctx: bool; if True, runs context analysis on directives - :param check_args: bool; if True, runs arg count analysis on directives - :returns: a payload that describes the parsed nginx config - """ - config_dir = os.path.dirname(filename) - payload = { 'status': 'ok', 'errors': [], 'config': [], } - # start with the main nginx config file/context - includes = [(filename, ())] # stores (filename, config context) tuples - included = {filename: 0} # stores {filename: array index} map + # start with the main nginx config context + includes = [(initial_file, ())] # stores (filename, config context) tuples + included = {initial_file: 0} # stores {filename: array index} map def _handle_error(parsing, e): """Adds representaions of an error to the payload""" @@ -215,10 +199,16 @@ def _parse(parsing, tokens, ctx=(), consume=False): return parsed # the includes list grows as "include" directives are found in _parse - for fname, ctx in includes: - tokens = lex(fname) + for index, (fname, ctx) in enumerate(includes): + if index == 0: + tokens = initial_tokens + parsing_file = initial_file + else: + tokens = lex(fname) + parsing_file = fname + parsing = { - 'file': fname, + 'file': parsing_file, 'status': 'ok', 'errors': [], 'parsed': [] @@ -236,6 +226,65 @@ def _parse(parsing, tokens, ctx=(), consume=False): return payload +def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False, + comments=False, strict=False, combine=False, check_ctx=True, + check_args=True): + """ + Parses an nginx config file and returns a nested dict payload + + :param filename: string contianing the name of the config file to parse + :param onerror: function that determines what's saved in "callback" + :param catch_errors: bool; if False, parse stops after first error + :param ignore: list or tuple of directives to exclude from the payload + :param combine: bool; if True, use includes to create a single config obj + :param single: bool; if True, including from other files doesn't happen + :param comments: bool; if True, including comments to json payload + :param strict: bool; if True, unrecognized directives raise errors + :param check_ctx: bool; if True, runs context analysis on directives + :param check_args: bool; if True, runs arg count analysis on directives + :returns: a payload that describes the parsed nginx config + """ + config_dir = os.path.dirname(filename) + initial_tokens = lex(filename) + return _parse_with_initial_tokens( + initial_tokens, filename, config_dir, + onerror=onerror, catch_errors=catch_errors, ignore=ignore, + single=single, comments=comments, strict=strict, combine=combine, + check_ctx=check_ctx, check_args=check_args + ) + + +def parse_string(text, filename=None, onerror=None, catch_errors=True, ignore=(), + single=False, comments=False, strict=False, combine=False, check_ctx=True, + check_args=True): + """ + Parses an nginx config provided as a string and returns a nested dict payload + + :param text: string containing the nginx config to parse + :param filename: optional filename used for error messages and include base + :param onerror: function that determines what's saved in "callback" + :param catch_errors: bool; if False, parse stops after first error + :param ignore: list or tuple of directives to exclude from the payload + :param combine: bool; if True, use includes to create a single config obj + :param single: bool; if True, including from other files doesn't happen + :param comments: bool; if True, including comments to json payload + :param strict: bool; if True, unrecognized directives raise errors + :param check_ctx: bool; if True, runs context analysis on directives + :param check_args: bool; if True, runs arg count analysis on directives + :returns: a payload that describes the parsed nginx config + """ + # resolve base directory for relative include paths + base_filename = filename if filename is not None else '' + config_dir = os.path.dirname(os.path.abspath(filename)) if filename else os.getcwd() + initial_tokens = lex_string(text, filename=filename) + return _parse_with_initial_tokens( + initial_tokens, base_filename, config_dir, + onerror=onerror, catch_errors=catch_errors, ignore=ignore, + single=single, comments=comments, strict=strict, combine=combine, + check_ctx=check_ctx, check_args=check_args + ) + + def _combine_parsed_configs(old_payload): """ Combines config files into one by using include directives. diff --git a/tests/test_parse.py b/tests/test_parse.py index c3d8e7d..3a5a6b6 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -109,6 +109,36 @@ def test_includes_regular(): } +def test_parse_string_simple_matches_file(): + dirname = os.path.join(here, 'configs', 'simple') + config_path = os.path.join(dirname, 'nginx.conf') + with open(config_path, 'r') as f: + text = f.read() + payload_file = crossplane.parse(config_path) + payload_string = crossplane.parse_string(text, filename=config_path) + assert payload_file == payload_string + + +def test_parse_string_includes_regular_matches_file(): + dirname = os.path.join(here, 'configs', 'includes-regular') + config_path = os.path.join(dirname, 'nginx.conf') + with open(config_path, 'r') as f: + text = f.read() + payload_file = crossplane.parse(config_path) + payload_string = crossplane.parse_string(text, filename=config_path) + assert payload_file == payload_string + + +def test_parse_string_includes_globbed_combined_matches_file(): + dirname = os.path.join(here, 'configs', 'includes-globbed') + config_path = os.path.join(dirname, 'nginx.conf') + with open(config_path, 'r') as f: + text = f.read() + payload_file = crossplane.parse(config_path, combine=True) + payload_string = crossplane.parse_string(text, filename=config_path, combine=True) + assert payload_file == payload_string + + def test_includes_globbed(): dirname = os.path.join(here, 'configs', 'includes-globbed') config = os.path.join(dirname, 'nginx.conf')