Support string API

README.md

@@ -18,8 +18,10 @@
       - [crossplane minify](#crossplane-minify)
   - [Python Module](#python-module)
       - [crossplane.parse()](#crossplaneparse)
+      - [crossplane.parse_string()](#crossplaneparse_string)
       - [crossplane.build()](#crossplanebuild)
       - [crossplane.lex()](#crossplanelex)
+      - [crossplane.lex_string()](#crossplanelex_string)
   - [Other Languages](#other-languages)
 
 ## Install
@@ -469,8 +471,8 @@ optional arguments:
 ## Python Module
 
 In addition to the command line tool, you can import `crossplane` as a
-python module. There are two basic functions that the module will
-provide you: `parse` and `lex`.
+python module. There are four basic functions that the module will
+provide you: `parse`, `parse_string`, `lex`, and `lex_string`.
 
 ### crossplane.parse()
 
@@ -483,6 +485,23 @@ This will return the same payload as described in the [crossplane
 parse](#crossplane-parse) section, except it will be Python dicts and
 not one giant JSON string.
 
+### crossplane.parse_string()
+
+```python
+import crossplane
+text = """
+events {}
+http {
+    include conf.d/*.conf;
+}
+"""
+payload = crossplane.parse_string(text, filename='/etc/nginx/nginx.conf')
+```
+
+Parses configuration provided as a string. If you pass `filename`, relative include
+patterns are resolved against its directory, and error messages reference it. Options
+mirror `crossplane.parse`.
+
 ### crossplane.build()
 
 ```python
@@ -515,6 +534,17 @@ will result in a long list similar to what you can see in the
 is used, except it will obviously be a Python list of tuples and not one
 giant JSON string.
 
+### crossplane.lex_string()
+
+```python
+import crossplane
+text = "events { worker_connections 1024; }"
+tokens = list(crossplane.lex_string(text, filename='<string>'))
+```
+
+Lexes tokens from a configuration string. The optional `filename` is used for error
+reporting when brace-balance errors are detected.
+
 ## Other Languages
 
 - Go port by [@aluttik](https://github.com/aluttik):

crossplane/__init__.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
-from .parser import parse
-from .lexer import lex
+from .parser import parse, parse_string
+from .lexer import lex, lex_string
 from .builder import build
 from .formatter import format
 from .ext.lua import LuaBlockPlugin
 
-__all__ = ['parse', 'lex', 'build', 'format']
+__all__ = ['parse', 'parse_string', 'lex', 'lex_string', 'build', 'format']
 
 __title__ = 'crossplane'
 __summary__ = 'Reliable and fast NGINX configuration file parser.'

crossplane/lexer.py

@@ -155,3 +155,16 @@ def lex(filename):
 def register_external_lexer(directives, lexer):
     for directive in directives:
         EXTERNAL_LEXERS[directive] = lexer
+
+
+def lex_string(text, filename=None):
+    """Generates tokens from an nginx config string.
+
+    :param text: configuration text to lex
+    :param filename: optional filename to use in error reporting
+    """
+    f = io.StringIO(text)
+    it = _lex_file_object(f)
+    it = _balance_braces(it, filename)
+    for token, line, quoted in it:
+        yield (token, line, quoted)

crossplane/parser.py

@@ -2,7 +2,7 @@
 import glob
 import os
 
-from .lexer import lex
+from .lexer import lex, lex_string
 from .analyzer import analyze, enter_block_ctx
 from .errors import NgxParserDirectiveError
 
@@ -22,35 +22,19 @@ def _prepare_if_args(stmt):
         args[:] = args[start:end]
 
 
-def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False,
+def _parse_with_initial_tokens(initial_tokens, initial_file, config_dir,
+        onerror=None, catch_errors=True, ignore=(), single=False,
         comments=False, strict=False, combine=False, check_ctx=True,
         check_args=True):
-    """
-    Parses an nginx config file and returns a nested dict payload
-
-    :param filename: string contianing the name of the config file to parse
-    :param onerror: function that determines what's saved in "callback"
-    :param catch_errors: bool; if False, parse stops after first error
-    :param ignore: list or tuple of directives to exclude from the payload
-    :param combine: bool; if True, use includes to create a single config obj
-    :param single: bool; if True, including from other files doesn't happen
-    :param comments: bool; if True, including comments to json payload
-    :param strict: bool; if True, unrecognized directives raise errors
-    :param check_ctx: bool; if True, runs context analysis on directives
-    :param check_args: bool; if True, runs arg count analysis on directives
-    :returns: a payload that describes the parsed nginx config
-    """
-    config_dir = os.path.dirname(filename)
-
     payload = {
         'status': 'ok',
         'errors': [],
         'config': [],
     }
 
-    # start with the main nginx config file/context
-    includes = [(filename, ())]  # stores (filename, config context) tuples
-    included = {filename: 0} # stores {filename: array index} map
+    # start with the main nginx config context
+    includes = [(initial_file, ())]  # stores (filename, config context) tuples
+    included = {initial_file: 0}  # stores {filename: array index} map
 
     def _handle_error(parsing, e):
         """Adds representaions of an error to the payload"""
@@ -215,10 +199,16 @@ def _parse(parsing, tokens, ctx=(), consume=False):
         return parsed
 
     # the includes list grows as "include" directives are found in _parse
-    for fname, ctx in includes:
-        tokens = lex(fname)
+    for index, (fname, ctx) in enumerate(includes):
+        if index == 0:
+            tokens = initial_tokens
+            parsing_file = initial_file
+        else:
+            tokens = lex(fname)
+            parsing_file = fname
+
         parsing = {
-            'file': fname,
+            'file': parsing_file,
             'status': 'ok',
             'errors': [],
             'parsed': []
@@ -236,6 +226,65 @@ def _parse(parsing, tokens, ctx=(), consume=False):
         return payload
 
 
+def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False,
+        comments=False, strict=False, combine=False, check_ctx=True,
+        check_args=True):
+    """
+    Parses an nginx config file and returns a nested dict payload
+
+    :param filename: string contianing the name of the config file to parse
+    :param onerror: function that determines what's saved in "callback"
+    :param catch_errors: bool; if False, parse stops after first error
+    :param ignore: list or tuple of directives to exclude from the payload
+    :param combine: bool; if True, use includes to create a single config obj
+    :param single: bool; if True, including from other files doesn't happen
+    :param comments: bool; if True, including comments to json payload
+    :param strict: bool; if True, unrecognized directives raise errors
+    :param check_ctx: bool; if True, runs context analysis on directives
+    :param check_args: bool; if True, runs arg count analysis on directives
+    :returns: a payload that describes the parsed nginx config
+    """
+    config_dir = os.path.dirname(filename)
+    initial_tokens = lex(filename)
+    return _parse_with_initial_tokens(
+        initial_tokens, filename, config_dir,
+        onerror=onerror, catch_errors=catch_errors, ignore=ignore,
+        single=single, comments=comments, strict=strict, combine=combine,
+        check_ctx=check_ctx, check_args=check_args
+    )
+
+
+def parse_string(text, filename=None, onerror=None, catch_errors=True, ignore=(),
+        single=False, comments=False, strict=False, combine=False, check_ctx=True,
+        check_args=True):
+    """
+    Parses an nginx config provided as a string and returns a nested dict payload
+
+    :param text: string containing the nginx config to parse
+    :param filename: optional filename used for error messages and include base
+    :param onerror: function that determines what's saved in "callback"
+    :param catch_errors: bool; if False, parse stops after first error
+    :param ignore: list or tuple of directives to exclude from the payload
+    :param combine: bool; if True, use includes to create a single config obj
+    :param single: bool; if True, including from other files doesn't happen
+    :param comments: bool; if True, including comments to json payload
+    :param strict: bool; if True, unrecognized directives raise errors
+    :param check_ctx: bool; if True, runs context analysis on directives
+    :param check_args: bool; if True, runs arg count analysis on directives
+    :returns: a payload that describes the parsed nginx config
+    """
+    # resolve base directory for relative include paths
+    base_filename = filename if filename is not None else '<string>'
+    config_dir = os.path.dirname(os.path.abspath(filename)) if filename else os.getcwd()
+    initial_tokens = lex_string(text, filename=filename)
+    return _parse_with_initial_tokens(
+        initial_tokens, base_filename, config_dir,
+        onerror=onerror, catch_errors=catch_errors, ignore=ignore,
+        single=single, comments=comments, strict=strict, combine=combine,
+        check_ctx=check_ctx, check_args=check_args
+    )
+
+
 def _combine_parsed_configs(old_payload):
     """
     Combines config files into one by using include directives.

tests/test_parse.py

@@ -109,6 +109,36 @@ def test_includes_regular():
     }
 
 
+def test_parse_string_simple_matches_file():
+    dirname = os.path.join(here, 'configs', 'simple')
+    config_path = os.path.join(dirname, 'nginx.conf')
+    with open(config_path, 'r') as f:
+        text = f.read()
+    payload_file = crossplane.parse(config_path)
+    payload_string = crossplane.parse_string(text, filename=config_path)
+    assert payload_file == payload_string
+
+
+def test_parse_string_includes_regular_matches_file():
+    dirname = os.path.join(here, 'configs', 'includes-regular')
+    config_path = os.path.join(dirname, 'nginx.conf')
+    with open(config_path, 'r') as f:
+        text = f.read()
+    payload_file = crossplane.parse(config_path)
+    payload_string = crossplane.parse_string(text, filename=config_path)
+    assert payload_file == payload_string
+
+
+def test_parse_string_includes_globbed_combined_matches_file():
+    dirname = os.path.join(here, 'configs', 'includes-globbed')
+    config_path = os.path.join(dirname, 'nginx.conf')
+    with open(config_path, 'r') as f:
+        text = f.read()
+    payload_file = crossplane.parse(config_path, combine=True)
+    payload_string = crossplane.parse_string(text, filename=config_path, combine=True)
+    assert payload_file == payload_string
+
+
 def test_includes_globbed():
     dirname = os.path.join(here, 'configs', 'includes-globbed')
     config = os.path.join(dirname, 'nginx.conf')