Implement YAML block additional feature

This implements support for parsing YAML blocks and front-matter
following the Pandoc rules:

  1. Delimited by three hyphens (---) at the top
  2. Delimited by three hyphens (---) or dots (...) at the bottom
  3. May occur anywhere in the document, but if not at the beginning
     must be preceded by a blank line.

BlockTag.YamlBlock is defined, and FencedCodeData is used to store the
closing fence character (either `-` or `.`) and to indicate whether the
closing fence has been seen in the same manner as BlockTag.FencedCode.

YAML support may be enabled via two-flavors:

  1. CommonMarkAdditionalFeatures.YamlBlocks: this allows for any
     number of YAML blocks anywhere in the document.

  2. CommonMarkAdditionalFeatures.YamlFrontMatterOnly: allows for
     exactly one YAML block, defined on the first line of the document.

The HTML formatters treat BlockTag.YamlBlock blocks the same way as
BlockTag.FencedCode blocks, except that instead of writing the info
string, a `class="language-yaml"` attribute will be written.

Author: abock

CommonMark.Tests/CommonMark.Tests.csproj

@@ -75,6 +75,7 @@
       <DesignTime>True</DesignTime>
       <DependentUpon>Specs.tt</DependentUpon>
     </Compile>
+    <Compile Include="YamlBlockTests.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="Specification\Specs.tt">

CommonMark.Tests/YamlBlockTests.cs

@@ -0,0 +1,112 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+using CommonMark.Syntax;
+
+namespace CommonMark.Tests
+{
+    [TestClass]
+    public class YamlBlockTests
+    {
+        private const string Category = "Container blocks - YAML";
+
+        private static CommonMarkSettings GetSettings(bool frontMatterOnly = false)
+        {
+            var settings = CommonMarkSettings.Default.Clone();
+            settings.AdditionalFeatures = frontMatterOnly
+                ? CommonMarkAdditionalFeatures.YamlFrontMatterOnly
+                : CommonMarkAdditionalFeatures.YamlBlocks;
+            return settings;
+        }
+
+        [TestMethod]
+        [TestCategory(Category)]
+        public void YamlDisabled()
+        {
+            Helpers.ExecuteTest(
+                "---\nparagraph\n...",
+                "<hr />\n<p>paragraph\n...</p>");
+        }
+
+        [TestMethod]
+        [TestCategory(Category)]
+        public void YamlEmpty()
+        {
+            Helpers.ExecuteTest(
+                "---\n\n...",
+                "<pre><code class=\"language-yaml\">\n</code></pre>",
+                GetSettings());
+        }
+
+        [TestMethod]
+        [TestCategory(Category)]
+        public void YamlFrontMatterOnly()
+        {
+            Helpers.ExecuteTest(
+                "---\nfrontmatter\n...\n\nparagraph\n\n---\nline 1\nline 2\n...\n",
+                "<pre><code class=\"language-yaml\">frontmatter\n</code></pre>\n" +
+                "<p>paragraph</p>\n" +
+                "<hr />\n" +
+                "<p>line 1\nline 2\n...</p>",
+                GetSettings(true));
+        }
+
+        [TestMethod]
+        [TestCategory(Category)]
+        public void YamlMultipleBlocks()
+        {
+            Helpers.ExecuteTest(
+                "---\nfrontmatter\n...\n\nparagraph\n\n---\nline 1\nline 2\n...\n",
+                "<pre><code class=\"language-yaml\">frontmatter\n</code></pre>\n" +
+                "<p>paragraph</p>\n" +
+                "<pre><code class=\"language-yaml\">line 1\nline 2\n</code></pre>",
+                GetSettings());
+        }
+
+        [TestMethod]
+        [TestCategory(Category)]
+        public void YamlAndThematicBreak()
+        {
+            Helpers.ExecuteTest(
+                "----\n\nnot yaml\n---\nalso not yaml\n\n---\nbut this\nis\nyaml\n...\npara",
+                "<hr />\n" +
+                "<h2>not yaml</h2>\n" +
+                "<p>also not yaml</p>\n" +
+                "<pre><code class=\"language-yaml\">but this\nis\nyaml\n</code></pre>\n" +
+                "<p>para</p>",
+                GetSettings());
+        }
+
+        [TestMethod]
+        [TestCategory(Category)]
+        public void YamlClosingFenceDash()
+        {
+            AssertYamlClosingFenceAndAst("---");
+        }
+
+        [TestMethod]
+        [TestCategory(Category)]
+        public void YamlClosingFenceDot()
+        {
+            AssertYamlClosingFenceAndAst("...");
+        }
+
+        private static void AssertYamlClosingFenceAndAst(string fence)
+        {
+            var markdown = "---\nyaml\n" + fence;
+            Helpers.ExecuteTest(
+                markdown,
+                "<pre><code class=\"language-yaml\">yaml\n</code></pre>",
+                GetSettings());
+
+            var doc = CommonMarkConverter.Parse(markdown, GetSettings());
+
+            Assert.IsNotNull(doc.FirstChild);
+            Assert.AreEqual(BlockTag.YamlBlock, doc.FirstChild.Tag);
+            Assert.IsNotNull(doc.FirstChild.FencedCodeData);
+            Assert.AreEqual(0, doc.FirstChild.FencedCodeData.FenceOffset);
+            Assert.AreEqual(-1, doc.FirstChild.FencedCodeData.FenceLength);
+            Assert.AreEqual(fence[0], doc.FirstChild.FencedCodeData.FenceChar);
+            Assert.IsNull(doc.FirstChild.FencedCodeData.Info);
+        }
+    }
+}

CommonMark/CommonMarkAdditionalFeatures.cs

@@ -25,6 +25,19 @@ public enum CommonMarkAdditionalFeatures
         /// </summary>
         PlaceholderBracket = 2,
 
+        /// <summary>
+        /// Allow YAML blocks (delimited by a line starting with exactly <c>---</c> and a line ending
+        /// with exactly <c>---</c> or <c>...</c>. YAML blocks will take precedence over a <c>---</c>
+        /// that might otherwise yield a thematic break.
+        /// </summary>
+        YamlBlocks = 4,
+
+        /// <summary>
+        /// Like <see cref="YamlBlocks"/> but will yield a maximum of one block, and only if the first
+        /// line is exactly <c>---</c>.
+        /// </summary>
+        YamlFrontMatterOnly = 8,
+
         /// <summary>
         /// All additional features are enabled.
         /// </summary>

CommonMark/CommonMarkConverter.cs

@@ -111,7 +111,7 @@ public static Syntax.Block ProcessStage1(TextReader source, CommonMarkSettings s
                 reader.ReadLine(line);
                 while (line.Line != null)
                 {
-                    BlockMethods.IncorporateLine(line, ref cur);
+                    BlockMethods.IncorporateLine(line, ref cur, settings);
                     reader.ReadLine(line);
                 }
             }

CommonMark/Formatters/HtmlFormatter.cs

@@ -219,6 +219,7 @@ protected virtual void WriteBlock(Block block, bool isOpening, bool isClosing, o
 
                 case BlockTag.IndentedCode:
                 case BlockTag.FencedCode:
+                case BlockTag.YamlBlock:
 
                     ignoreChildNodes = true;
 
@@ -237,6 +238,11 @@ protected virtual void WriteBlock(Block block, bool isOpening, bool isClosing, o
                         WriteEncodedHtml(new StringPart(info, 0, x));
                         Write('\"');
                     }
+                    else if (block.Tag == BlockTag.YamlBlock)
+                    {
+                        Write(" class=\"language-yaml\"");
+                    }
+
                     Write('>');
                     WriteEncodedHtml(block.StringContent);
                     WriteLine("</code></pre>");

CommonMark/Formatters/HtmlFormatterSlim.cs

@@ -318,6 +318,7 @@ private static void BlocksToHtmlInner(HtmlTextWriter writer, Block block, Common
 
                     case BlockTag.IndentedCode:
                     case BlockTag.FencedCode:
+                    case BlockTag.YamlBlock:
                         writer.EnsureLine();
                         writer.WriteConstant("<pre><code");
                         if (trackPositions) PrintPosition(writer, block);
@@ -333,6 +334,11 @@ private static void BlocksToHtmlInner(HtmlTextWriter writer, Block block, Common
                             EscapeHtml(new StringPart(info, 0, x), writer);
                             writer.Write('\"');
                         }
+                        else if (block.Tag == BlockTag.YamlBlock)
+                        {
+                            writer.WriteConstant(" class=\"language-yaml\"");
+                        }
+
                         writer.Write('>');
                         EscapeHtml(block.StringContent, writer);
                         writer.WriteLineConstant("</code></pre>");

CommonMark/Formatters/Printer.cs

@@ -162,6 +162,14 @@ public static void PrintBlocks(TextWriter writer, Block block, CommonMarkSetting
                                format_str(block.StringContent.ToString(buffer), buffer));
                         break;
 
+                    case BlockTag.YamlBlock:
+                        writer.Write("yaml_block");
+                        PrintPosition(trackPositions, writer, block);
+                        writer.Write(" closing_fence_char={0} {1}",
+                               block.FencedCodeData.FenceChar,
+                               format_str(block.StringContent.ToString(buffer), buffer));
+                        break;
+
                     case BlockTag.HtmlBlock:
                         writer.Write("html_block");
                         PrintPosition(trackPositions, writer, block);

CommonMark/Parser/BlockMethods.cs

@@ -29,7 +29,8 @@ private static bool AcceptsLines(BlockTag block_type)
             return (block_type == BlockTag.Paragraph ||
                     block_type == BlockTag.AtxHeading ||
                     block_type == BlockTag.IndentedCode ||
-                    block_type == BlockTag.FencedCode);
+                    block_type == BlockTag.FencedCode ||
+                    block_type == BlockTag.YamlBlock);
         }
 
         private static void AddLine(Block block, LineInfo lineInfo, string ln, int offset, int remainingSpaces, int length = -1)
@@ -137,9 +138,12 @@ public static void Finalize(Block b, LineInfo line)
                     break;
 
                 case BlockTag.FencedCode:
+                case BlockTag.YamlBlock:
                     // first line of contents becomes info
                     var firstlinelen = b.StringContent.IndexOf('\n') + 1;
-                    b.FencedCodeData.Info = InlineMethods.Unescape(b.StringContent.TakeFromStart(firstlinelen, true).Trim());
+                    var firstline = b.StringContent.TakeFromStart(firstlinelen, true);
+                    if (b.Tag == BlockTag.FencedCode)
+                        b.FencedCodeData.Info = InlineMethods.Unescape(firstline.Trim());
                     break;
 
                 case BlockTag.List: // determine tight/loose status
@@ -464,7 +468,7 @@ private static void AdvanceOffset(string line, int count, bool columns, ref int
         // Process one line at a time, modifying a block.
         // Returns 0 if successful.  curptr is changed to point to
         // the currently open block.
-        public static void IncorporateLine(LineInfo line, ref Block curptr)
+        public static void IncorporateLine(LineInfo line, ref Block curptr, CommonMarkSettings settings)
         {
             var ln = line.Line;
 
@@ -571,6 +575,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
                         }
 
                     case BlockTag.FencedCode:
+                    case BlockTag.YamlBlock:
                         {
                             // -1 means we've seen closer 
                             if (container.FencedCodeData.FenceLength == -1)
@@ -632,6 +637,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
             // unless last matched container is code block, try new container starts:
             while (container.Tag != BlockTag.FencedCode &&
                    container.Tag != BlockTag.IndentedCode &&
+                   container.Tag != BlockTag.YamlBlock &&
                    container.Tag != BlockTag.HtmlBlock)
             {
 
@@ -670,6 +676,20 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
 
                     AdvanceOffset(ln, first_nonspace + matched - offset, false, ref offset, ref column, ref remainingSpaces);
 
+                }
+                else if (!indented &&
+                         ((container.IsLastLineBlank && (settings.AdditionalFeatures & CommonMarkAdditionalFeatures.YamlBlocks) != 0)
+                         || (line.LineNumber == 1 && (settings.AdditionalFeatures & (CommonMarkAdditionalFeatures.YamlFrontMatterOnly | CommonMarkAdditionalFeatures.YamlBlocks)) != 0))
+                         && ln == "---\n")
+                {
+
+                    container = CreateChildBlock(container, line, BlockTag.YamlBlock, first_nonspace);
+                    container.FencedCodeData = new FencedCodeData();
+                    container.FencedCodeData.FenceChar = '-';
+                    container.FencedCodeData.FenceLength = 3;
+
+                    AdvanceOffset(ln, 3, false, ref offset, ref column, ref remainingSpaces);
+
                 }
                 else if (!indented && curChar == '<' && 
                     (0 != (matched = (int)Scanner.scan_html_block_start(ln, first_nonspace, ln.Length))
@@ -795,6 +815,7 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
                                           container.Tag != BlockTag.BlockQuote &&
                                           container.Tag != BlockTag.SetextHeading &&
                                           container.Tag != BlockTag.FencedCode &&
+                                          container.Tag != BlockTag.YamlBlock &&
                                           !(container.Tag == BlockTag.ListItem &&
                                             container.FirstChild == null &&
                                             container.SourcePosition >= line.LineOffset));
@@ -851,6 +872,20 @@ public static void IncorporateLine(LineInfo line, ref Block curptr)
                         AddLine(container, line, ln, offset, remainingSpaces);
                     }
 
+                }
+                else if (container.Tag == BlockTag.YamlBlock)
+                {
+
+                    if ((curChar == '-' && ln == "---\n") || (curChar == '.' && ln == "...\n"))
+                    {
+                        container.FencedCodeData.FenceLength = -1;
+                        container.FencedCodeData.FenceChar = ln[0];
+                    }
+                    else
+                    {
+                        AddLine(container, line, ln, offset, remainingSpaces);
+                    }
+
                 }
                 else if (container.Tag == BlockTag.HtmlBlock)
                 {

CommonMark/Syntax/BlockTag.cs

@@ -83,6 +83,14 @@ public enum BlockTag : byte
         /// <summary>
         /// A text block that contains only link reference definitions.
         /// </summary>
-        ReferenceDefinition
+        ReferenceDefinition,
+
+        /// <summary>
+        /// A YAML metadta block (for example, <c>---\nyaml: metadata\n...</c>).
+        /// Only present if <see cref="CommonMarkAdditionalFeatures.YamlBlocks"/> or
+        /// <see cref="CommonMarkAdditionalFeatures.YamlFrontMatterOnly"/> are enabled
+        /// The block is structured like a <see cref="FencedCode"/> block.
+        /// </summary>
+        YamlBlock
     }
 }