Enable break_on_newline extension by default for Markdown

Enable the existing break_on_newline extension in DEFAULT_EXTENSIONS so
the Markdown parser converts soft line breaks to HardBreak objects. This
produces visible <br> line breaks in HTML output, matching GFM rendering.

The conversion happens in the Markdown parser's paragraph() method,
which is the proper place for Markdown-specific behavior. The generic
accept_paragraph in ToHtml is unchanged and has no Markdown-specific
logic.

Also remove the CJK-aware newline-to-space gsub from accept_paragraph
that is no longer needed. The RDoc markup parser already handles newline
joining at parse time in build_paragraph.

Author: st0012

doc/markup_reference/markdown.md

@@ -579,7 +579,7 @@ This section compares RDoc's Markdown implementation with the
 | Fenced Code (tildes) | ✅ `~~~` | ❌ | Conflicts with strikethrough syntax |
 | Info strings (language) | ✅ any | ⚠️ limited | `ruby`/`rb`, `c`, and `bash`/`sh`/`shell`/`console` highlighted; others accepted as CSS class |
 | Blockquotes | ✅ | ✅ | Full match, nested supported |
-| Lazy Continuation | ✅ | ⚠️ | Continuation text is included in blockquote but line break is lost (becomes a space) |
+| Lazy Continuation | ✅ | ⚠️ | Continuation text is included in blockquote; line break is preserved as `<br>` |
 | Bullet Lists | ✅ | ✅ | `*`, `+`, `-` supported |
 | Ordered Lists | ✅ `.` `)` | ⚠️ `.` only | RDoc doesn't support `)` delimiter; numbers are always renumbered from 1 |
 | Nested Lists | ✅ | ✅ | 4-space indentation |
@@ -620,6 +620,7 @@ RDoc uses a whitelist of block-level tags defined in
 | Link titles | ✅ | ⚠️ | Parsed but not rendered |
 | Images | ✅ | ✅ | Full match |
 | Autolinks `<url>` | ✅ | ✅ | Full match |
+| Soft line breaks | ✅ | ⚠️ | Newlines within paragraphs produce `<br>`; GFM spec renders as a space |
 | Hard line breaks | ✅ | ⚠️ | 2+ trailing spaces only; backslash `\` at EOL not supported |
 | Backslash escapes | ✅ | ⚠️ | Subset of GFM's escapable characters (e.g., `~` not escapable) |
 | HTML entities | ✅ | ✅ | Named, decimal, hex |

lib/rdoc/markdown.kpeg

@@ -202,6 +202,7 @@
   # Extensions enabled by default
 
   DEFAULT_EXTENSIONS = [
+    :break_on_newline,
     :definition_lists,
     :github,
     :html,

lib/rdoc/markdown.rb

@@ -587,6 +587,7 @@ def self.rule_info(name, rendered)
   # Extensions enabled by default
 
   DEFAULT_EXTENSIONS = [
+    :break_on_newline,
     :definition_lists,
     :github,
     :html,

lib/rdoc/markup/to_html.rb

@@ -313,9 +313,6 @@ def accept_block_quote(block_quote)
   def accept_paragraph(paragraph)
     @res << "\n<p>"
     text = paragraph.text @hard_break
-    text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K\r?\n(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) {
-      defined?($2) && ' '
-    }
     @res << to_html(text)
     @res << "</p>\n"
   end

test/rdoc/markup/to_html_test.rb

@@ -236,7 +236,7 @@ def accept_paragraph_br
   end
 
   def accept_paragraph_break
-    assert_equal "\n<p>hello<br> world</p>\n", @to.res.join
+    assert_equal "\n<p>hello<br>\nworld</p>\n", @to.res.join
   end
 
   def accept_paragraph_i
@@ -411,43 +411,13 @@ def test_accept_heading_dedup_resets_on_start_accepting
   end
 
   def test_accept_paragraph_newline
-    hellos = ["hello", "\u{393 3b5 3b9 3ac} \u{3c3 3bf 3c5}"]
-    worlds = ["world", "\u{3ba 3cc 3c3 3bc 3bf 3c2}"]
-    ohayo, sekai = %W"\u{304a 306f 3088 3046} \u{4e16 754c}"
-
-    hellos.product(worlds) do |hello, world|
-      @to.start_accepting
-      @to.accept_paragraph para("#{hello}\n", "#{world}\n")
-      assert_equal "\n<p>#{hello} #{world}</p>\n", @to.res.join
-    end
-
-    hellos.each do |hello|
-      @to.start_accepting
-      @to.accept_paragraph para("#{hello}\n", "#{sekai}\n")
-      assert_equal "\n<p>#{hello}#{sekai}</p>\n", @to.res.join
-    end
-
-    worlds.each do |world|
-      @to.start_accepting
-      @to.accept_paragraph para("#{ohayo}\n", "#{world}\n")
-      assert_equal "\n<p>#{ohayo}#{world}</p>\n", @to.res.join
-    end
-
     @to.start_accepting
-    @to.accept_paragraph para("#{ohayo}\n", "#{sekai}\n")
-    assert_equal "\n<p>#{ohayo}#{sekai}</p>\n", @to.res.join
+    @to.accept_paragraph para("hello\n", "world\n")
+    assert_equal "\n<p>hello\nworld\n</p>\n", @to.res.join
 
     @to.start_accepting
     @to.accept_paragraph para("+hello+\n", "world\n")
-    assert_equal "\n<p><code>hello</code> world</p>\n", @to.res.join
-
-    @to.start_accepting
-    @to.accept_paragraph para("hello\n", "+world+\n")
-    assert_equal "\n<p>hello <code>world</code></p>\n", @to.res.join
-
-    @to.start_accepting
-    @to.accept_paragraph para("+hello+\n", "+world+\n")
-    assert_equal "\n<p><code>hello</code> <code>world</code></p>\n", @to.res.join
+    assert_equal "\n<p><code>hello</code>\nworld\n</p>\n", @to.res.join
   end
 
   def test_accept_heading_output_decoration

test/rdoc/parser/changelog_test.rb

@@ -421,8 +421,8 @@ def test_scan_git
                 'Masataka Pocke Kuwabara', 'kuwabara@pocke.me', '2021-01-01 14:25:08 +0900',
                 [head(4, 'Make args info for RubyVM::AST to available on endless method without parens'),
                  head(5, 'Problem'),
-                 para("Arguments information is missing for endless method without parens.\n" +
-                      "For example:"),
+                 para("Arguments information is missing for endless method without parens.",
+                      hard_break, "For example:"),
                  verb("# ok\n").tap {|v| v.format = :ruby},
                  para('It causes an error if a program expects <code>args</code> node exists.'),
                  head(5, 'Solution'),

test/rdoc/rdoc_markdown_test.rb

@@ -56,7 +56,7 @@ def test_parse_block_quote
     expected =
       doc(
         block(
-          para("this is\na block quote")))
+          para("this is", hard_break, "a block quote")))
 
     assert_equal expected, doc
   end
@@ -70,11 +70,22 @@ def test_parse_block_quote_continue
     expected =
       doc(
         block(
-          para("this is\na block quote")))
+          para("this is", hard_break, "a block quote")))
 
     assert_equal expected, doc
   end
 
+  def test_parse_block_quote_continue_html
+    doc = parse <<-BLOCK_QUOTE
+> this is
+a block quote
+    BLOCK_QUOTE
+
+    html = doc.accept(RDoc::Markup::ToHtml.new)
+
+    assert_include html, "<p>this is<br>\na block quote</p>"
+  end
+
   def test_parse_block_quote_list
     doc = parse <<-BLOCK_QUOTE
 > text
@@ -104,7 +115,7 @@ def test_parse_block_quote_newline
     expected =
       doc(
         block(
-          para("this is\na block quote")))
+          para("this is", hard_break, "a block quote")))
 
     assert_equal expected, doc
   end
@@ -120,7 +131,7 @@ def test_parse_block_quote_separate
     expected =
       doc(
         block(
-          para("this is\na block quote"),
+          para("this is", hard_break, "a block quote"),
           para("that continues")))
 
     assert_equal expected, doc
@@ -262,7 +273,7 @@ def test_parse_code_github
     assert_equal expected, parse(doc)
 
     expected =
-      doc(para("Example:\n<code>\n""code goes here\n</code>"))
+      doc(para("Example:", hard_break, "<code>\n""code goes here\n</code>"))
 
     assert_equal expected, parse(doc.sub(/^\n/, ''))
   end
@@ -296,7 +307,7 @@ def test_parse_code_github_format
     assert_equal expected, parse(doc)
 
     expected =
-      doc(para("Example:\n<code>ruby\n""code goes here\n</code>"))
+      doc(para("Example:", hard_break, "<code>ruby\n""code goes here\n</code>"))
 
     assert_equal expected, parse(doc.sub(/^\n/, ''))
   end
@@ -343,7 +354,7 @@ def test_parse_definition_list_indents
         item(%w[one],   para("Indented one characters")),
         item(%w[two],   para("Indented two characters")),
         item(%w[three], para("Indented three characters"))),
-      para("four\n : Indented four characters"))
+      para("four", hard_break, " : Indented four characters"))
 
     assert_equal expected, doc
   end
@@ -392,9 +403,9 @@ def test_parse_definition_list_multi_line
     expected = doc(
       list(:NOTE,
         item(%w[one],
-          para("This is a definition\nthat extends to two lines")),
+          para("This is a definition", hard_break, "that extends to two lines")),
         item(%w[two],
-          para("This is another definition\nthat also extends to two lines"))))
+          para("This is another definition", hard_break, "that also extends to two lines"))))
 
     assert_equal expected, doc
   end
@@ -430,8 +441,8 @@ def test_parse_definition_list_no
     MD
 
     expected = doc(
-        para("one\n: This is a definition"),
-        para("two\n: This is another definition"))
+        para("one", hard_break, ": This is a definition"),
+        para("two", hard_break, ": This is another definition"))
 
     assert_equal expected, doc
   end
@@ -779,7 +790,7 @@ def test_parse_list_bullet_multiline
 
     expected = doc(
       list(:BULLET,
-        item(nil, para("one\n two"))))
+        item(nil, para("one", hard_break, " two"))))
 
     assert_equal expected, doc
   end
@@ -832,7 +843,7 @@ def test_parse_list_bullet_nest_continue
           para("outer"),
           list(:BULLET,
             item(nil,
-              para("inner\n continue inner")))),
+              para("inner", hard_break, " continue inner")))),
         item(nil,
           para("outer 2"))))
 
@@ -899,7 +910,7 @@ def test_parse_note_indent
     expected = doc(
       para("Some text.{*1}[rdoc-label:foottext-1:footmark-1]"),
       rule(1),
-      para("{^1}[rdoc-label:footmark-1:foottext-1] With a footnote\n\nmore"))
+      para("{^1}[rdoc-label:footmark-1:foottext-1] With a footnote", hard_break, "more"))
 
     assert_equal expected, doc
   end
@@ -940,8 +951,10 @@ def test_parse_note_multiple
     MD
 
     expected = doc(
-      para("Some text{*1}[rdoc-label:foottext-1:footmark-1]\n" +
-           "with inline notes{*2}[rdoc-label:foottext-2:footmark-2]\n" +
+      para("Some text{*1}[rdoc-label:foottext-1:footmark-1]",
+           hard_break,
+           "with inline notes{*2}[rdoc-label:foottext-2:footmark-2]",
+           hard_break,
            "and an extra note.{*3}[rdoc-label:foottext-3:footmark-3]"),
 
       rule(1),
@@ -1040,7 +1053,7 @@ def test_parse_paragraph_indent_three
   def test_parse_paragraph_multiline
     doc = parse "one\ntwo"
 
-    expected = doc(para("one\ntwo"))
+    expected = doc(para("one", hard_break, "two"))
 
     assert_equal expected, doc
   end
@@ -1444,7 +1457,7 @@ def test_info_string_css_classes
   def test_lazy_continuation_in_blockquote
     html = render("> Foo\nBar\n")
     assert_match(%r{<blockquote>.*Foo.*Bar.*</blockquote>}m, html)
-    assert_match(%r{Foo Bar}, html)
+    assert_match(%r{Foo<br>\nBar}, html)
   end
 
   def test_ordered_list_paren_delimiter_not_supported
@@ -1483,8 +1496,12 @@ def test_autolinks
   end
 
   def test_backslash_line_break_not_supported
+    # Backslash line break syntax (\<newline>) is not specifically supported,
+    # but break_on_newline converts all newlines to <br>, so <br> is present.
+    # The backslash itself appears literally in the output.
     html = render("Line one\\\nLine two\n")
-    assert_not_match(%r{<br>}, html)
+    assert_match(%r{<br>}, html)
+    assert_match(%r{Line one\\}, html)
   end
 
   def test_escape_tilde_not_supported