Fix markdown render behaviors (#34122)

* Fix #27645 * Add config options `MATH_CODE_BLOCK_DETECTION`, problematic syntaxes are disabled by default * Fix #33639 * Add config options `RENDER_OPTIONS_*`, old behaviors are kept
2025-06-03 10:54:42 +08:00 · 2025-04-05 11:56:48 +08:00
parent ee6929d96b
commit e1c2d05bde
33 changed files with 418 additions and 222 deletions
--- a/modules/markup/markdown/goldmark.go
+++ b/modules/markup/markdown/goldmark.go
@ -9,7 +9,6 @@ import (
 	"code.gitea.io/gitea/modules/container"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/internal"
-	"code.gitea.io/gitea/modules/setting"

 	"github.com/yuin/goldmark/ast"
 	east "github.com/yuin/goldmark/extension/ast"
@ -69,16 +68,8 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa
 			g.transformList(ctx, v, rc)
 		case *ast.Text:
 			if v.SoftLineBreak() && !v.HardLineBreak() {
-				// TODO: this was a quite unclear part, old code: `if metas["mode"] != "document" { use comment link break setting }`
-				// many places render non-comment contents with no mode=document, then these contents also use comment's hard line break setting
-				// especially in many tests.
-				markdownLineBreakStyle := ctx.RenderOptions.Metas["markdownLineBreakStyle"]
-				switch markdownLineBreakStyle {
-				case "comment":
-					v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInComments)
-				case "document":
-					v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInDocuments)
-				}
+				newLineHardBreak := ctx.RenderOptions.Metas["markdownNewLineHardBreak"] == "true"
+				v.SetHardLineBreak(newLineHardBreak)
 			}
 		case *ast.CodeSpan:
 			g.transformCodeSpan(ctx, v, reader)
--- a/modules/markup/markdown/markdown.go
+++ b/modules/markup/markdown/markdown.go
@ -126,11 +126,11 @@ func SpecializedMarkdown(ctx *markup.RenderContext) *GlodmarkRender {
 				highlighting.WithWrapperRenderer(r.highlightingRenderer),
 			),
 			math.NewExtension(&ctx.RenderInternal, math.Options{
-				Enabled:           setting.Markdown.EnableMath,
-				ParseDollarInline: true,
-				ParseDollarBlock:  true,
-				ParseSquareBlock:  true, // TODO: this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping, it should be deprecated in the future (by some config options)
-				// ParseBracketInline: true, // TODO: this is also a bad syntax "\( ... \)", it also conflicts, it should be deprecated in the future
+				Enabled:                  setting.Markdown.EnableMath,
+				ParseInlineDollar:        setting.Markdown.MathCodeBlockOptions.ParseInlineDollar,
+				ParseInlineParentheses:   setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses, // this is a bad syntax "\( ... \)", it conflicts with normal markdown escaping
+				ParseBlockDollar:         setting.Markdown.MathCodeBlockOptions.ParseBlockDollar,
+				ParseBlockSquareBrackets: setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets, //  this is a bad syntax "\[ ... \]", it conflicts with normal markdown escaping
 			}),
 			meta.Meta,
 		),
--- a/modules/markup/markdown/markdown_math_test.go
+++ b/modules/markup/markdown/markdown_math_test.go
@ -8,6 +8,8 @@ import (
 	"testing"

 	"code.gitea.io/gitea/modules/markup"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"

 	"github.com/stretchr/testify/assert"
 )
@ -15,6 +17,7 @@ import (
 const nl = "\n"

 func TestMathRender(t *testing.T) {
+	setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true}
 	testcases := []struct {
 		testcase string
 		expected string
@ -69,7 +72,7 @@ func TestMathRender(t *testing.T) {
 		},
 		{
 			"$$a$$",
-			`<code class="language-math display">a</code>` + nl,
+			`<p><code class="language-math">a</code></p>` + nl,
 		},
 		{
 			"$$a$$ test",
@ -111,6 +114,7 @@ func TestMathRender(t *testing.T) {
 }

 func TestMathRenderBlockIndent(t *testing.T) {
+	setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{ParseBlockDollar: true, ParseBlockSquareBrackets: true}
 	testcases := []struct {
 		name     string
 		testcase string
@ -243,3 +247,64 @@ x
 		})
 	}
 }
+
+func TestMathRenderOptions(t *testing.T) {
+	setting.Markdown.MathCodeBlockOptions = setting.MarkdownMathCodeBlockOptions{}
+	defer test.MockVariableValue(&setting.Markdown.MathCodeBlockOptions)
+	test := func(t *testing.T, expected, input string) {
+		res, err := RenderString(markup.NewTestRenderContext(), input)
+		assert.NoError(t, err)
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(res)), "input: %s", input)
+	}
+
+	// default (non-conflict) inline syntax
+	test(t, `<p><code class="language-math">a</code></p>`, "$`a`$")
+
+	// ParseInlineDollar
+	test(t, `<p>$a$</p>`, `$a$`)
+	setting.Markdown.MathCodeBlockOptions.ParseInlineDollar = true
+	test(t, `<p><code class="language-math">a</code></p>`, `$a$`)
+
+	// ParseInlineParentheses
+	test(t, `<p>(a)</p>`, `\(a\)`)
+	setting.Markdown.MathCodeBlockOptions.ParseInlineParentheses = true
+	test(t, `<p><code class="language-math">a</code></p>`, `\(a\)`)
+
+	// ParseBlockDollar
+	test(t, `<p>$$
+a
+$$</p>
+`, `
+$$
+a
+$$
+`)
+	setting.Markdown.MathCodeBlockOptions.ParseBlockDollar = true
+	test(t, `<pre class="code-block is-loading"><code class="language-math display">
+a
+</code></pre>
+`, `
+$$
+a
+$$
+`)
+
+	// ParseBlockSquareBrackets
+	test(t, `<p>[
+a
+]</p>
+`, `
+\[
+a
+\]
+`)
+	setting.Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true
+	test(t, `<pre class="code-block is-loading"><code class="language-math display">
+a
+</code></pre>
+`, `
+\[
+a
+\]
+`)
+}
--- a/modules/markup/markdown/math/inline_parser.go
+++ b/modules/markup/markdown/math/inline_parser.go
@ -15,26 +15,26 @@ type inlineParser struct {
 	trigger              []byte
 	endBytesSingleDollar []byte
 	endBytesDoubleDollar []byte
-	endBytesBracket      []byte
+	endBytesParentheses  []byte
+	enableInlineDollar   bool
 }

-var defaultInlineDollarParser = &inlineParser{
-	trigger:              []byte{'$'},
-	endBytesSingleDollar: []byte{'$'},
-	endBytesDoubleDollar: []byte{'$', '$'},
+func NewInlineDollarParser(enableInlineDollar bool) parser.InlineParser {
+	return &inlineParser{
+		trigger:              []byte{'$'},
+		endBytesSingleDollar: []byte{'$'},
+		endBytesDoubleDollar: []byte{'$', '$'},
+		enableInlineDollar:   enableInlineDollar,
+	}
 }

-func NewInlineDollarParser() parser.InlineParser {
-	return defaultInlineDollarParser
+var defaultInlineParenthesesParser = &inlineParser{
+	trigger:             []byte{'\\', '('},
+	endBytesParentheses: []byte{'\\', ')'},
 }

-var defaultInlineBracketParser = &inlineParser{
-	trigger:         []byte{'\\', '('},
-	endBytesBracket: []byte{'\\', ')'},
-}
-
-func NewInlineBracketParser() parser.InlineParser {
-	return defaultInlineBracketParser
+func NewInlineParenthesesParser() parser.InlineParser {
+	return defaultInlineParenthesesParser
 }

 // Trigger triggers this parser on $ or \
@ -46,7 +46,7 @@ func isPunctuation(b byte) bool {
 	return b == '.' || b == '!' || b == '?' || b == ',' || b == ';' || b == ':'
 }

-func isBracket(b byte) bool {
+func isParenthesesClose(b byte) bool {
 	return b == ')'
 }

@ -86,7 +86,11 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser.
 		}
 	} else {
 		startMarkLen = 2
-		stopMark = parser.endBytesBracket
+		stopMark = parser.endBytesParentheses
+	}
+
+	if line[0] == '$' && !parser.enableInlineDollar && (len(line) == 1 || line[1] != '`') {
+		return nil
 	}

 	if checkSurrounding {
@ -110,7 +114,7 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser.
 				succeedingCharacter = line[i+len(stopMark)]
 			}
 			// check valid ending character
-			isValidEndingChar := isPunctuation(succeedingCharacter) || isBracket(succeedingCharacter) ||
+			isValidEndingChar := isPunctuation(succeedingCharacter) || isParenthesesClose(succeedingCharacter) ||
 				succeedingCharacter == ' ' || succeedingCharacter == '\n' || succeedingCharacter == 0
 			if checkSurrounding && !isValidEndingChar {
 				break
--- a/modules/markup/markdown/math/math.go
+++ b/modules/markup/markdown/math/math.go
@ -14,10 +14,11 @@ import (
 )

 type Options struct {
-	Enabled           bool
-	ParseDollarInline bool
-	ParseDollarBlock  bool
-	ParseSquareBlock  bool
+	Enabled                  bool
+	ParseInlineDollar        bool // inline $$ xxx $$ text
+	ParseInlineParentheses   bool // inline \( xxx \) text
+	ParseBlockDollar         bool // block $$ multiple-line $$ text
+	ParseBlockSquareBrackets bool // block \[ multiple-line \] text
 }

 // Extension is a math extension
@ -42,16 +43,16 @@ func (e *Extension) Extend(m goldmark.Markdown) {
 		return
 	}

-	inlines := []util.PrioritizedValue{util.Prioritized(NewInlineBracketParser(), 501)}
-	if e.options.ParseDollarInline {
-		inlines = append(inlines, util.Prioritized(NewInlineDollarParser(), 502))
+	var inlines []util.PrioritizedValue
+	if e.options.ParseInlineParentheses {
+		inlines = append(inlines, util.Prioritized(NewInlineParenthesesParser(), 501))
 	}
+	inlines = append(inlines, util.Prioritized(NewInlineDollarParser(e.options.ParseInlineDollar), 502))
+
 	m.Parser().AddOptions(parser.WithInlineParsers(inlines...))
-
 	m.Parser().AddOptions(parser.WithBlockParsers(
-		util.Prioritized(NewBlockParser(e.options.ParseDollarBlock, e.options.ParseSquareBlock), 701),
+		util.Prioritized(NewBlockParser(e.options.ParseBlockDollar, e.options.ParseBlockSquareBrackets), 701),
 	))
-
 	m.Renderer().AddOptions(renderer.WithNodeRenderers(
 		util.Prioritized(NewBlockRenderer(e.renderInternal), 501),
 		util.Prioritized(NewInlineRenderer(e.renderInternal), 502),
--- a/modules/markup/render.go
+++ b/modules/markup/render.go
@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"

@ -46,7 +47,7 @@ type RenderOptions struct {
 	// user&repo, format&style&regexp (for external issue pattern), teams&org (for mention)
 	// RefTypeNameSubURL (for iframe&asciicast)
 	// markupAllowShortIssuePattern
-	// markdownLineBreakStyle (comment, document)
+	// markdownNewLineHardBreak
 	Metas map[string]string

 	// used by external render. the router "/org/repo/render/..." will output the rendered content in a standalone page
@ -247,7 +248,8 @@ func Init(renderHelpFuncs *RenderHelperFuncs) {
 }

 func ComposeSimpleDocumentMetas() map[string]string {
-	return map[string]string{"markdownLineBreakStyle": "document"}
+	// TODO: there is no separate config option for "simple document" rendering, so temporarily use the same config as "repo file"
+	return map[string]string{"markdownNewLineHardBreak": strconv.FormatBool(setting.Markdown.RenderOptionsRepoFile.NewLineHardBreak)}
 }

 type TestRenderHelper struct {
--- a/modules/setting/markup.go
+++ b/modules/setting/markup.go
@ -8,6 +8,7 @@ import (
 	"strings"

 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
 )

 // ExternalMarkupRenderers represents the external markup renderers
@ -23,18 +24,33 @@ const (
 	RenderContentModeIframe      = "iframe"
 )

+type MarkdownRenderOptions struct {
+	NewLineHardBreak  bool
+	ShortIssuePattern bool // Actually it is a "markup" option because it is used in "post processor"
+}
+
+type MarkdownMathCodeBlockOptions struct {
+	ParseInlineDollar        bool
+	ParseInlineParentheses   bool
+	ParseBlockDollar         bool
+	ParseBlockSquareBrackets bool
+}
+
 // Markdown settings
 var Markdown = struct {
-	EnableHardLineBreakInComments  bool
-	EnableHardLineBreakInDocuments bool
-	CustomURLSchemes               []string `ini:"CUSTOM_URL_SCHEMES"`
-	FileExtensions                 []string
-	EnableMath                     bool
+	RenderOptionsComment  MarkdownRenderOptions `ini:"-"`
+	RenderOptionsWiki     MarkdownRenderOptions `ini:"-"`
+	RenderOptionsRepoFile MarkdownRenderOptions `ini:"-"`
+
+	CustomURLSchemes []string `ini:"CUSTOM_URL_SCHEMES"` // Actually it is a "markup" option because it is used in "post processor"
+	FileExtensions   []string
+
+	EnableMath             bool
+	MathCodeBlockDetection []string
+	MathCodeBlockOptions   MarkdownMathCodeBlockOptions `ini:"-"`
 }{
-	EnableHardLineBreakInComments:  true,
-	EnableHardLineBreakInDocuments: false,
-	FileExtensions:                 strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","),
-	EnableMath:                     true,
+	FileExtensions: strings.Split(".md,.markdown,.mdown,.mkd,.livemd", ","),
+	EnableMath:     true,
 }

 // MarkupRenderer defines the external parser configured in ini
@ -60,6 +76,56 @@ type MarkupSanitizerRule struct {

 func loadMarkupFrom(rootCfg ConfigProvider) {
 	mustMapSetting(rootCfg, "markdown", &Markdown)
+	const none = "none"
+
+	const renderOptionShortIssuePattern = "short-issue-pattern"
+	const renderOptionNewLineHardBreak = "new-line-hard-break"
+	cfgMarkdown := rootCfg.Section("markdown")
+	parseMarkdownRenderOptions := func(key string, defaults []string) (ret MarkdownRenderOptions) {
+		options := cfgMarkdown.Key(key).Strings(",")
+		options = util.IfEmpty(options, defaults)
+		for _, opt := range options {
+			switch opt {
+			case renderOptionShortIssuePattern:
+				ret.ShortIssuePattern = true
+			case renderOptionNewLineHardBreak:
+				ret.NewLineHardBreak = true
+			case none:
+				ret = MarkdownRenderOptions{}
+			case "":
+			default:
+				log.Error("Unknown markdown render option in %s: %s", key, opt)
+			}
+		}
+		return ret
+	}
+	Markdown.RenderOptionsComment = parseMarkdownRenderOptions("RENDER_OPTIONS_COMMENT", []string{renderOptionShortIssuePattern, renderOptionNewLineHardBreak})
+	Markdown.RenderOptionsWiki = parseMarkdownRenderOptions("RENDER_OPTIONS_WIKI", []string{renderOptionShortIssuePattern})
+	Markdown.RenderOptionsRepoFile = parseMarkdownRenderOptions("RENDER_OPTIONS_REPO_FILE", nil)
+
+	const mathCodeInlineDollar = "inline-dollar"
+	const mathCodeInlineParentheses = "inline-parentheses"
+	const mathCodeBlockDollar = "block-dollar"
+	const mathCodeBlockSquareBrackets = "block-square-brackets"
+	Markdown.MathCodeBlockDetection = util.IfEmpty(Markdown.MathCodeBlockDetection, []string{mathCodeInlineDollar, mathCodeBlockDollar})
+	Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{}
+	for _, s := range Markdown.MathCodeBlockDetection {
+		switch s {
+		case mathCodeInlineDollar:
+			Markdown.MathCodeBlockOptions.ParseInlineDollar = true
+		case mathCodeInlineParentheses:
+			Markdown.MathCodeBlockOptions.ParseInlineParentheses = true
+		case mathCodeBlockDollar:
+			Markdown.MathCodeBlockOptions.ParseBlockDollar = true
+		case mathCodeBlockSquareBrackets:
+			Markdown.MathCodeBlockOptions.ParseBlockSquareBrackets = true
+		case none:
+			Markdown.MathCodeBlockOptions = MarkdownMathCodeBlockOptions{}
+		case "":
+		default:
+			log.Error("Unknown math code block detection option: %s", s)
+		}
+	}

 	MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(5000)
 	ExternalMarkupRenderers = make([]*MarkupRenderer, 0, 10)
--- a/modules/setting/markup_test.go
+++ b/modules/setting/markup_test.go
@ -0,0 +1,51 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package setting
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadMarkup(t *testing.T) {
+	cfg, _ := NewConfigProviderFromData(``)
+	loadMarkupFrom(cfg)
+	assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseBlockDollar: true}, Markdown.MathCodeBlockOptions)
+	assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsComment)
+	assert.Equal(t, MarkdownRenderOptions{ShortIssuePattern: true}, Markdown.RenderOptionsWiki)
+	assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsRepoFile)
+
+	t.Run("Math", func(t *testing.T) {
+		cfg, _ = NewConfigProviderFromData(`
+[markdown]
+MATH_CODE_BLOCK_DETECTION = none
+`)
+		loadMarkupFrom(cfg)
+		assert.Equal(t, MarkdownMathCodeBlockOptions{}, Markdown.MathCodeBlockOptions)
+
+		cfg, _ = NewConfigProviderFromData(`
+[markdown]
+MATH_CODE_BLOCK_DETECTION = inline-dollar, inline-parentheses, block-dollar, block-square-brackets
+`)
+		loadMarkupFrom(cfg)
+		assert.Equal(t, MarkdownMathCodeBlockOptions{ParseInlineDollar: true, ParseInlineParentheses: true, ParseBlockDollar: true, ParseBlockSquareBrackets: true}, Markdown.MathCodeBlockOptions)
+	})
+
+	t.Run("Render", func(t *testing.T) {
+		cfg, _ = NewConfigProviderFromData(`
+[markdown]
+RENDER_OPTIONS_COMMENT = none
+`)
+		loadMarkupFrom(cfg)
+		assert.Equal(t, MarkdownRenderOptions{}, Markdown.RenderOptionsComment)
+
+		cfg, _ = NewConfigProviderFromData(`
+[markdown]
+RENDER_OPTIONS_REPO_FILE = short-issue-pattern, new-line-hard-break
+`)
+		loadMarkupFrom(cfg)
+		assert.Equal(t, MarkdownRenderOptions{NewLineHardBreak: true, ShortIssuePattern: true}, Markdown.RenderOptionsRepoFile)
+	})
+}
--- a/modules/templates/util_render_test.go
+++ b/modules/templates/util_render_test.go
@ -51,7 +51,7 @@ var testMetas = map[string]string{
 	"user":                         "user13",
 	"repo":                         "repo11",
 	"repoPath":                     "../../tests/gitea-repositories-meta/user13/repo11.git/",
-	"markdownLineBreakStyle":       "comment",
+	"markdownNewLineHardBreak":     "true",
 	"markupAllowShortIssuePattern": "true",
 }

--- a/modules/util/util.go
+++ b/modules/util/util.go
@ -219,6 +219,13 @@ func IfZero[T comparable](v, def T) T {
 	return v
 }

+func IfEmpty[T any](v, def []T) []T {
+	if len(v) == 0 {
+		return def
+	}
+	return v
+}
+
 // OptionalArg helps the "optional argument" in Golang:
 //
 //	func foo(optArg ...int) { return OptionalArg(optArg) }