text/template: add CommentNode to template parse tree

Fixes #34652

Change-Id: Icf6e3eda593fed826736f34f95a9d66f5450cc98
Reviewed-on: https://go-review.googlesource.com/c/go/+/229398
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>

api/next.txt

@@ -3,3 +3,17 @@ pkg unicode, var Chorasmian *RangeTable
 pkg unicode, var Dives_Akuru *RangeTable
 pkg unicode, var Khitan_Small_Script *RangeTable
 pkg unicode, var Yezidi *RangeTable
+pkg text/template/parse, const NodeComment = 20
+pkg text/template/parse, const NodeComment NodeType
+pkg text/template/parse, const ParseComments = 1
+pkg text/template/parse, const ParseComments Mode
+pkg text/template/parse, method (*CommentNode) Copy() Node
+pkg text/template/parse, method (*CommentNode) String() string
+pkg text/template/parse, method (CommentNode) Position() Pos
+pkg text/template/parse, method (CommentNode) Type() NodeType
+pkg text/template/parse, type CommentNode struct
+pkg text/template/parse, type CommentNode struct, Text string
+pkg text/template/parse, type CommentNode struct, embedded NodeType
+pkg text/template/parse, type CommentNode struct, embedded Pos
+pkg text/template/parse, type Mode uint
+pkg text/template/parse, type Tree struct, Mode Mode

doc/go1.16.html

@@ -121,6 +121,16 @@ Do not send CLs removing the interior tags from such phrases.
   with <code>"use of closed network connection"</code>.
 </p>
 
+
+<h3 id="text/template/parse"><a href="/pkg/text/template/parse/">text/template/parse</a></h3>
+
+<p><!-- CL 229398, golang.org/issue/34652 -->
+  A new <a href="/pkg/text/template/parse/#CommentNode"><code>CommentNode</code></a>
+  was added to the parse tree. The <a href="/pkg/text/template/parse/#Mode"><code>Mode</code></a>
+  field in the <code>parse.Tree</code> enables access to it.
+</p>
+<!-- text/template/parse -->
+
 <h3 id="unicode"><a href="/pkg/unicode/">unicode</a></h3>
 
 <p><!-- CL 248765 -->

src/html/template/escape.go

@@ -124,6 +124,8 @@ func (e *escaper) escape(c context, n parse.Node) context {
 	switch n := n.(type) {
 	case *parse.ActionNode:
 		return e.escapeAction(c, n)
+	case *parse.CommentNode:
+		return c
 	case *parse.IfNode:
 		return e.escapeBranch(c, &n.BranchNode, "if")
 	case *parse.ListNode:

src/html/template/template_test.go

@@ -10,6 +10,7 @@ import (
 	. "html/template"
 	"strings"
 	"testing"
+	"text/template/parse"
 )
 
 func TestTemplateClone(t *testing.T) {
@@ -160,6 +161,21 @@ func TestStringsInScriptsWithJsonContentTypeAreCorrectlyEscaped(t *testing.T) {
 	}
 }
 
+func TestSkipEscapeComments(t *testing.T) {
+	c := newTestCase(t)
+	tr := parse.New("root")
+	tr.Mode = parse.ParseComments
+	newT, err := tr.Parse("{{/* A comment */}}{{ 1 }}{{/* Another comment */}}", "", "", make(map[string]*parse.Tree))
+	if err != nil {
+		t.Fatalf("Cannot parse template text: %v", err)
+	}
+	c.root, err = c.root.AddParseTree("root", newT)
+	if err != nil {
+		t.Fatalf("Cannot add parse tree to template: %v", err)
+	}
+	c.mustExecute(c.root, nil, "1")
+}
+
 type testCase struct {
 	t    *testing.T
 	root *Template

src/text/template/exec.go

@@ -256,6 +256,7 @@ func (s *state) walk(dot reflect.Value, node parse.Node) {
 		if len(node.Pipe.Decl) == 0 {
 			s.printValue(node, val)
 		}
+	case *parse.CommentNode:
 	case *parse.IfNode:
 		s.walkIfOrWith(parse.NodeIf, dot, node.Pipe, node.List, node.ElseList)
 	case *parse.ListNode:

src/text/template/parse/lex.go

@@ -41,6 +41,7 @@ const (
 	itemBool                         // boolean constant
 	itemChar                         // printable ASCII character; grab bag for comma etc.
 	itemCharConstant                 // character constant
+	itemComment                      // comment text
 	itemComplex                      // complex constant (1+2i); imaginary is just a number
 	itemAssign                       // equals ('=') introducing an assignment
 	itemDeclare                      // colon-equals (':=') introducing a declaration
@@ -112,6 +113,7 @@ type lexer struct {
 	leftDelim      string    // start of action
 	rightDelim     string    // end of action
 	trimRightDelim string    // end of action with trim marker
+	emitComment    bool      // emit itemComment tokens.
 	pos            Pos       // current position in the input
 	start          Pos       // start position of this item
 	width          Pos       // width of last rune read from input
@@ -203,7 +205,7 @@ func (l *lexer) drain() {
 }
 
 // lex creates a new scanner for the input string.
-func lex(name, input, left, right string) *lexer {
+func lex(name, input, left, right string, emitComment bool) *lexer {
 	if left == "" {
 		left = leftDelim
 	}
@@ -216,6 +218,7 @@ func lex(name, input, left, right string) *lexer {
 		leftDelim:      left,
 		rightDelim:     right,
 		trimRightDelim: rightTrimMarker + right,
+		emitComment:    emitComment,
 		items:          make(chan item),
 		line:           1,
 		startLine:      1,
@@ -323,6 +326,9 @@ func lexComment(l *lexer) stateFn {
 	if !delim {
 		return l.errorf("comment ends before closing delimiter")
 	}
+	if l.emitComment {
+		l.emit(itemComment)
+	}
 	if trimSpace {
 		l.pos += trimMarkerLen
 	}

src/text/template/parse/lex_test.go

@@ -15,6 +15,7 @@ var itemName = map[itemType]string{
 	itemBool:         "bool",
 	itemChar:         "char",
 	itemCharConstant: "charconst",
+	itemComment:      "comment",
 	itemComplex:      "complex",
 	itemDeclare:      ":=",
 	itemEOF:          "EOF",
@@ -90,6 +91,7 @@ var lexTests = []lexTest{
 	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
 	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
 		mkItem(itemText, "hello-"),
+		mkItem(itemComment, "/* this is a comment */"),
 		mkItem(itemText, "-world"),
 		tEOF,
 	}},
@@ -311,6 +313,7 @@ var lexTests = []lexTest{
 	}},
 	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
 		mkItem(itemText, "hello-"),
+		mkItem(itemComment, "/* hello */"),
 		mkItem(itemText, "-world"),
 		tEOF,
 	}},
@@ -389,7 +392,7 @@ var lexTests = []lexTest{
 
 // collect gathers the emitted items into a slice.
 func collect(t *lexTest, left, right string) (items []item) {
-	l := lex(t.name, t.input, left, right)
+	l := lex(t.name, t.input, left, right, true)
 	for {
 		item := l.nextItem()
 		items = append(items, item)
@@ -529,7 +532,7 @@ func TestPos(t *testing.T) {
 func TestShutdown(t *testing.T) {
 	// We need to duplicate template.Parse here to hold on to the lexer.
 	const text = "erroneous{{define}}{{else}}1234"
-	lexer := lex("foo", text, "{{", "}}")
+	lexer := lex("foo", text, "{{", "}}", false)
 	_, err := New("root").parseLexer(lexer)
 	if err == nil {
 		t.Fatalf("expected error")

src/text/template/parse/node.go

@@ -70,6 +70,7 @@ const (
 	NodeTemplate                   // A template invocation action.
 	NodeVariable                   // A $ variable.
 	NodeWith                       // A with action.
+	NodeComment                    // A comment.
 )
 
 // Nodes.
@@ -149,6 +150,38 @@ func (t *TextNode) Copy() Node {
 	return &TextNode{tr: t.tr, NodeType: NodeText, Pos: t.Pos, Text: append([]byte{}, t.Text...)}
 }
 
+// CommentNode holds a comment.
+type CommentNode struct {
+	NodeType
+	Pos
+	tr   *Tree
+	Text string // Comment text.
+}
+
+func (t *Tree) newComment(pos Pos, text string) *CommentNode {
+	return &CommentNode{tr: t, NodeType: NodeComment, Pos: pos, Text: text}
+}
+
+func (c *CommentNode) String() string {
+	var sb strings.Builder
+	c.writeTo(&sb)
+	return sb.String()
+}
+
+func (c *CommentNode) writeTo(sb *strings.Builder) {
+	sb.WriteString("{{")
+	sb.WriteString(c.Text)
+	sb.WriteString("}}")
+}
+
+func (c *CommentNode) tree() *Tree {
+	return c.tr
+}
+
+func (c *CommentNode) Copy() Node {
+	return &CommentNode{tr: c.tr, NodeType: NodeComment, Pos: c.Pos, Text: c.Text}
+}
+
 // PipeNode holds a pipeline with optional declaration
 type PipeNode struct {
 	NodeType

src/text/template/parse/parse.go

@@ -21,6 +21,7 @@ type Tree struct {
 	Name      string    // name of the template represented by the tree.
 	ParseName string    // name of the top-level template during parsing, for error messages.
 	Root      *ListNode // top-level root of the tree.
+	Mode      Mode      // parsing mode.
 	text      string    // text parsed to create the template (or its parent)
 	// Parsing only; cleared after parse.
 	funcs     []map[string]interface{}
@@ -29,8 +30,16 @@ type Tree struct {
 	peekCount int
 	vars      []string // variables defined at the moment.
 	treeSet   map[string]*Tree
+	mode      Mode
 }
 
+// A mode value is a set of flags (or 0). Modes control parser behavior.
+type Mode uint
+
+const (
+	ParseComments Mode = 1 << iota // parse comments and add them to AST
+)
+
 // Copy returns a copy of the Tree. Any parsing state is discarded.
 func (t *Tree) Copy() *Tree {
 	if t == nil {
@@ -220,7 +229,8 @@ func (t *Tree) stopParse() {
 func (t *Tree) Parse(text, leftDelim, rightDelim string, treeSet map[string]*Tree, funcs ...map[string]interface{}) (tree *Tree, err error) {
 	defer t.recover(&err)
 	t.ParseName = t.Name
-	t.startParse(funcs, lex(t.Name, text, leftDelim, rightDelim), treeSet)
+	emitComment := t.Mode&ParseComments != 0
+	t.startParse(funcs, lex(t.Name, text, leftDelim, rightDelim, emitComment), treeSet)
 	t.text = text
 	t.parse()
 	t.add()
@@ -240,12 +250,14 @@ func (t *Tree) add() {
 	}
 }
 
-// IsEmptyTree reports whether this tree (node) is empty of everything but space.
+// IsEmptyTree reports whether this tree (node) is empty of everything but space or comments.
 func IsEmptyTree(n Node) bool {
 	switch n := n.(type) {
 	case nil:
 		return true
 	case *ActionNode:
+	case *CommentNode:
+		return true
 	case *IfNode:
 	case *ListNode:
 		for _, node := range n.Nodes {
@@ -276,6 +288,7 @@ func (t *Tree) parse() {
 			if t.nextNonSpace().typ == itemDefine {
 				newT := New("definition") // name will be updated once we know it.
 				newT.text = t.text
+				newT.Mode = t.Mode
 				newT.ParseName = t.ParseName
 				newT.startParse(t.funcs, t.lex, t.treeSet)
 				newT.parseDefinition()
@@ -331,13 +344,15 @@ func (t *Tree) itemList() (list *ListNode, next Node) {
 }
 
 // textOrAction:
-//	text | action
+//	text | comment | action
 func (t *Tree) textOrAction() Node {
 	switch token := t.nextNonSpace(); token.typ {
 	case itemText:
 		return t.newText(token.pos, token.val)
 	case itemLeftDelim:
 		return t.action()
+	case itemComment:
+		return t.newComment(token.pos, token.val)
 	default:
 		t.unexpected(token, "input")
 	}
@@ -539,6 +554,7 @@ func (t *Tree) blockControl() Node {
 
 	block := New(name) // name will be updated once we know it.
 	block.text = t.text
+	block.Mode = t.Mode
 	block.ParseName = t.ParseName
 	block.startParse(t.funcs, t.lex, t.treeSet)
 	var end Node

src/text/template/parse/parse_test.go

@@ -348,6 +348,30 @@ func TestParseCopy(t *testing.T) {
 	testParse(true, t)
 }
 
+func TestParseWithComments(t *testing.T) {
+	textFormat = "%q"
+	defer func() { textFormat = "%s" }()
+	tests := [...]parseTest{
+		{"comment", "{{/*\n\n\n*/}}", noError, "{{/*\n\n\n*/}}"},
+		{"comment trim left", "x \r\n\t{{- /* hi */}}", noError, `"x"{{/* hi */}}`},
+		{"comment trim right", "{{/* hi */ -}}\n\n\ty", noError, `{{/* hi */}}"y"`},
+		{"comment trim left and right", "x \r\n\t{{- /* */ -}}\n\n\ty", noError, `"x"{{/* */}}"y"`},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tr := New(test.name)
+			tr.Mode = ParseComments
+			tmpl, err := tr.Parse(test.input, "", "", make(map[string]*Tree))
+			if err != nil {
+				t.Errorf("%q: expected error; got none", test.name)
+			}
+			if result := tmpl.Root.String(); result != test.result {
+				t.Errorf("%s=(%q): got\n\t%v\nexpected\n\t%v", test.name, test.input, result, test.result)
+			}
+		})
+	}
+}
+
 type isEmptyTest struct {
 	name  string
 	input string
@@ -358,6 +382,7 @@ var isEmptyTests = []isEmptyTest{
 	{"empty", ``, true},
 	{"nonempty", `hello`, false},
 	{"spaces only", " \t\n \t\n", true},
+	{"comment only", "{{/* comment */}}", true},
 	{"definition", `{{define "x"}}something{{end}}`, true},
 	{"definitions and space", "{{define `x`}}something{{end}}\n\n{{define `y`}}something{{end}}\n\n", true},
 	{"definitions and text", "{{define `x`}}something{{end}}\nx\n{{define `y`}}something{{end}}\ny\n", false},