[release-branch.go1.24] cmd/compile: fix out of memory when inlining closure

CL 629195 strongly favor closure inlining, allowing closures to be
inlined more aggressively.

However, if the closure body contains a call to a function, which itself
is one of the call arguments, it causes the infinite inlining.

Fixing this by prevent this kind of functions from being inlinable.

Fixes #72067

Change-Id: I5fb5723a819b1e2c5aadb57c1023ec84ca9fa53c
Reviewed-on: https://go-review.googlesource.com/c/go/+/654195
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-on: https://go-review.googlesource.com/c/go/+/654517
Commit-Queue: Junyang Shao <shaojunyang@google.com>
Auto-Submit: Junyang Shao <shaojunyang@google.com>
Reviewed-by: Junyang Shao <shaojunyang@google.com>
Reviewed-by: Jorropo <jorropo.pgm@gmail.com>

src/cmd/compile/internal/inline/inl.go

@@ -1040,6 +1040,28 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
 			return false, 0, false
 		}
 	}
+	do := func(fn *ir.Func) bool {
+		// Can't recursively inline a function if the function body contains
+		// a call to a function f, which the function f is one of the call arguments.
+		return ir.Any(fn, func(node ir.Node) bool {
+			if call, ok := node.(*ir.CallExpr); ok {
+				for _, arg := range call.Args {
+					if call.Fun == arg {
+						return true
+					}
+				}
+			}
+			return false
+		})
+	}
+	for _, fn := range []*ir.Func{callerfn, callee} {
+		if do(fn) {
+			if log && logopt.Enabled() {
+				logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to function: %s", ir.FuncName(fn)))
+			}
+			return false, 0, false
+		}
+	}
 
 	if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
 		// Runtime package must not be instrumented.

test/fixedbugs/issue72063.go

@@ -0,0 +1,40 @@
+// run
+
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "fmt"
+
+// Y is the Y-combinator based on https://dreamsongs.com/Files/WhyOfY.pdf
+func Y[Endo ~func(RecFct) RecFct, RecFct ~func(T) R, T, R any](f Endo) RecFct {
+
+	type internal[RecFct ~func(T) R, T, R any] func(internal[RecFct, T, R]) RecFct
+
+	g := func(h internal[RecFct, T, R]) RecFct {
+		return func(t T) R {
+			return f(h(h))(t)
+		}
+	}
+	return g(g)
+}
+
+func main() {
+
+	fct := Y(func(r func(int) int) func(int) int {
+		return func(n int) int {
+			if n <= 0 {
+				return 1
+			}
+			return n * r(n-1)
+		}
+	})
+
+	want := 3628800
+	if got := fct(10); got != want {
+		msg := fmt.Sprintf("unexpected result, got: %d, want: %d", got, want)
+		panic(msg)
+	}
+}