[release-branch.go1.11] cmd/compile: prevent overflow in walkinrange

In the compiler frontend, walkinrange indiscriminately calls Int64()
on const CTINT nodes, even though Int64's return value is undefined
for anything over 2⁶³ (in practise, it'll return a negative number).

This causes the introduction of bad constants during rewrites of
unsigned expressions, which make the compiler reject valid Go
programs.

This change introduces a preliminary check that Int64() is safe to
call on the consts on hand. If it isn't, walkinrange exits without
doing any rewrite.

Fixes #27246

Change-Id: I2017073cae65468a521ff3262d4ea8ab0d7098d9
Reviewed-on: https://go-review.googlesource.com/130735
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>
(cherry picked from commit 42cc4ca30a)
Reviewed-on: https://go-review.googlesource.com/131596
Reviewed-by: Robert Griesemer <gri@golang.org>
Run-TryBot: Andrew Bonventre <andybons@golang.org>

src/cmd/compile/internal/gc/const.go

@@ -121,6 +121,17 @@ func (n *Node) Int64() int64 {
 	return n.Val().U.(*Mpint).Int64()
 }
 
+// CanInt64 reports whether it is safe to call Int64() on n.
+func (n *Node) CanInt64() bool {
+	if !Isconst(n, CTINT) {
+		return false
+	}
+
+	// if the value inside n cannot be represented as an int64, the
+	// return value of Int64 is undefined
+	return n.Val().U.(*Mpint).CmpInt64(n.Int64()) == 0
+}
+
 // Bool returns n as a bool.
 // n must be a boolean constant.
 func (n *Node) Bool() bool {

src/cmd/compile/internal/gc/walk.go

@@ -3701,6 +3701,12 @@ func walkinrange(n *Node, init *Nodes) *Node {
 		return n
 	}
 
+	// Ensure that Int64() does not overflow on a and c (it'll happen
+	// for any const above 2**63; see issue #27143).
+	if !a.CanInt64() || !c.CanInt64() {
+		return n
+	}
+
 	if opl == OLT {
 		// We have a < b && ...
 		// We need a ≤ b && ... to safely use unsigned comparison tricks.

test/fixedbugs/issue27143.go

@@ -0,0 +1,17 @@
+// compile
+
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Issue 27143: cmd/compile: erroneous application of walkinrange
+// optimization for const over 2**63
+
+package p
+
+var c uint64
+
+var b1 bool = 0x7fffffffffffffff < c && c < 0x8000000000000000
+var b2 bool = c < 0x8000000000000000 && 0x7fffffffffffffff < c
+var b3 bool = 0x8000000000000000 < c && c < 0x8000000000000001
+var b4 bool = c < 0x8000000000000001 && 0x8000000000000000 < c