crypto/rsa: ensure that generating toy RSA keys doesn't loop.

If there are too few primes of the given length then it can be
impossible to generate an RSA key with n distinct primes.

This change approximates the expected number of candidate primes and
causes key generation to return an error if it's unlikely to succeed.

Fixes #16596.

Change-Id: I53b60d0cb90e2d0e6f0662befa64d13f24af51a7
Reviewed-on: https://go-review.googlesource.com/28969
Reviewed-by: Minux Ma <minux@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Minux Ma <minux@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/crypto/rsa/rsa.go

@@ -27,6 +27,7 @@ import (
 	"errors"
 	"hash"
 	"io"
+	"math"
 	"math/big"
 )
 
@@ -214,6 +215,21 @@ func GenerateMultiPrimeKey(random io.Reader, nprimes int, bits int) (*PrivateKey
 		return nil, errors.New("crypto/rsa: GenerateMultiPrimeKey: nprimes must be >= 2")
 	}
 
+	if bits < 64 {
+		primeLimit := float64(uint64(1) << uint(bits/nprimes))
+		// pi approximates the number of primes less than primeLimit
+		pi := primeLimit / (math.Log(primeLimit) - 1)
+		// Generated primes start with 11 (in binary) so we can only
+		// use a quarter of them.
+		pi /= 4
+		// Use a factor of two to ensure that key generation terminates
+		// in a reasonable amount of time.
+		pi /= 2
+		if pi <= float64(nprimes) {
+			return nil, errors.New("crypto/rsa: too few primes of given length to generate an RSA key")
+		}
+	}
+
 	primes := make([]*big.Int, nprimes)
 
 NextSetOfPrimes:

src/crypto/rsa/rsa_test.go

@@ -73,6 +73,17 @@ func TestNPrimeKeyGeneration(t *testing.T) {
 	}
 }
 
+func TestImpossibleKeyGeneration(t *testing.T) {
+	// This test ensures that trying to generate toy RSA keys doesn't enter
+	// an infinite loop.
+	for i := 0; i < 32; i++ {
+		GenerateKey(rand.Reader, i)
+		GenerateMultiPrimeKey(rand.Reader, 3, i)
+		GenerateMultiPrimeKey(rand.Reader, 4, i)
+		GenerateMultiPrimeKey(rand.Reader, 5, i)
+	}
+}
+
 func TestGnuTLSKey(t *testing.T) {
 	// This is a key generated by `certtool --generate-privkey --bits 128`.
 	// It's such that de ≢ 1 mod φ(n), but is congruent mod the order of