crypto/x509: update iOS root certs.

Apple changed the format of its support page, so we need to
restructure the HTML parser. The HTML table is now parsed using
regular expressions, and certificates are then found in macOS
trust store by their fingerprint.

Fixes #22181

Change-Id: I29e7a40d37770bb005d728f1832299c528691f7e
Reviewed-on: https://go-review.googlesource.com/77252
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Adam Langley <agl@golang.org>

src/crypto/x509/root_darwin_arm_gen.go

@@ -18,16 +18,18 @@ package main
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"crypto/x509"
+	"encoding/hex"
 	"encoding/pem"
 	"flag"
 	"fmt"
 	"go/format"
 	"io/ioutil"
 	"log"
-	"math/big"
 	"net/http"
 	"os/exec"
+	"regexp"
 	"strings"
 )
 
@@ -41,7 +43,7 @@ func main() {
 
 	buf := new(bytes.Buffer)
 
-	fmt.Fprintf(buf, "// Created by root_darwin_arm_gen --output %s; DO NOT EDIT\n", *output)
+	fmt.Fprintf(buf, "// Code generated by root_darwin_arm_gen --output %s; DO NOT EDIT.\n", *output)
 	fmt.Fprintf(buf, "%s", header)
 
 	fmt.Fprintf(buf, "const systemRootsPEM = `\n")
@@ -78,36 +80,22 @@ func selectCerts() ([]*x509.Certificate, error) {
 
 	var certs []*x509.Certificate
 	for _, id := range ids {
-		sn, ok := big.NewInt(0).SetString(id.serialNumber, 0) // 0x prefix selects hex
+		if c, ok := scerts[id.fingerprint]; ok {
-		if !ok {
+			certs = append(certs, c)
-			return nil, fmt.Errorf("invalid serial number: %q", id.serialNumber)
+		} else {
-		}
+			fmt.Printf("WARNING: cannot find certificate: %s (fingerprint: %s)\n", id.name, id.fingerprint)
-		ski, ok := big.NewInt(0).SetString(id.subjectKeyID, 0)
-		if !ok {
-			return nil, fmt.Errorf("invalid Subject Key ID: %q", id.subjectKeyID)
-		}
-
-		for _, cert := range scerts {
-			if sn.Cmp(cert.SerialNumber) != 0 {
-				continue
-			}
-			cski := big.NewInt(0).SetBytes(cert.SubjectKeyId)
-			if ski.Cmp(cski) != 0 {
-				continue
-			}
-			certs = append(certs, cert)
-			break
 		}
 	}
 	return certs, nil
 }
 
-func sysCerts() (certs []*x509.Certificate, err error) {
+func sysCerts() (certs map[string]*x509.Certificate, err error) {
 	cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", "/System/Library/Keychains/SystemRootCertificates.keychain")
 	data, err := cmd.Output()
 	if err != nil {
 		return nil, err
 	}
+	certs = make(map[string]*x509.Certificate)
 	for len(data) > 0 {
 		var block *pem.Block
 		block, data = pem.Decode(data)
@@ -122,19 +110,23 @@ func sysCerts() (certs []*x509.Certificate, err error) {
 		if err != nil {
 			continue
 		}
-		certs = append(certs, cert)
+
+		fingerprint := sha256.Sum256(cert.Raw)
+		certs[hex.EncodeToString(fingerprint[:])] = cert
 	}
 	return certs, nil
 }
 
 type certID struct {
-	serialNumber string
+	name        string
-	subjectKeyID string
+	fingerprint string
 }
 
 // fetchCertIDs fetches IDs of iOS X509 certificates from apple.com.
 func fetchCertIDs() ([]certID, error) {
-	resp, err := http.Get("https://support.apple.com/en-us/HT204132")
+	// Download the iOS 11 support page. The index for all iOS versions is here:
+	// https://support.apple.com/en-us/HT204132
+	resp, err := http.Get("https://support.apple.com/en-us/HT208125")
 	if err != nil {
 		return nil, err
 	}
@@ -144,31 +136,33 @@ func fetchCertIDs() ([]certID, error) {
 		return nil, err
 	}
 	text := string(body)
-	text = text[strings.Index(text, "<section id=trusted"):]
+	text = text[strings.Index(text, "<div id=trusted"):]
-	text = text[:strings.Index(text, "</section>")]
+	text = text[:strings.Index(text, "</div>")]
 
-	lines := strings.Split(text, "\n")
 	var ids []certID
-	var id certID
+	cols := make(map[string]int)
-	for i, ln := range lines {
+	for i, rowmatch := range regexp.MustCompile("(?s)<tr>(.*?)</tr>").FindAllStringSubmatch(text, -1) {
-		if i == len(lines)-1 {
+		row := rowmatch[1]
-			break
+		if i == 0 {
-		}
+			// Parse table header row to extract column names
-		const sn = "Serial Number:"
+			for i, match := range regexp.MustCompile("(?s)<th>(.*?)</th>").FindAllStringSubmatch(row, -1) {
-		if ln == sn {
+				cols[match[1]] = i
-			id.serialNumber = "0x" + strings.Replace(strings.TrimSpace(lines[i+1]), ":", "", -1)
+			}
 			continue
 		}
-		if strings.HasPrefix(ln, sn) {
+
-			// extract hex value from parentheses.
+		values := regexp.MustCompile("(?s)<td>(.*?)</td>").FindAllStringSubmatch(row, -1)
-			id.serialNumber = ln[strings.Index(ln, "(")+1 : len(ln)-1]
+		name := values[cols["Certificate name"]][1]
-			continue
+		fingerprint := values[cols["Fingerprint (SHA-256)"]][1]
-		}
+		fingerprint = strings.Replace(fingerprint, "<br>", "", -1)
-		if strings.TrimSpace(ln) == "X509v3 Subject Key Identifier:" {
+		fingerprint = strings.Replace(fingerprint, "\n", "", -1)
-			id.subjectKeyID = "0x" + strings.Replace(strings.TrimSpace(lines[i+1]), ":", "", -1)
+		fingerprint = strings.Replace(fingerprint, " ", "", -1)
-			ids = append(ids, id)
+		fingerprint = strings.ToLower(fingerprint)
-			id = certID{}
+
-		}
+		ids = append(ids, certID{
+			name:        name,
+			fingerprint: fingerprint,
+		})
 	}
 	return ids, nil
 }
@@ -180,7 +174,7 @@ const header = `
 
 // +build cgo
 // +build darwin
-// +build arm arm64
+// +build arm arm64 ios
 
 package x509