cmd/go: add user provided auth mode for GOAUTH

This CL adds support for a custom authenticator as a valid GOAUTH command.
This follows the specification in
https://go.dev/issue/26232#issuecomment-461525141

For #26232

Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest,gotip-windows-amd64-longtest
Change-Id: Id1d4b309f11eb9c7ce14793021a9d8caf3b192ff
Reviewed-on: https://go-review.googlesource.com/c/go/+/605298
Auto-Submit: Sam Thanawalla <samthanawalla@google.com>
Reviewed-by: Michael Matloob <matloob@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

src/cmd/go/alldocs.go

@@ -43,6 +43,7 @@
 //	cache           build and test caching
 //	environment     environment variables
 //	filetype        file types
+//	goauth          GOAUTH environment variable
 //	go.mod          the go.mod file
 //	gopath          GOPATH environment variable
 //	goproxy         module proxy protocol
@@ -2275,12 +2276,8 @@
 //		The architecture, or processor, for which to compile code.
 //		Examples are amd64, 386, arm, ppc64.
 //	GOAUTH
-//		A semicolon-separated list of authentication commands for go-import and
+//		Controls authentication for go-import and HTTPS module mirror interactions.
-//		HTTPS module mirror interactions. Currently supports
+//		See 'go help goauth'.
-//		"off" (disables authentication),
-//		"netrc" (uses credentials from NETRC or the .netrc file in your home directory),
-//		"git dir" (runs 'git credential fill' in dir and uses its credentials).
-//		The default is netrc.
 //	GOBIN
 //		The directory where 'go install' will install a command.
 //	GOCACHE
@@ -2511,6 +2508,75 @@
 // line comment. See the go/build package documentation for
 // more details.
 //
+// # GOAUTH environment variable
+//
+// GOAUTH is a semicolon-separated list of authentication commands for go-import and
+// HTTPS module mirror interactions. The default is netrc.
+//
+// The supported authentication commands are:
+//
+// off
+//
+//	Disables authentication.
+//
+// netrc
+//
+//	Uses credentials from NETRC or the .netrc file in your home directory.
+//
+// git dir
+//
+//	Runs 'git credential fill' in dir and uses its credentials. The
+//	go command will run 'git credential approve/reject' to update
+//	the credential helper's cache.
+//
+// command
+//
+//	Executes the given command (a space-separated argument list) and attaches
+//	the provided headers to HTTPS requests.
+//	The command must produce output in the following format:
+//		Response      = { CredentialSet } .
+//		CredentialSet = URLLine { URLLine } BlankLine { HeaderLine } BlankLine .
+//		URLLine       = /* URL that starts with "https://" */ '\n' .
+//		HeaderLine    = /* HTTP Request header */ '\n' .
+//		BlankLine     = '\n' .
+//
+//	Example:
+//		https://example.com/
+//		https://example.net/api/
+//
+//		Authorization: Basic <token>
+//
+//		https://another-example.org/
+//
+//		Example: Data
+//
+//	If the server responds with any 4xx code, the go command will write the
+//	following to the programs' stdin:
+//		Response      = StatusLine { HeaderLine } BlankLine .
+//		StatusLine    = Protocol Space Status '\n' .
+//		Protocol      = /* HTTP protocol */ .
+//		Space         = ' ' .
+//		Status        = /* HTTP status code */ .
+//		BlankLine     = '\n' .
+//		HeaderLine    = /* HTTP Response's header */ '\n' .
+//
+//	Example:
+//		HTTP/1.1 401 Unauthorized
+//		Content-Length: 19
+//		Content-Type: text/plain; charset=utf-8
+//		Date: Thu, 07 Nov 2024 18:43:09 GMT
+//
+//	Note: at least for HTTP 1.1, the contents written to stdin can be parsed
+//	as an HTTP response.
+//
+// Before the first HTTPS fetch, the go command will invoke each GOAUTH
+// command in the list with no additional arguments and no input.
+// If the server responds with any 4xx code, the go command will invoke the
+// GOAUTH commands again with the URL as an additional command-line argument
+// and the HTTP Response to the program's stdin.
+// If the server responds with an error again, the fetch fails: a URL-specific
+// GOAUTH will only be attempted once per fetch.
+//
 // # The go.mod file
 //
 // A module version is defined by a tree of source files, with a go.mod

src/cmd/go/internal/auth/auth.go

@@ -28,7 +28,8 @@ var (
 // as specified by the GOAUTH environment variable.
 // It returns whether any matching credentials were found.
 // req must use HTTPS or this function will panic.
-func AddCredentials(client *http.Client, req *http.Request, prefix string) bool {
+// res is used for the custom GOAUTH command's stdin.
+func AddCredentials(client *http.Client, req *http.Request, res *http.Response, url string) bool {
 	if req.URL.Scheme != "https" {
 		panic("GOAUTH called without https")
 	}
@@ -37,41 +38,31 @@ func AddCredentials(client *http.Client, req *http.Request, prefix string) bool
 	}
 	// Run all GOAUTH commands at least once.
 	authOnce.Do(func() {
-		runGoAuth(client, "")
+		runGoAuth(client, res, "")
 	})
-	if prefix != "" {
+	if url != "" {
-		// First fetch must have failed; re-invoke GOAUTH commands with prefix.
+		// First fetch must have failed; re-invoke GOAUTH commands with url.
-		runGoAuth(client, prefix)
+		runGoAuth(client, res, url)
 	}
-	currentPrefix := strings.TrimPrefix(req.URL.String(), "https://")
+	return loadCredential(req, req.URL.String())
-	// Iteratively try prefixes, moving up the path hierarchy.
-	for currentPrefix != "/" && currentPrefix != "." && currentPrefix != "" {
-		if loadCredential(req, currentPrefix) {
-			return true
-		}
-
-		// Move to the parent directory.
-		currentPrefix = path.Dir(currentPrefix)
-	}
-	return false
 }
 
 // runGoAuth executes authentication commands specified by the GOAUTH
 // environment variable handling 'off', 'netrc', and 'git' methods specially,
 // and storing retrieved credentials for future access.
-func runGoAuth(client *http.Client, prefix string) {
+func runGoAuth(client *http.Client, res *http.Response, url string) {
 	var cmdErrs []error // store GOAUTH command errors to log later.
 	goAuthCmds := strings.Split(cfg.GOAUTH, ";")
 	// The GOAUTH commands are processed in reverse order to prioritize
 	// credentials in the order they were specified.
 	slices.Reverse(goAuthCmds)
-	for _, cmdStr := range goAuthCmds {
+	for _, command := range goAuthCmds {
-		cmdStr = strings.TrimSpace(cmdStr)
+		command = strings.TrimSpace(command)
-		cmdParts := strings.Fields(cmdStr)
+		words := strings.Fields(command)
-		if len(cmdParts) == 0 {
+		if len(words) == 0 {
 			base.Fatalf("GOAUTH encountered an empty command (GOAUTH=%s)", cfg.GOAUTH)
 		}
-		switch cmdParts[0] {
+		switch words[0] {
 		case "off":
 			if len(goAuthCmds) != 1 {
 				base.Fatalf("GOAUTH=off cannot be combined with other authentication commands (GOAUTH=%s)", cfg.GOAUTH)
@@ -85,13 +76,13 @@ func runGoAuth(client *http.Client, prefix string) {
 			for _, l := range lines {
 				r := http.Request{Header: make(http.Header)}
 				r.SetBasicAuth(l.login, l.password)
-				storeCredential([]string{l.machine}, r.Header)
+				storeCredential(l.machine, r.Header)
 			}
 		case "git":
-			if len(cmdParts) != 2 {
+			if len(words) != 2 {
 				base.Fatalf("GOAUTH=git dir method requires an absolute path to the git working directory")
 			}
-			dir := cmdParts[1]
+			dir := words[1]
 			if !filepath.IsAbs(dir) {
 				base.Fatalf("GOAUTH=git dir method requires an absolute path to the git working directory, dir is not absolute")
 			}
@@ -103,28 +94,37 @@ func runGoAuth(client *http.Client, prefix string) {
 				base.Fatalf("GOAUTH=git dir method requires an absolute path to the git working directory, dir is not a directory")
 			}
 
-			if prefix == "" {
+			if url == "" {
 				// Skip the initial GOAUTH run since we need to provide an
-				// explicit prefix to runGitAuth.
+				// explicit url to runGitAuth.
 				continue
 			}
-			prefix, header, err := runGitAuth(client, dir, prefix)
+			prefix, header, err := runGitAuth(client, dir, url)
 			if err != nil {
 				// Save the error, but don't print it yet in case another
 				// GOAUTH command might succeed.
-				cmdErrs = append(cmdErrs, fmt.Errorf("GOAUTH=%s: %v", cmdStr, err))
+				cmdErrs = append(cmdErrs, fmt.Errorf("GOAUTH=%s: %v", command, err))
 			} else {
-				storeCredential([]string{strings.TrimPrefix(prefix, "https://")}, header)
+				storeCredential(prefix, header)
 			}
 		default:
-			base.Fatalf("unimplemented: %s", cmdStr)
+			credentials, err := runAuthCommand(command, url, res)
+			if err != nil {
+				// Save the error, but don't print it yet in case another
+				// GOAUTH command might succeed.
+				cmdErrs = append(cmdErrs, fmt.Errorf("GOAUTH=%s: %v", command, err))
+				continue
+			}
+			for prefix := range credentials {
+				storeCredential(prefix, credentials[prefix])
+			}
 		}
 	}
-	// If no GOAUTH command provided a credential for the given prefix
+	// If no GOAUTH command provided a credential for the given url
 	// and an error occurred, log the error.
-	if cfg.BuildX && prefix != "" {
+	if cfg.BuildX && url != "" {
-		if _, ok := credentialCache.Load(prefix); !ok && len(cmdErrs) > 0 {
+		if ok := loadCredential(&http.Request{}, url); !ok && len(cmdErrs) > 0 {
-			log.Printf("GOAUTH encountered errors for %s:", prefix)
+			log.Printf("GOAUTH encountered errors for %s:", url)
 			for _, err := range cmdErrs {
 				log.Printf("  %v", err)
 			}
@@ -132,29 +132,36 @@ func runGoAuth(client *http.Client, prefix string) {
 	}
 }
 
-// loadCredential retrieves cached credentials for the given url prefix and adds
+// loadCredential retrieves cached credentials for the given url and adds
 // them to the request headers.
-func loadCredential(req *http.Request, prefix string) bool {
+func loadCredential(req *http.Request, url string) bool {
-	headers, ok := credentialCache.Load(prefix)
+	currentPrefix := strings.TrimPrefix(url, "https://")
-	if !ok {
+	// Iteratively try prefixes, moving up the path hierarchy.
-		return false
+	for currentPrefix != "/" && currentPrefix != "." && currentPrefix != "" {
-	}
+		headers, ok := credentialCache.Load(currentPrefix)
-	for key, values := range headers.(http.Header) {
+		if !ok {
-		for _, value := range values {
+			// Move to the parent directory.
-			req.Header.Add(key, value)
+			currentPrefix = path.Dir(currentPrefix)
+			continue
 		}
+		for key, values := range headers.(http.Header) {
+			for _, value := range values {
+				req.Header.Add(key, value)
+			}
+		}
+		return true
 	}
-	return true
+	return false
 }
 
 // storeCredential caches or removes credentials (represented by HTTP headers)
 // associated with given URL prefixes.
-func storeCredential(prefixes []string, header http.Header) {
+func storeCredential(prefix string, header http.Header) {
-	for _, prefix := range prefixes {
+	// Trim "https://" prefix to match the format used in .netrc files.
-		if len(header) == 0 {
+	prefix = strings.TrimPrefix(prefix, "https://")
-			credentialCache.Delete(prefix)
+	if len(header) == 0 {
-		} else {
+		credentialCache.Delete(prefix)
-			credentialCache.Store(prefix, header)
+	} else {
-		}
+		credentialCache.Store(prefix, header)
 	}
 }

src/cmd/go/internal/auth/auth_test.go

@@ -21,7 +21,7 @@ func TestCredentialCache(t *testing.T) {
 	for _, tc := range testCases {
 		want := http.Request{Header: make(http.Header)}
 		want.SetBasicAuth(tc.login, tc.password)
-		storeCredential([]string{tc.machine}, want.Header)
+		storeCredential(tc.machine, want.Header)
 		got := &http.Request{Header: make(http.Header)}
 		ok := loadCredential(got, tc.machine)
 		if !ok || !reflect.DeepEqual(got.Header, want.Header) {
@@ -34,7 +34,7 @@ func TestCredentialCacheDelete(t *testing.T) {
 	// Store a credential for api.github.com
 	want := http.Request{Header: make(http.Header)}
 	want.SetBasicAuth("user", "pwd")
-	storeCredential([]string{"api.github.com"}, want.Header)
+	storeCredential("api.github.com", want.Header)
 	got := &http.Request{Header: make(http.Header)}
 	ok := loadCredential(got, "api.github.com")
 	if !ok || !reflect.DeepEqual(got.Header, want.Header) {
@@ -42,7 +42,7 @@ func TestCredentialCacheDelete(t *testing.T) {
 	}
 	// Providing an empty header for api.github.com should clear credentials.
 	want = http.Request{Header: make(http.Header)}
-	storeCredential([]string{"api.github.com"}, want.Header)
+	storeCredential("api.github.com", want.Header)
 	got = &http.Request{Header: make(http.Header)}
 	ok = loadCredential(got, "api.github.com")
 	if ok {

src/cmd/go/internal/auth/gitauth.go

@@ -23,18 +23,18 @@ import (
 
 const maxTries = 3
 
-// runGitAuth retrieves credentials for the given prefix using
+// runGitAuth retrieves credentials for the given url using
 // 'git credential fill', validates them with a HEAD request
 // (using the provided client) and updates the credential helper's cache.
 // It returns the matching credential prefix, the http.Header with the
 // Basic Authentication header set, or an error.
 // The caller must not mutate the header.
-func runGitAuth(client *http.Client, dir, prefix string) (string, http.Header, error) {
+func runGitAuth(client *http.Client, dir, url string) (string, http.Header, error) {
-	if prefix == "" {
+	if url == "" {
-		// No explicit prefix was passed, but 'git credential'
+		// No explicit url was passed, but 'git credential'
 		// provides no way to enumerate existing credentials.
-		// Wait for a request for a specific prefix.
+		// Wait for a request for a specific url.
-		return "", nil, fmt.Errorf("no explicit prefix was passed")
+		return "", nil, fmt.Errorf("no explicit url was passed")
 	}
 	if dir == "" {
 		// Prevent config-injection attacks by requiring an explicit working directory.
@@ -43,18 +43,18 @@ func runGitAuth(client *http.Client, dir, prefix string) (string, http.Header, e
 	}
 	cmd := exec.Command("git", "credential", "fill")
 	cmd.Dir = dir
-	cmd.Stdin = strings.NewReader(fmt.Sprintf("url=%s\n", prefix))
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("url=%s\n", url))
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return "", nil, fmt.Errorf("'git credential fill' failed (url=%s): %w\n%s", prefix, err, out)
+		return "", nil, fmt.Errorf("'git credential fill' failed (url=%s): %w\n%s", url, err, out)
 	}
 	parsedPrefix, username, password := parseGitAuth(out)
 	if parsedPrefix == "" {
-		return "", nil, fmt.Errorf("'git credential fill' failed for url=%s, could not parse url\n", prefix)
+		return "", nil, fmt.Errorf("'git credential fill' failed for url=%s, could not parse url\n", url)
 	}
 	// Check that the URL Git gave us is a prefix of the one we requested.
-	if !strings.HasPrefix(prefix, parsedPrefix) {
+	if !strings.HasPrefix(url, parsedPrefix) {
-		return "", nil, fmt.Errorf("requested a credential for %s, but 'git credential fill' provided one for %s\n", prefix, parsedPrefix)
+		return "", nil, fmt.Errorf("requested a credential for %s, but 'git credential fill' provided one for %s\n", url, parsedPrefix)
 	}
 	req, err := http.NewRequest("HEAD", parsedPrefix, nil)
 	if err != nil {
@@ -69,7 +69,7 @@ func runGitAuth(client *http.Client, dir, prefix string) (string, http.Header, e
 	// The request is intercepted for testing purposes to simulate interactions
 	// with the credential helper.
 	intercept.Request(req)
-	go updateCredentialHelper(client, req, out)
+	go updateGitCredentialHelper(client, req, out)
 
 	// Return the parsed prefix and headers, even if credential validation fails.
 	// The caller is responsible for the primary validation.
@@ -115,10 +115,10 @@ func parseGitAuth(data []byte) (parsedPrefix, username, password string) {
 	return prefix.String(), username, password
 }
 
-// updateCredentialHelper validates the given credentials by sending a HEAD request
+// updateGitCredentialHelper validates the given credentials by sending a HEAD request
 // and updates the git credential helper's cache accordingly. It retries the
 // request up to maxTries times.
-func updateCredentialHelper(client *http.Client, req *http.Request, credentialOutput []byte) {
+func updateGitCredentialHelper(client *http.Client, req *http.Request, credentialOutput []byte) {
 	for range maxTries {
 		release, err := base.AcquireNet()
 		if err != nil {

src/cmd/go/internal/auth/userauth.go

@@ -0,0 +1,136 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package auth provides access to user-provided authentication credentials.
+package auth
+
+import (
+	"bufio"
+	"bytes"
+	"cmd/internal/quoted"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"net/textproto"
+	"os/exec"
+	"strings"
+)
+
+// runAuthCommand executes a user provided GOAUTH command, parses its output, and
+// returns a mapping of prefix → http.Header.
+// It uses the client to verify the credential and passes the status to the
+// command's stdin.
+// res is used for the GOAUTH command's stdin.
+func runAuthCommand(command string, url string, res *http.Response) (map[string]http.Header, error) {
+	if command == "" {
+		panic("GOAUTH invoked an empty authenticator command:" + command) // This should be caught earlier.
+	}
+	cmd, err := buildCommand(command)
+	if err != nil {
+		return nil, err
+	}
+	if url != "" {
+		cmd.Args = append(cmd.Args, url)
+	}
+	cmd.Stderr = new(strings.Builder)
+	if res != nil && writeResponseToStdin(cmd, res) != nil {
+		return nil, fmt.Errorf("could not run command %s: %v\n%s", command, err, cmd.Stderr)
+	}
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("could not run command %s: %v\n%s", command, err, cmd.Stderr)
+	}
+	credentials, err := parseUserAuth(bytes.NewReader(out))
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse output of GOAUTH command %s: %v", command, err)
+	}
+	return credentials, nil
+}
+
+// parseUserAuth parses the output from a GOAUTH command and
+// returns a mapping of prefix → http.Header without the leading "https://"
+// or an error if the data does not follow the expected format.
+// Returns an nil error and an empty map if the data is empty.
+// See the expected format in 'go help goauth'.
+func parseUserAuth(data io.Reader) (map[string]http.Header, error) {
+	credentials := make(map[string]http.Header)
+	reader := textproto.NewReader(bufio.NewReader(data))
+	for {
+		// Return the processed credentials if the reader is at EOF.
+		if _, err := reader.R.Peek(1); err == io.EOF {
+			return credentials, nil
+		}
+		urls, err := readURLs(reader)
+		if err != nil {
+			return nil, err
+		}
+		if len(urls) == 0 {
+			return nil, fmt.Errorf("invalid format: expected url prefix")
+		}
+		mimeHeader, err := reader.ReadMIMEHeader()
+		if err != nil {
+			return nil, err
+		}
+		header := http.Header(mimeHeader)
+		// Process the block (urls and headers).
+		credentialMap := mapHeadersToPrefixes(urls, header)
+		maps.Copy(credentials, credentialMap)
+	}
+}
+
+// readURLs reads URL prefixes from the given reader until an empty line
+// is encountered or an error occurs. It returns the list of URLs or an error
+// if the format is invalid.
+func readURLs(reader *textproto.Reader) (urls []string, err error) {
+	for {
+		line, err := reader.ReadLine()
+		if err != nil {
+			return nil, err
+		}
+		trimmedLine := strings.TrimSpace(line)
+		if trimmedLine != line {
+			return nil, fmt.Errorf("invalid format: leading or trailing white space")
+		}
+		if strings.HasPrefix(line, "https://") {
+			urls = append(urls, line)
+		} else if line == "" {
+			return urls, nil
+		} else {
+			return nil, fmt.Errorf("invalid format: expected url prefix or empty line")
+		}
+	}
+}
+
+// mapHeadersToPrefixes returns a mapping of prefix → http.Header without
+// the leading "https://".
+func mapHeadersToPrefixes(prefixes []string, header http.Header) map[string]http.Header {
+	prefixToHeaders := make(map[string]http.Header, len(prefixes))
+	for _, p := range prefixes {
+		p = strings.TrimPrefix(p, "https://")
+		prefixToHeaders[p] = header.Clone() // Clone the header to avoid sharing
+	}
+	return prefixToHeaders
+}
+
+func buildCommand(command string) (*exec.Cmd, error) {
+	words, err := quoted.Split(command)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse GOAUTH command %s: %v", command, err)
+	}
+	cmd := exec.Command(words[0], words[1:]...)
+	return cmd, nil
+}
+
+// writeResponseToStdin writes the HTTP response to the command's stdin.
+func writeResponseToStdin(cmd *exec.Cmd, res *http.Response) error {
+	var output strings.Builder
+	output.WriteString(res.Proto + " " + res.Status + "\n")
+	if err := res.Header.Write(&output); err != nil {
+		return err
+	}
+	output.WriteString("\n")
+	cmd.Stdin = strings.NewReader(output.String())
+	return nil
+}

src/cmd/go/internal/auth/userauth_test.go

@@ -0,0 +1,169 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package auth
+
+import (
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParseUserAuth(t *testing.T) {
+	data := `https://example.com
+
+Authorization: Basic YWxhZGRpbjpvcGVuc2VzYW1l
+Authorization: Basic jpvcGVuc2VzYW1lYWxhZGRpb
+
+https://hello.com
+
+Authorization: Basic GVuc2VzYW1lYWxhZGRpbjpvc
+Authorization: Basic 1lYWxhZGRplW1lYWxhZGRpbs
+Data: Test567
+
+`
+	// Build the expected header
+	header1 := http.Header{
+		"Authorization": []string{
+			"Basic YWxhZGRpbjpvcGVuc2VzYW1l",
+			"Basic jpvcGVuc2VzYW1lYWxhZGRpb",
+		},
+	}
+	header2 := http.Header{
+		"Authorization": []string{
+			"Basic GVuc2VzYW1lYWxhZGRpbjpvc",
+			"Basic 1lYWxhZGRplW1lYWxhZGRpbs",
+		},
+		"Data": []string{
+			"Test567",
+		},
+	}
+	credentials, err := parseUserAuth(strings.NewReader(data))
+	if err != nil {
+		t.Errorf("parseUserAuth(%s): %v", data, err)
+	}
+	gotHeader, ok := credentials["example.com"]
+	if !ok || !reflect.DeepEqual(gotHeader, header1) {
+		t.Errorf("parseUserAuth(%s):\nhave %q\nwant %q", data, gotHeader, header1)
+	}
+	gotHeader, ok = credentials["hello.com"]
+	if !ok || !reflect.DeepEqual(gotHeader, header2) {
+		t.Errorf("parseUserAuth(%s):\nhave %q\nwant %q", data, gotHeader, header2)
+	}
+}
+
+func TestParseUserAuthInvalid(t *testing.T) {
+	testCases := []string{
+		// Missing new line after url.
+		`https://example.com
+Authorization: Basic AVuc2VzYW1lYWxhZGRpbjpvc
+
+`,
+		// Missing url.
+		`Authorization: Basic AVuc2VzYW1lYWxhZGRpbjpvc
+
+`,
+		// Missing url.
+		`https://example.com
+
+Authorization: Basic YWxhZGRpbjpvcGVuc2VzYW1l
+Authorization: Basic jpvcGVuc2VzYW1lYWxhZGRpb
+
+Authorization: Basic GVuc2VzYW1lYWxhZGRpbjpvc
+Authorization: Basic 1lYWxhZGRplW1lYWxhZGRpbs
+Data: Test567
+
+`,
+		// Wrong order.
+		`Authorization: Basic AVuc2VzYW1lYWxhZGRpbjpvc
+
+https://example.com
+
+`,
+		// Missing new lines after URL.
+		`https://example.com
+`,
+		// Missing new line after empty header.
+		`https://example.com
+
+`,
+		// Missing new line between blocks.
+		`https://example.com
+
+Authorization: Basic YWxhZGRpbjpvcGVuc2VzYW1l
+Authorization: Basic jpvcGVuc2VzYW1lYWxhZGRpb
+https://hello.com
+
+Authorization: Basic GVuc2VzYW1lYWxhZGRpbjpvc
+Authorization: Basic 1lYWxhZGRplW1lYWxhZGRpbs
+Data: Test567
+
+`,
+	}
+	for _, tc := range testCases {
+		if credentials, err := parseUserAuth(strings.NewReader(tc)); err == nil {
+			t.Errorf("parseUserAuth(%s) should have failed, but got: %v", tc, credentials)
+		}
+	}
+}
+
+func TestParseUserAuthDuplicated(t *testing.T) {
+	data := `https://example.com
+
+Authorization: Basic YWxhZGRpbjpvcGVuc2VzYW1l
+Authorization: Basic jpvcGVuc2VzYW1lYWxhZGRpb
+
+https://example.com
+
+Authorization: Basic GVuc2VzYW1lYWxhZGRpbjpvc
+Authorization: Basic 1lYWxhZGRplW1lYWxhZGRpbs
+Data: Test567
+
+`
+	// Build the expected header
+	header := http.Header{
+		"Authorization": []string{
+			"Basic GVuc2VzYW1lYWxhZGRpbjpvc",
+			"Basic 1lYWxhZGRplW1lYWxhZGRpbs",
+		},
+		"Data": []string{
+			"Test567",
+		},
+	}
+	credentials, err := parseUserAuth(strings.NewReader(data))
+	if err != nil {
+		t.Errorf("parseUserAuth(%s): %v", data, err)
+	}
+	gotHeader, ok := credentials["example.com"]
+	if !ok || !reflect.DeepEqual(gotHeader, header) {
+		t.Errorf("parseUserAuth(%s):\nhave %q\nwant %q", data, gotHeader, header)
+	}
+}
+
+func TestParseUserAuthEmptyHeader(t *testing.T) {
+	data := "https://example.com\n\n\n"
+	// Build the expected header
+	header := http.Header{}
+	credentials, err := parseUserAuth(strings.NewReader(data))
+	if err != nil {
+		t.Errorf("parseUserAuth(%s): %v", data, err)
+	}
+	gotHeader, ok := credentials["example.com"]
+	if !ok || !reflect.DeepEqual(gotHeader, header) {
+		t.Errorf("parseUserAuth(%s):\nhave %q\nwant %q", data, gotHeader, header)
+	}
+}
+
+func TestParseUserAuthEmpty(t *testing.T) {
+	data := ``
+	// Build the expected header
+	credentials, err := parseUserAuth(strings.NewReader(data))
+	if err != nil {
+		t.Errorf("parseUserAuth(%s) should have succeeded", data)
+	}
+	if credentials == nil {
+		t.Errorf("parseUserAuth(%s) should have returned a non-nil credential map, but got %v", data, credentials)
+	}
+}

src/cmd/go/internal/help/helpdoc.go

@@ -501,12 +501,8 @@ General-purpose environment variables:
 		The architecture, or processor, for which to compile code.
 		Examples are amd64, 386, arm, ppc64.
 	GOAUTH
-		A semicolon-separated list of authentication commands for go-import and
+		Controls authentication for go-import and HTTPS module mirror interactions.
-		HTTPS module mirror interactions. Currently supports
+		See 'go help goauth'.
-		"off" (disables authentication),
-		"netrc" (uses credentials from NETRC or the .netrc file in your home directory),
-		"git dir" (runs 'git credential fill' in dir and uses its credentials).
-		The default is netrc.
 	GOBIN
 		The directory where 'go install' will install a command.
 	GOCACHE
@@ -982,3 +978,69 @@ has a term for a Go major release, the language version used when compiling
 the file will be the minimum version implied by the build constraint.
 `,
 }
+
+var HelpGoAuth = &base.Command{
+	UsageLine: "goauth",
+	Short:     "GOAUTH environment variable",
+	Long: `
+GOAUTH is a semicolon-separated list of authentication commands for go-import and
+HTTPS module mirror interactions. The default is netrc.
+
+The supported authentication commands are:
+
+off
+	Disables authentication.
+netrc
+	Uses credentials from NETRC or the .netrc file in your home directory.
+git dir
+	Runs 'git credential fill' in dir and uses its credentials. The
+	go command will run 'git credential approve/reject' to update
+	the credential helper's cache.
+command
+	Executes the given command (a space-separated argument list) and attaches
+	the provided headers to HTTPS requests.
+	The command must produce output in the following format:
+		Response      = { CredentialSet } .
+		CredentialSet = URLLine { URLLine } BlankLine { HeaderLine } BlankLine .
+		URLLine       = /* URL that starts with "https://" */ '\n' .
+		HeaderLine    = /* HTTP Request header */ '\n' .
+		BlankLine     = '\n' .
+
+	Example:
+		https://example.com/
+		https://example.net/api/
+
+		Authorization: Basic <token>
+
+		https://another-example.org/
+
+		Example: Data
+
+	If the server responds with any 4xx code, the go command will write the
+	following to the programs' stdin:
+		Response      = StatusLine { HeaderLine } BlankLine .
+		StatusLine    = Protocol Space Status '\n' .
+		Protocol      = /* HTTP protocol */ .
+		Space         = ' ' .
+		Status        = /* HTTP status code */ .
+		BlankLine     = '\n' .
+		HeaderLine    = /* HTTP Response's header */ '\n' .
+
+	Example:
+		HTTP/1.1 401 Unauthorized
+		Content-Length: 19
+		Content-Type: text/plain; charset=utf-8
+		Date: Thu, 07 Nov 2024 18:43:09 GMT
+
+	Note: at least for HTTP 1.1, the contents written to stdin can be parsed
+	as an HTTP response.
+
+Before the first HTTPS fetch, the go command will invoke each GOAUTH
+command in the list with no additional arguments and no input.
+If the server responds with any 4xx code, the go command will invoke the
+GOAUTH commands again with the URL as an additional command-line argument
+and the HTTP Response to the program's stdin.
+If the server responds with an error again, the fetch fails: a URL-specific
+GOAUTH will only be attempted once per fetch.
+`,
+}

src/cmd/go/internal/web/http.go

@@ -140,7 +140,7 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
 		}
 		if url.Scheme == "https" {
 			// Use initial GOAUTH credentials.
-			auth.AddCredentials(client, req, "")
+			auth.AddCredentials(client, req, nil, "")
 		}
 		if intercepted {
 			req.Host = req.URL.Host
@@ -170,7 +170,7 @@ func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
 			if err != nil {
 				return nil, err
 			}
-			auth.AddCredentials(client, req, url.String())
+			auth.AddCredentials(client, req, res, url.String())
 			intercept.Request(req)
 			res, err = client.Do(req)
 		}

src/cmd/go/main.go

@@ -76,6 +76,7 @@ func init() {
 		help.HelpCache,
 		help.HelpEnvironment,
 		help.HelpFileType,
+		help.HelpGoAuth,
 		modload.HelpGoMod,
 		help.HelpGopath,
 		modfetch.HelpGoproxy,

src/cmd/go/testdata/script/goauth_userauth.txt

@@ -0,0 +1,79 @@
+# This test covers the HTTP authentication mechanism over GOAUTH by using a custom authenticator.
+# See golang.org/issue/26232
+
+env GOPROXY=direct
+env GOSUMDB=off
+
+# Use a custom authenticator to provide custom credentials
+mkdir $WORK/bin
+env PATH=$WORK/bin${:}$PATH
+cd auth
+go build -o $WORK/bin/my-auth$GOEXE .
+cd ..
+
+# Without credentials, downloading a module from a path that requires HTTPS
+# basic auth should fail.
+env GOAUTH=off
+cp go.mod.orig go.mod
+! go get vcs-test.golang.org/auth/or401
+stderr '^\tserver response: ACCESS DENIED, buddy$'
+# go imports should fail as well.
+! go mod tidy
+stderr '^\tserver response: ACCESS DENIED, buddy$'
+
+# With credentials from the my-auth binary, it should succeed.
+env GOAUTH='my-auth'$GOEXE' --arg1 "value with spaces"'
+cp go.mod.orig go.mod
+go get vcs-test.golang.org/auth/or401
+# go imports should resolve correctly as well.
+go mod tidy
+go list all
+stdout vcs-test.golang.org/auth/or401
+
+-- auth/main.go --
+package main
+
+import(
+	"bufio"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+)
+
+func main() {
+	arg1 := flag.String("arg1", "", "")
+	flag.Parse()
+	if *arg1 != "value with spaces" {
+		log.Fatal("argument with spaces does not work")
+	}
+	// wait for re-invocation
+	if !strings.HasPrefix(flag.Arg(0), "https://vcs-test.golang.org") {
+		return
+	}
+	input, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		log.Fatal("unexpected error while reading from stdin")
+	}
+	reader := bufio.NewReader(strings.NewReader(string(input)))
+	resp, err := http.ReadResponse(reader, nil)
+	if err != nil {
+		log.Fatal("could not parse HTTP response")
+	}
+	if resp.StatusCode != 401 {
+		log.Fatal("expected 401 error code")
+	}
+	fmt.Printf("https://vcs-test.golang.org\n\nAuthorization: Basic YWxhZGRpbjpvcGVuc2VzYW1l\n\n")
+}
+
+-- auth/go.mod --
+module my-auth
+-- go.mod.orig --
+module private.example.com
+-- main.go --
+package useprivate
+
+import "vcs-test.golang.org/auth/or401"