encoding/hex: fix Decode output check regression

CL 461958 fixed a potential panic,
but also introduced an observable regression where
invalid input could be detected before the panic occurs.
Adjust the check to preserve prior behavior,
while also preventing the panic.

Change-Id: I52819f88a6a64883fbc9fd512697c38c29ca0ccd
Reviewed-on: https://go-review.googlesource.com/c/go/+/465855
Auto-Submit: Joseph Tsai <joetsai@digital-static.net>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
Run-TryBot: Joseph Tsai <joetsai@digital-static.net>
Reviewed-by: Ian Lance Taylor <iant@google.com>

src/encoding/hex/hex.go

@@ -75,9 +75,6 @@ func DecodedLen(x int) int { return x / 2 }
 // If the input is malformed, Decode returns the number
 // of bytes decoded before the error.
 func Decode(dst, src []byte) (int, error) {
-	if len(dst) < DecodedLen(len(src)) {
-		return 0, errors.New("encoding/hex: output buffer too small")
-	}
 	i, j := 0, 1
 	for ; j < len(src); j += 2 {
 		p := src[j-1]
@@ -91,6 +88,9 @@ func Decode(dst, src []byte) (int, error) {
 		if b > 0x0f {
 			return i, InvalidByteError(q)
 		}
+		if i >= len(dst) {
+			return i, errors.New("encoding/hex: output buffer too small")
+		}
 		dst[i] = (a << 4) | b
 		i++
 	}

src/encoding/hex/hex_test.go

@@ -55,13 +55,16 @@ func TestDecode(t *testing.T) {
 	}
 }
 
-func TestDecode_tooFewDstBytes(t *testing.T) {
+func TestDecodeDstTooSmall(t *testing.T) {
 	dst := make([]byte, 1)
 	src := []byte{'0', '1', '2', '3'}
-	_, err := Decode(dst, src)
+	n, err := Decode(dst, src)
 	if err == nil {
 		t.Errorf("expected Decode to return an error, but it returned none")
 	}
+	if !bytes.Equal(dst[:n], []byte{0x01}) {
+		t.Errorf("output mismatch: got %x, want 01", dst[:n])
+	}
 }
 
 func TestEncodeToString(t *testing.T) {