dns/dnsmessage: reject too large of names early during unpack

The existing implementation of the Name.unpack method does not check the
length of the domain name until parsing is complete. This allows a
malicious user to supply an unreasonably large name and wastle cycles
parsing. This change moves an equivalent check into the loop during
process to short-circuit if we've created too large of a name.

For golang/go#77540

Change-Id: I4c4bf20c0342825a09cefd9b0b3c0bdce0c80137
Reviewed-on: https://go-review.googlesource.com/c/net/+/750260
Reviewed-by: Roland Shoemaker <roland@golang.org>
Auto-Submit: Roland Shoemaker <roland@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>

Author: srri

dns/dnsmessage/message.go

@@ -2085,7 +2085,11 @@ Loop:
 					return off, errInvalidName
 				}
 			}
-
+			// Reject names that are too long while unpacking
+			// See issue golang/go#77540
+			if len(name)+(endOff-currOff) >= nonEncodedNameMax {
+				return off, errNameTooLong
+			}
 			name = append(name, msg[currOff:endOff]...)
 			name = append(name, '.')
 			currOff = endOff
@@ -2111,9 +2115,6 @@ Loop:
 	if len(name) == 0 {
 		name = append(name, '.')
 	}
-	if len(name) > nonEncodedNameMax {
-		return off, errNameTooLong
-	}
 	n.Length = uint8(len(name))
 	if ptr == 0 {
 		newOff = currOff

dns/dnsmessage/message_test.go

@@ -292,6 +292,8 @@ func TestNameUnpackTooLongName(t *testing.T) {
 	}{
 		{name: prepName(255)},
 		{name: prepName(256), err: errNameTooLong},
+		// too large to be valid, return error during unpack.
+		{name: prepName(300), err: errNameTooLong},
 	}
 
 	for i, test := range tests {