Test should disallow on FromStatus(500) like TestAgent correctly was

fixes #40
co-authored-by: https://github.com/masonlouchart

Author: temoto

robotstxt.go

@@ -18,18 +18,21 @@ import (
 )
 
 type RobotsData struct {
+	Host     string
+	Sitemaps []string
+
 	// private
-	groups      map[string]*Group
 	allowAll    bool
 	disallowAll bool
-	Host        string
-	Sitemaps    []string
+	groups      map[string]*Group
 }
 
 type Group struct {
 	rules      []*rule
 	Agent      string
 	CrawlDelay time.Duration
+
+	disallowAll bool
 }
 
 type rule struct {
@@ -59,6 +62,7 @@ func (e ParseError) Error() string {
 var allowAll = &RobotsData{allowAll: true}
 var disallowAll = &RobotsData{disallowAll: true}
 var emptyGroup = &Group{}
+var emptyDisallowGroup = &Group{disallowAll: true}
 
 func FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error) {
 	switch {
@@ -173,12 +177,19 @@ func (r *RobotsData) FindGroup(agent string) (ret *Group) {
 	}
 
 	if ret == nil {
-		return emptyGroup
+		if r.disallowAll {
+			return emptyDisallowGroup
+		} else {
+			return emptyGroup
+		}
 	}
 	return
 }
 
 func (g *Group) Test(path string) bool {
+	if g.disallowAll {
+		return false
+	}
 	if r := g.findRule(path); r != nil {
 		return r.allow
 	}

robotstxt_test.go

@@ -230,6 +230,16 @@ disallow: /c`
 	expectAccess(t, r, false, "/c", "c")
 }
 
+func TestDifferentApiDisallowAll(t *testing.T) {
+	r, err := FromStatusAndBytes(500, nil)
+	require.NoError(t, err)
+
+	a := r.TestAgent("/", "*")
+	b := r.FindGroup("*").Test("/")
+	assert.Equal(t, false, a, "Expected false (disallow) from TestAgent(/, *)")
+	assert.Equal(t, false, b, "Expected false (disallow) from FindGroup(*).Test(/)")
+}
+
 func BenchmarkParseFromString001(b *testing.B) {
 	input := robotsText001
 	b.ReportAllocs()