Unreserved 8 keywords (#851) (#928)

* parser.y: unreserve 8 keywords

* consistent_test: added an optional test to check against MySQL keywords

Author: kennytm

consistent_test.go

@@ -27,46 +27,51 @@ import (
 var _ = Suite(&testConsistentSuite{})
 
 type testConsistentSuite struct {
+	content string
+
+	reservedKeywords   []string
+	unreservedKeywords []string
+	notKeywordTokens   []string
+	tidbKeywords       []string
 }
 
-func (s *testConsistentSuite) TestKeywordConsistent(c *C) {
+func (s *testConsistentSuite) SetUpSuite(c *C) {
 	_, filename, _, _ := runtime.Caller(0)
 	parserFilename := path.Join(path.Dir(filename), "parser.y")
 	parserFile, err := os.Open(parserFilename)
 	c.Assert(err, IsNil)
 	data, err := ioutil.ReadAll(parserFile)
 	c.Assert(err, IsNil)
-	content := string(data)
+	s.content = string(data)
 
 	reservedKeywordStartMarker := "\t/* The following tokens belong to ReservedKeyword. Notice: make sure these tokens are contained in ReservedKeyword. */"
 	unreservedKeywordStartMarker := "\t/* The following tokens belong to UnReservedKeyword. Notice: make sure these tokens are contained in UnReservedKeyword. */"
 	notKeywordTokenStartMarker := "\t/* The following tokens belong to NotKeywordToken. Notice: make sure these tokens are contained in NotKeywordToken. */"
 	tidbKeywordStartMarker := "\t/* The following tokens belong to TiDBKeyword. Notice: make sure these tokens are contained in TiDBKeyword. */"
 	identTokenEndMarker := "%token\t<item>"
 
-	reservedKeywords := extractKeywords(content, reservedKeywordStartMarker, unreservedKeywordStartMarker)
-
-	unreservedKeywords := extractKeywords(content, unreservedKeywordStartMarker, notKeywordTokenStartMarker)
-
-	notKeywordTokens := extractKeywords(content, notKeywordTokenStartMarker, tidbKeywordStartMarker)
-
-	tidbKeywords := extractKeywords(content, tidbKeywordStartMarker, identTokenEndMarker)
+	s.reservedKeywords = extractKeywords(s.content, reservedKeywordStartMarker, unreservedKeywordStartMarker)
+	s.unreservedKeywords = extractKeywords(s.content, unreservedKeywordStartMarker, notKeywordTokenStartMarker)
+	s.notKeywordTokens = extractKeywords(s.content, notKeywordTokenStartMarker, tidbKeywordStartMarker)
+	s.tidbKeywords = extractKeywords(s.content, tidbKeywordStartMarker, identTokenEndMarker)
+}
 
+func (s *testConsistentSuite) TestKeywordConsistent(c *C) {
 	for k, v := range aliases {
-		c.Assert(k != v, IsTrue)
+		c.Assert(k, Not(Equals), v)
 		c.Assert(tokenMap[k], Equals, tokenMap[v])
 	}
-	keywordCount := len(reservedKeywords) + len(unreservedKeywords) + len(notKeywordTokens) + len(tidbKeywords)
+	keywordCount := len(s.reservedKeywords) + len(s.unreservedKeywords) + len(s.notKeywordTokens) + len(s.tidbKeywords)
 	c.Assert(len(tokenMap)-len(aliases), Equals, keywordCount-len(windowFuncTokenMap))
 
-	unreservedCollectionDef := extractKeywordsFromCollectionDef(content, "\nUnReservedKeyword:")
-	c.Assert(unreservedKeywords, DeepEquals, unreservedCollectionDef)
+	unreservedCollectionDef := extractKeywordsFromCollectionDef(s.content, "\nUnReservedKeyword:")
+	c.Assert(s.unreservedKeywords, DeepEquals, unreservedCollectionDef)
 
-	notKeywordTokensCollectionDef := extractKeywordsFromCollectionDef(content, "\nNotKeywordToken:")
-	c.Assert(notKeywordTokens, DeepEquals, notKeywordTokensCollectionDef)
+	notKeywordTokensCollectionDef := extractKeywordsFromCollectionDef(s.content, "\nNotKeywordToken:")
+	c.Assert(s.notKeywordTokens, DeepEquals, notKeywordTokensCollectionDef)
 
-	tidbKeywordsCollectionDef := extractKeywordsFromCollectionDef(content, "\nTiDBKeyword:")
-	c.Assert(tidbKeywords, DeepEquals, tidbKeywordsCollectionDef)
+	tidbKeywordsCollectionDef := extractKeywordsFromCollectionDef(s.content, "\nTiDBKeyword:")
+	c.Assert(s.tidbKeywords, DeepEquals, tidbKeywordsCollectionDef)
 }
 
 func extractMiddle(str, startMarker, endMarker string) string {

go.mod1

@@ -7,6 +7,7 @@ require (
 	github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8
 	github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186
 	github.com/cznic/y v0.0.0-20170802143616-045f81c6662a
+	github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
 	github.com/pingcap/errors v0.11.4
 	github.com/pingcap/tidb v0.0.0-20190321025159-e8299209340c

go.sum1

@@ -51,6 +51,7 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4 h1:3DFRjZdCDhzvxDf0U6/1qAryzOqD7Y5iAj0DJRRl1bs=
 github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=