Skip to content

Unreserved 8 keywords #851

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kennytm merged 3 commits into from
May 12, 2020
Merged

Unreserved 8 keywords #851

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0fd10f6
parser.y: unreserve 8 keywords
kennytm May 10, 2020
0bba6c2
consistent_test: added an optional test to check against MySQL keywords
kennytm May 10, 2020
5890b78
Merge branch 'master' into kennytm/error-is-not-a-keyword
kennytm May 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 22 additions & 17 deletions consistent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,46 +27,51 @@ import (
var _ = Suite(&testConsistentSuite{})

type testConsistentSuite struct {
content string

reservedKeywords []string
unreservedKeywords []string
notKeywordTokens []string
tidbKeywords []string
}

func (s *testConsistentSuite) TestKeywordConsistent(c *C) {
func (s *testConsistentSuite) SetUpSuite(c *C) {
_, filename, _, _ := runtime.Caller(0)
parserFilename := path.Join(path.Dir(filename), "parser.y")
parserFile, err := os.Open(parserFilename)
c.Assert(err, IsNil)
data, err := ioutil.ReadAll(parserFile)
c.Assert(err, IsNil)
content := string(data)
s.content = string(data)

reservedKeywordStartMarker := "\t/* The following tokens belong to ReservedKeyword. Notice: make sure these tokens are contained in ReservedKeyword. */"
unreservedKeywordStartMarker := "\t/* The following tokens belong to UnReservedKeyword. Notice: make sure these tokens are contained in UnReservedKeyword. */"
notKeywordTokenStartMarker := "\t/* The following tokens belong to NotKeywordToken. Notice: make sure these tokens are contained in NotKeywordToken. */"
tidbKeywordStartMarker := "\t/* The following tokens belong to TiDBKeyword. Notice: make sure these tokens are contained in TiDBKeyword. */"
identTokenEndMarker := "%token\t<item>"

reservedKeywords := extractKeywords(content, reservedKeywordStartMarker, unreservedKeywordStartMarker)

unreservedKeywords := extractKeywords(content, unreservedKeywordStartMarker, notKeywordTokenStartMarker)

notKeywordTokens := extractKeywords(content, notKeywordTokenStartMarker, tidbKeywordStartMarker)

tidbKeywords := extractKeywords(content, tidbKeywordStartMarker, identTokenEndMarker)
s.reservedKeywords = extractKeywords(s.content, reservedKeywordStartMarker, unreservedKeywordStartMarker)
s.unreservedKeywords = extractKeywords(s.content, unreservedKeywordStartMarker, notKeywordTokenStartMarker)
s.notKeywordTokens = extractKeywords(s.content, notKeywordTokenStartMarker, tidbKeywordStartMarker)
s.tidbKeywords = extractKeywords(s.content, tidbKeywordStartMarker, identTokenEndMarker)
}

func (s *testConsistentSuite) TestKeywordConsistent(c *C) {
for k, v := range aliases {
c.Assert(k != v, IsTrue)
c.Assert(k, Not(Equals), v)
c.Assert(tokenMap[k], Equals, tokenMap[v])
}
keywordCount := len(reservedKeywords) + len(unreservedKeywords) + len(notKeywordTokens) + len(tidbKeywords)
keywordCount := len(s.reservedKeywords) + len(s.unreservedKeywords) + len(s.notKeywordTokens) + len(s.tidbKeywords)
c.Assert(len(tokenMap)-len(aliases), Equals, keywordCount-len(windowFuncTokenMap))

unreservedCollectionDef := extractKeywordsFromCollectionDef(content, "\nUnReservedKeyword:")
c.Assert(unreservedKeywords, DeepEquals, unreservedCollectionDef)
unreservedCollectionDef := extractKeywordsFromCollectionDef(s.content, "\nUnReservedKeyword:")
c.Assert(s.unreservedKeywords, DeepEquals, unreservedCollectionDef)

notKeywordTokensCollectionDef := extractKeywordsFromCollectionDef(content, "\nNotKeywordToken:")
c.Assert(notKeywordTokens, DeepEquals, notKeywordTokensCollectionDef)
notKeywordTokensCollectionDef := extractKeywordsFromCollectionDef(s.content, "\nNotKeywordToken:")
c.Assert(s.notKeywordTokens, DeepEquals, notKeywordTokensCollectionDef)

tidbKeywordsCollectionDef := extractKeywordsFromCollectionDef(content, "\nTiDBKeyword:")
c.Assert(tidbKeywords, DeepEquals, tidbKeywordsCollectionDef)
tidbKeywordsCollectionDef := extractKeywordsFromCollectionDef(s.content, "\nTiDBKeyword:")
c.Assert(s.tidbKeywords, DeepEquals, tidbKeywordsCollectionDef)
}

func extractMiddle(str, startMarker, endMarker string) string {
Expand Down
3 changes: 1 addition & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@ require (
github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8
github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186
github.com/cznic/y v0.0.0-20170802143616-045f81c6662a
github.com/golang/protobuf v1.3.2 // indirect
github.com/go-sql-driver/mysql v1.5.0
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
github.com/pingcap/errors v0.11.4
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9
github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
go.uber.org/zap v1.12.0
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2
Expand Down
6 changes: 2 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
Expand All @@ -31,8 +31,6 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 h1:rRMLMjIMFulCX9sGKZ1hoov/iROMsKyC8Snc02nSukw=
github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down
Loading
⚡