Realign TOML lexers with the Pygments counterpart

This also helps to address a number of TOML bugs around handling of the
value attributes.

Author: tancnle

lib/rouge/lexers/toml.rb

@@ -11,55 +11,60 @@ class TOML < RegexLexer
       filenames '*.toml', 'Pipfile', 'poetry.lock'
       mimetypes 'text/x-toml'
 
-      # bare keys and quoted keys
-      identifier = %r/(?:\S+|"[^"]+"|'[^']+')/
+      state :root do
+        mixin :whitespace
 
-      state :basic do
-        rule %r/\s+/, Text
-        rule %r/#.*?$/, Comment
-        rule %r/(true|false)/, Keyword::Constant
+        mixin :key
 
-        rule %r/(#{identifier})(\s*)(=)(\s*)(\{)/ do
-          groups Name::Property, Text, Operator, Text, Punctuation
-          push :inline
+        rule %r/(=)(\s*)/ do
+          groups Operator, Text::Whitespace
+          push :value
         end
+
+        rule %r/\[\[?/, Keyword, :table_key
       end
 
-      state :root do
-        mixin :basic
+      state :key do
+        rule %r/[A-Za-z0-9_-]+/, Name
 
-        rule %r/(?<!=)\s*\[.*?\]+/, Name::Namespace
+        rule %r/"/, Str, :dq
+        rule %r/'/, Str, :sq
+        rule %r/\./, Punctuation
+      end
 
-        rule %r/(#{identifier})(\s*)(=)/ do
-          groups Name::Property, Text, Punctuation
-          push :value
-        end
+      state :table_key do
+        rule %r/[A-Za-z0-9_-]+/, Name
+
+        rule %r/"/, Str, :dq
+        rule %r/'/, Str, :sq
+        rule %r/\./, Keyword
+        rule %r/\]\]?/, Keyword, :pop!
+        rule %r/[ \t]+/, Text::Whitespace
       end
 
       state :value do
-        rule %r/\n/, Text, :pop!
-        mixin :content
+        rule %r/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/, Literal::Date, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float, :pop!
+        rule %r/[+-]?(?:nan|inf)/, Num::Float, :pop!
+        rule %r/0x\h+(?:_\h+)*/, Num::Hex, :pop!
+        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct, :pop!
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer, :pop!
+
+        rule %r/"""/, Str, [:pop!, :mdq]
+        rule %r/"/, Str, [:pop!, :dq]
+        rule %r/'''/, Str, [:pop!, :msq]
+        rule %r/'/, Str, [:pop!, :sq]
+
+        rule %r/(true|false)/, Keyword::Constant, :pop!
+        rule %r/\[/, Punctuation, [:pop!, :array]
+        rule %r/\{/, Punctuation, [:pop!, :inline]
       end
 
       state :content do
         mixin :basic
 
-        rule %r/(#{identifier})(\s*)(=)/ do
-          groups Name::Property, Text, Punctuation
-        end
-
-        rule %r/\d{4}-\d{2}-\d{2}(?:[Tt ]\d{2}:\d{2}:\d{2}(?:[Zz]|[+-]\d{2}:\d{2})?)?/, Literal::Date
-        rule %r/\d{2}:\d{2}:\d{2}/, Literal::Date
-
-        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float
-        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float
-        rule %r/[+-]?(?:nan|inf)/, Num::Float
-
-        rule %r/0x\h+(?:_\h+)*/, Num::Hex
-        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct
-        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
-        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer
-
         rule %r/"""/, Str, :mdq
         rule %r/"/, Str, :dq
         rule %r/'''/, Str, :msq
@@ -101,15 +106,31 @@ class TOML < RegexLexer
       end
 
       state :array do
-        mixin :content
+        mixin :whitespace
+        rule %r/,/, Punctuation
+
         rule %r/\]/, Punctuation, :pop!
+
+        rule %r//, Token, :value
       end
 
       state :inline do
-        mixin :content
+        rule %r/[ \t]+/, Text::Whitespace
 
+        mixin :key
+        rule %r/(=)(\s*)/ do
+          groups Punctuation, Text::Whitespace
+          push :value
+        end
+
+        rule %r/,/, Punctuation
         rule %r/\}/, Punctuation, :pop!
       end
+
+      state :whitespace do
+        rule %r/\s+/, Text
+        rule %r/#.*?$/, Comment
+      end
     end
   end
 end

spec/visual/samples/toml

@@ -8,10 +8,10 @@ organization = "GitHub"
 bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
 # First class dates? Why not?
 dob = [
-	1979-05-27T07:32:00Z,
-	1979-05-27 07:32:00
-	1979-05-27,
-	07:32:00,
+  1979-05-27T07:32:00Z,
+  1979-05-27 07:32:00
+  1979-05-27,
+  07:32:00,
 ]
 
 [database]
@@ -187,8 +187,16 @@ inf-3 = {inf=inf, nan=nan}
 str-inf-1 = 'inf'
 str-inf-2 = "inf"
 str-inf-3 = """ inf nan
-	inf nan
+  inf nan
 """
 str-inf-4 = """ inf nan
-	inf nan
+  inf nan
 """
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.11"
+idna = "3.4"
+inflection = "0.5.1"
+influxdb = "5.3.1"
+requests = "2.28.1"
+pywinpty = {version = ">=1.1.0", markers = "os_name == \"nt\""}