Realign TOML lexers with the Pygments counterpart

This also helps to address a number of TOML bugs around handling of the
value attributes.

Author: tancnle

lib/rouge/lexers/toml.rb

@@ -11,55 +11,60 @@ class TOML < RegexLexer
       filenames '*.toml', 'Pipfile', 'poetry.lock'
       mimetypes 'text/x-toml'
 
-      # bare keys and quoted keys
-      identifier = %r/(?:\S+|"[^"]+"|'[^']+')/
+      state :root do
+        mixin :whitespace
 
-      state :basic do
-        rule %r/\s+/, Text
-        rule %r/#.*?$/, Comment
-        rule %r/(true|false)/, Keyword::Constant
+        mixin :key
 
-        rule %r/(#{identifier})(\s*)(=)(\s*)(\{)/ do
-          groups Name::Property, Text, Operator, Text, Punctuation
-          push :inline
+        rule %r/(=)(\s*)/ do
+          groups Operator, Text::Whitespace
+          push :value
         end
+
+        rule %r/\[\[?/, Keyword, :table_key
       end
 
-      state :root do
-        mixin :basic
+      state :key do
+        rule %r/[A-Za-z0-9_-]+/, Name
 
-        rule %r/(?<!=)\s*\[.*?\]+/, Name::Namespace
+        rule %r/"/, Str, :dq
+        rule %r/'/, Str, :sq
+        rule %r/\./, Punctuation
+      end
 
-        rule %r/(#{identifier})(\s*)(=)/ do
-          groups Name::Property, Text, Punctuation
-          push :value
-        end
+      state :table_key do
+        rule %r/[A-Za-z0-9_-]+/, Name
+
+        rule %r/"/, Str, :dq
+        rule %r/'/, Str, :sq
+        rule %r/\./, Keyword
+        rule %r/\]\]?/, Keyword, :pop!
+        rule %r/[ \t]+/, Text::Whitespace
       end
 
       state :value do
-        rule %r/\n/, Text, :pop!
-        mixin :content
+        rule %r/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/, Literal::Date, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float, :pop!
+        rule %r/[+-]?(?:nan|inf)/, Num::Float, :pop!
+        rule %r/0x\h+(?:_\h+)*/, Num::Hex, :pop!
+        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct, :pop!
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin, :pop!
+        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer, :pop!
+
+        rule %r/"""/, Str, [:pop!, :mdq]
+        rule %r/"/, Str, [:pop!, :dq]
+        rule %r/'''/, Str, [:pop!, :msq]
+        rule %r/'/, Str, [:pop!, :sq]
+
+        rule %r/(true|false)/, Keyword::Constant, :pop!
+        rule %r/\[/, Punctuation, [:pop!, :array]
+        rule %r/\{/, Punctuation, [:pop!, :inline]
       end
 
       state :content do
         mixin :basic
 
-        rule %r/(#{identifier})(\s*)(=)/ do
-          groups Name::Property, Text, Punctuation
-        end
-
-        rule %r/\d{4}-\d{2}-\d{2}(?:[Tt ]\d{2}:\d{2}:\d{2}(?:[Zz]|[+-]\d{2}:\d{2})?)?/, Literal::Date
-        rule %r/\d{2}:\d{2}:\d{2}/, Literal::Date
-
-        rule %r/[+-]?\d+(?:_\d+)*\.\d+(?:_\d+)*(?:[eE][+-]?\d+(?:_\d+)*)?/, Num::Float
-        rule %r/[+-]?\d+(?:_\d+)*[eE][+-]?\d+(?:_\d+)*/, Num::Float
-        rule %r/[+-]?(?:nan|inf)/, Num::Float
-
-        rule %r/0x\h+(?:_\h+)*/, Num::Hex
-        rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct
-        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
-        rule %r/[+-]?\d+(?:_\d+)*/, Num::Integer
-
         rule %r/"""/, Str, :mdq
         rule %r/"/, Str, :dq
         rule %r/'''/, Str, :msq
@@ -101,15 +106,31 @@ class TOML < RegexLexer
       end
 
       state :array do
-        mixin :content
+        mixin :whitespace
+        rule %r/,/, Punctuation
+
         rule %r/\]/, Punctuation, :pop!
+
+        rule %r//, Token, :value
       end
 
       state :inline do
-        mixin :content
+        rule %r/[ \t]+/, Text::Whitespace
 
+        mixin :key
+        rule %r/(=)(\s*)/ do
+          groups Punctuation, Text::Whitespace
+          push :value
+        end
+
+        rule %r/,/, Punctuation
         rule %r/\}/, Punctuation, :pop!
       end
+
+      state :whitespace do
+        rule %r/\s+/, Text
+        rule %r/#.*?$/, Comment
+      end
     end
   end
 end

spec/visual/samples/toml

@@ -6,13 +6,7 @@ title = "TOML Example"
 name = "Tom Preston-Werner"
 organization = "GitHub"
 bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
-# First class dates? Why not?
-dob = [
-	1979-05-27T07:32:00Z,
-	1979-05-27 07:32:00
-	1979-05-27,
-	07:32:00,
-]
+dob = 1979-05-27T07:32:00Z # First class dates? Why not?
 
 [database]
 server = "192.168.1.1"
@@ -135,7 +129,6 @@ nested_arrays_of_ints = [ [ 1, 2 ], [3, 4, 5] ]
 nested_mixed_array = [ [ 1, 2 ], ["a", "b", "c"] ]
 string_array = [ "all", 'strings', """are the same""", '''type''' ]
 link-libraries = ["mylib::mylib"]
-points = [ { x = 1, y = 2 }, { x = 3, y = 4 } ]
 
 # Dotted keys
 physical.color = "orange"
@@ -174,21 +167,10 @@ name = "Nail"
 sku = 284758393
 color = "gray"
 
-[special-floats]
-inflection = "don't highlight"
-influxdb = 1
-
-nan = nan
-
-inf-1 = inf
-inf-2 = [inf, +nan, -nan, +inf, -inf]
-inf-3 = {inf=inf, nan=nan}
-
-str-inf-1 = 'inf'
-str-inf-2 = "inf"
-str-inf-3 = """ inf nan
-	inf nan
-"""
-str-inf-4 = """ inf nan
-	inf nan
-"""
+[tool.poetry.dependencies]
+python = ">=3.10,<3.11"
+idna = "3.4"
+inflection = "0.5.1"
+influxdb = "5.3.1"
+requests = "2.28.1"
+pywinpty = {version = ">=1.1.0", markers = "os_name == \"nt\""}