53: def scan_tokens encoder, options
54:
55: value_expected = nil
56: states = [:initial]
57:
58: until eos?
59:
60: if match = scan(/\s+/)
61: encoder.text_token match, :space
62:
63: elsif case states.last
64: when :initial, :media
65: if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
66: encoder.text_token match, :type
67: next
68: elsif match = scan(RE::Class)
69: encoder.text_token match, :class
70: next
71: elsif match = scan(RE::Id)
72: encoder.text_token match, :constant
73: next
74: elsif match = scan(RE::PseudoClass)
75: encoder.text_token match, :pseudo_class
76: next
77: elsif match = scan(RE::AttributeSelector)
78:
79: encoder.text_token match[0,1], :operator
80: encoder.text_token match[1..-2], :attribute_name if match.size > 2
81: encoder.text_token match[-1,1], :operator if match[-1] == ?]
82: next
83: elsif match = scan(/@media/)
84: encoder.text_token match, :directive
85: states.push :media_before_name
86: next
87: end
88:
89: when :block
90: if match = scan(/(?>#{RE::Ident})(?!\()/ox)
91: if value_expected
92: encoder.text_token match, :value
93: else
94: encoder.text_token match, :key
95: end
96: next
97: end
98:
99: when :media_before_name
100: if match = scan(RE::Ident)
101: encoder.text_token match, :type
102: states[-1] = :media_after_name
103: next
104: end
105:
106: when :media_after_name
107: if match = scan(/\{/)
108: encoder.text_token match, :operator
109: states[-1] = :media
110: next
111: end
112:
113: else
114:
115: raise_inspect 'Unknown state', encoder
116:
117:
118: end
119:
120: elsif match = scan(/\/\*(?:.*?\*\/|\z)/m)
121: encoder.text_token match, :comment
122:
123: elsif match = scan(/\{/)
124: value_expected = false
125: encoder.text_token match, :operator
126: states.push :block
127:
128: elsif match = scan(/\}/)
129: value_expected = false
130: if states.last == :block || states.last == :media
131: encoder.text_token match, :operator
132: states.pop
133: else
134: encoder.text_token match, :error
135: end
136:
137: elsif match = scan(/#{RE::String}/o)
138: encoder.begin_group :string
139: encoder.text_token match[0, 1], :delimiter
140: encoder.text_token match[1..-2], :content if match.size > 2
141: encoder.text_token match[-1, 1], :delimiter if match.size >= 2
142: encoder.end_group :string
143:
144: elsif match = scan(/#{RE::Function}/o)
145: encoder.begin_group :string
146: start = match[/^\w+\(/]
147: encoder.text_token start, :delimiter
148: if match[-1] == ?)
149: encoder.text_token match[start.size..-2], :content
150: encoder.text_token ')', :delimiter
151: else
152: encoder.text_token match[start.size..-1], :content
153: end
154: encoder.end_group :string
155:
156: elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
157: encoder.text_token match, :float
158:
159: elsif match = scan(/#{RE::Color}/o)
160: encoder.text_token match, :color
161:
162: elsif match = scan(/! *important/)
163: encoder.text_token match, :important
164:
165: elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
166: encoder.text_token match, :color
167:
168: elsif match = scan(RE::AtKeyword)
169: encoder.text_token match, :directive
170:
171: elsif match = scan(/ [+>:;,.=()\/] /x)
172: if match == ':'
173: value_expected = true
174: elsif match == ';'
175: value_expected = false
176: end
177: encoder.text_token match, :operator
178:
179: else
180: encoder.text_token getch, :error
181:
182: end
183:
184: end
185:
186: encoder
187: end