1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
|
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<title>Write your own lexer — Pygments</title>
<meta http-equiv="content-type" content="text/html; charset=utf-8">
<style type="text/css">
body {
background-color: #f2f2f2;
margin: 0;
padding: 0;
font-family: 'Georgia', serif;
color: #111;
}
#content {
background-color: white;
padding: 20px;
margin: 20px auto 20px auto;
max-width: 800px;
border: 4px solid #ddd;
}
h1 {
font-weight: normal;
font-size: 40px;
color: #09839A;
}
h2 {
font-weight: normal;
font-size: 30px;
color: #C73F00;
}
h1.heading {
margin: 0 0 30px 0;
}
h2.subheading {
margin: -30px 0 0 45px;
}
h3 {
margin-top: 30px;
}
table.docutils {
border-collapse: collapse;
border: 2px solid #aaa;
margin: 0.5em 1.5em 0.5em 1.5em;
}
table.docutils td {
padding: 2px;
border: 1px solid #ddd;
}
p, li, dd, dt, blockquote {
font-size: 15px;
color: #333;
}
p {
line-height: 150%;
margin-bottom: 0;
margin-top: 10px;
}
hr {
border-top: 1px solid #ccc;
border-bottom: 0;
border-right: 0;
border-left: 0;
margin-bottom: 10px;
margin-top: 20px;
}
dl {
margin-left: 10px;
}
li, dt {
margin-top: 5px;
}
dt {
font-weight: bold;
}
th {
text-align: left;
}
a {
color: #990000;
}
a:hover {
color: #c73f00;
}
pre {
background-color: #f9f9f9;
border-top: 1px solid #ccc;
border-bottom: 1px solid #ccc;
padding: 5px;
font-size: 13px;
font-family: Bitstream Vera Sans Mono,monospace;
}
tt {
font-size: 13px;
font-family: Bitstream Vera Sans Mono,monospace;
color: black;
padding: 1px 2px 1px 2px;
background-color: #f0f0f0;
}
cite {
/* abusing <cite>, it's generated by ReST for `x` */
font-size: 13px;
font-family: Bitstream Vera Sans Mono,monospace;
font-weight: bold;
font-style: normal;
}
#backlink {
float: right;
font-size: 11px;
color: #888;
}
div.toc {
margin: 0 0 10px 0;
}
div.toc h2 {
font-size: 20px;
}
.syntax .hll { background-color: #ffffcc }
.syntax { background: #ffffff; }
.syntax .c { color: #888888 } /* Comment */
.syntax .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.syntax .k { color: #008800; font-weight: bold } /* Keyword */
.syntax .cm { color: #888888 } /* Comment.Multiline */
.syntax .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.syntax .c1 { color: #888888 } /* Comment.Single */
.syntax .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.syntax .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.syntax .ge { font-style: italic } /* Generic.Emph */
.syntax .gr { color: #aa0000 } /* Generic.Error */
.syntax .gh { color: #303030 } /* Generic.Heading */
.syntax .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.syntax .go { color: #888888 } /* Generic.Output */
.syntax .gp { color: #555555 } /* Generic.Prompt */
.syntax .gs { font-weight: bold } /* Generic.Strong */
.syntax .gu { color: #606060 } /* Generic.Subheading */
.syntax .gt { color: #aa0000 } /* Generic.Traceback */
.syntax .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.syntax .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.syntax .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.syntax .kp { color: #008800 } /* Keyword.Pseudo */
.syntax .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.syntax .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.syntax .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.syntax .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.syntax .na { color: #336699 } /* Name.Attribute */
.syntax .nb { color: #003388 } /* Name.Builtin */
.syntax .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.syntax .no { color: #003366; font-weight: bold } /* Name.Constant */
.syntax .nd { color: #555555 } /* Name.Decorator */
.syntax .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.syntax .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.syntax .nl { color: #336699; font-style: italic } /* Name.Label */
.syntax .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.syntax .py { color: #336699; font-weight: bold } /* Name.Property */
.syntax .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.syntax .nv { color: #336699 } /* Name.Variable */
.syntax .ow { color: #008800 } /* Operator.Word */
.syntax .w { color: #bbbbbb } /* Text.Whitespace */
.syntax .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.syntax .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.syntax .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.syntax .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.syntax .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.syntax .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.syntax .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.syntax .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.syntax .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.syntax .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.syntax .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.syntax .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.syntax .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.syntax .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.syntax .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.syntax .bp { color: #003388 } /* Name.Builtin.Pseudo */
.syntax .vc { color: #336699 } /* Name.Variable.Class */
.syntax .vg { color: #dd7700 } /* Name.Variable.Global */
.syntax .vi { color: #3333bb } /* Name.Variable.Instance */
.syntax .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
</style>
</head>
<body>
<div id="content">
<h1 class="heading">Pygments</h1>
<h2 class="subheading">Write your own lexer</h2>
<a id="backlink" href="index.html">« Back To Index</a>
<div class="toc">
<h2>Contents</h2>
<ul class="contents">
<li><a href="#regexlexer">RegexLexer</a></li>
<li><a href="#regex-flags">Regex Flags</a></li>
<li><a href="#scanning-multiple-tokens-at-once">Scanning multiple tokens at once</a></li>
<li><a href="#changing-states">Changing states</a></li>
<li><a href="#advanced-state-tricks">Advanced state tricks</a></li>
<li><a href="#using-multiple-lexers">Using multiple lexers</a></li>
<li><a href="#delegating-lexer">Delegating Lexer</a></li>
<li><a href="#callbacks">Callbacks</a></li>
<li><a href="#the-extendedregexlexer-class">The ExtendedRegexLexer class</a></li>
<li><a href="#filtering-token-streams">Filtering Token Streams</a></li>
</ul>
</div>
<!-- -*- mode: rst -*- -->
<p>If a lexer for your favorite language is missing in the Pygments package, you can
easily write your own and extend Pygments.</p>
<p>All you need can be found inside the <cite>pygments.lexer</cite> module. As you can read in
the <a class="reference external" href="./api.html">API documentation</a>, a lexer is a class that is initialized with
some keyword arguments (the lexer options) and that provides a
<cite>get_tokens_unprocessed()</cite> method which is given a string or unicode object with
the data to parse.</p>
<p>The <cite>get_tokens_unprocessed()</cite> method must return an iterator or iterable
containing tuples in the form <tt class="docutils literal">(index, token, value)</tt>. Normally you don't need
to do this since there are numerous base lexers you can subclass.</p>
<div class="section" id="regexlexer">
<h3>RegexLexer</h3>
<p>A very powerful (but quite easy to use) lexer is the <cite>RegexLexer</cite>. This lexer
base class allows you to define lexing rules in terms of <em>regular expressions</em>
for different <em>states</em>.</p>
<p>States are groups of regular expressions that are matched against the input
string at the <em>current position</em>. If one of these expressions matches, a
corresponding action is performed (normally yielding a token with a specific
type), the current position is set to where the last match ended and the
matching process continues with the first regex of the current state.</p>
<p>Lexer states are kept in a state stack: each time a new state is entered, the
new state is pushed onto the stack. The most basic lexers (like the
<cite>DiffLexer</cite>) just need one state.</p>
<p>Each state is defined as a list of tuples in the form (<cite>regex</cite>, <cite>action</cite>,
<cite>new_state</cite>) where the last item is optional. In the most basic form, <cite>action</cite>
is a token type (like <cite>Name.Builtin</cite>). That means: When <cite>regex</cite> matches, emit a
token with the match text and type <cite>tokentype</cite> and push <cite>new_state</cite> on the state
stack. If the new state is <tt class="docutils literal">'#pop'</tt>, the topmost state is popped from the
stack instead. (To pop more than one state, use <tt class="docutils literal">'#pop:2'</tt> and so on.)
<tt class="docutils literal">'#push'</tt> is a synonym for pushing the current state on the
stack.</p>
<p>The following example shows the <cite>DiffLexer</cite> from the builtin lexers. Note that
it contains some additional attributes <cite>name</cite>, <cite>aliases</cite> and <cite>filenames</cite> which
aren't required for a lexer. They are used by the builtin lexer lookup
functions.</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">RegexLexer</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="o">*</span>
<span class="k">class</span> <span class="nc">DiffLexer</span><span class="p">(</span><span class="n">RegexLexer</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s">'Diff'</span>
<span class="n">aliases</span> <span class="o">=</span> <span class="p">[</span><span class="s">'diff'</span><span class="p">]</span>
<span class="n">filenames</span> <span class="o">=</span> <span class="p">[</span><span class="s">'*.diff'</span><span class="p">]</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r' .*\n'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'\+.*\n'</span><span class="p">,</span> <span class="n">Generic</span><span class="o">.</span><span class="n">Inserted</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'-.*\n'</span><span class="p">,</span> <span class="n">Generic</span><span class="o">.</span><span class="n">Deleted</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'@.*\n'</span><span class="p">,</span> <span class="n">Generic</span><span class="o">.</span><span class="n">Subheading</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'Index.*\n'</span><span class="p">,</span> <span class="n">Generic</span><span class="o">.</span><span class="n">Heading</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'=.*\n'</span><span class="p">,</span> <span class="n">Generic</span><span class="o">.</span><span class="n">Heading</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'.*\n'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>As you can see this lexer only uses one state. When the lexer starts scanning
the text, it first checks if the current character is a space. If this is true
it scans everything until newline and returns the parsed data as <cite>Text</cite> token.</p>
<p>If this rule doesn't match, it checks if the current char is a plus sign. And
so on.</p>
<p>If no rule matches at the current position, the current char is emitted as an
<cite>Error</cite> token that indicates a parsing error, and the position is increased by
1.</p>
</div>
<div class="section" id="regex-flags">
<h3>Regex Flags</h3>
<p>You can either define regex flags in the regex (<tt class="docutils literal"><span class="pre">r'(?x)foo</span> bar'</tt>) or by adding
a <cite>flags</cite> attribute to your lexer class. If no attribute is defined, it defaults
to <cite>re.MULTILINE</cite>. For more informations about regular expression flags see the
<a class="reference external" href="http://docs.python.org/lib/re-syntax.html">regular expressions</a> help page in the python documentation.</p>
</div>
<div class="section" id="scanning-multiple-tokens-at-once">
<h3>Scanning multiple tokens at once</h3>
<p>Here is a more complex lexer that highlights INI files. INI files consist of
sections, comments and key = value pairs:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">RegexLexer</span><span class="p">,</span> <span class="n">bygroups</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="o">*</span>
<span class="k">class</span> <span class="nc">IniLexer</span><span class="p">(</span><span class="n">RegexLexer</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s">'INI'</span>
<span class="n">aliases</span> <span class="o">=</span> <span class="p">[</span><span class="s">'ini'</span><span class="p">,</span> <span class="s">'cfg'</span><span class="p">]</span>
<span class="n">filenames</span> <span class="o">=</span> <span class="p">[</span><span class="s">'*.ini'</span><span class="p">,</span> <span class="s">'*.cfg'</span><span class="p">]</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'\s+'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="p">(</span><span class="s">r';.*?$'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'\[.*?\]$'</span><span class="p">,</span> <span class="n">Keyword</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'(.*?)(\s*)(=)(\s*)(.*?)$'</span><span class="p">,</span>
<span class="n">bygroups</span><span class="p">(</span><span class="n">Name</span><span class="o">.</span><span class="n">Attribute</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Operator</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">String</span><span class="p">))</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>The lexer first looks for whitespace, comments and section names. And later it
looks for a line that looks like a key, value pair, seperated by an <tt class="docutils literal">'='</tt>
sign, and optional whitespace.</p>
<p>The <cite>bygroups</cite> helper makes sure that each group is yielded with a different
token type. First the <cite>Name.Attribute</cite> token, then a <cite>Text</cite> token for the
optional whitespace, after that a <cite>Operator</cite> token for the equals sign. Then a
<cite>Text</cite> token for the whitespace again. The rest of the line is returned as
<cite>String</cite>.</p>
<p>Note that for this to work, every part of the match must be inside a capturing
group (a <tt class="docutils literal"><span class="pre">(...)</span></tt>), and there must not be any nested capturing groups. If you
nevertheless need a group, use a non-capturing group defined using this syntax:
<tt class="docutils literal"><span class="pre">r'(?:some|words|here)'</span></tt> (note the <tt class="docutils literal"><span class="pre">?:</span></tt> after the beginning parenthesis).</p>
<p>If you find yourself needing a capturing group inside the regex which
shouldn't be part of the output but is used in the regular expressions for
backreferencing (eg: <tt class="docutils literal"><span class="pre">r'(<(foo|bar)>)(.*?)(</\2>)'</span></tt>), you can pass <cite>None</cite>
to the bygroups function and it will skip that group will be skipped in the
output.</p>
</div>
<div class="section" id="changing-states">
<h3>Changing states</h3>
<p>Many lexers need multiple states to work as expected. For example, some
languages allow multiline comments to be nested. Since this is a recursive
pattern it's impossible to lex just using regular expressions.</p>
<p>Here is the solution:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">RegexLexer</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="o">*</span>
<span class="k">class</span> <span class="nc">ExampleLexer</span><span class="p">(</span><span class="n">RegexLexer</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s">'Example Lexer with states'</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'[^/]+'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'/\*'</span><span class="p">,</span> <span class="n">Comment</span><span class="o">.</span><span class="n">Multiline</span><span class="p">,</span> <span class="s">'comment'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'//.*?$'</span><span class="p">,</span> <span class="n">Comment</span><span class="o">.</span><span class="n">Singleline</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'/'</span><span class="p">,</span> <span class="n">Text</span><span class="p">)</span>
<span class="p">],</span>
<span class="s">'comment'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'[^*/]'</span><span class="p">,</span> <span class="n">Comment</span><span class="o">.</span><span class="n">Multiline</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'/\*'</span><span class="p">,</span> <span class="n">Comment</span><span class="o">.</span><span class="n">Multiline</span><span class="p">,</span> <span class="s">'#push'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'\*/'</span><span class="p">,</span> <span class="n">Comment</span><span class="o">.</span><span class="n">Multiline</span><span class="p">,</span> <span class="s">'#pop'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'[*/]'</span><span class="p">,</span> <span class="n">Comment</span><span class="o">.</span><span class="n">Multiline</span><span class="p">)</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>This lexer starts lexing in the <tt class="docutils literal">'root'</tt> state. It tries to match as much as
possible until it finds a slash (<tt class="docutils literal">'/'</tt>). If the next character after the slash
is a star (<tt class="docutils literal">'*'</tt>) the <cite>RegexLexer</cite> sends those two characters to the output
stream marked as <cite>Comment.Multiline</cite> and continues parsing with the rules
defined in the <tt class="docutils literal">'comment'</tt> state.</p>
<p>If there wasn't a star after the slash, the <cite>RegexLexer</cite> checks if it's a
singleline comment (eg: followed by a second slash). If this also wasn't the
case it must be a single slash (the separate regex for a single slash must also
be given, else the slash would be marked as an error token).</p>
<p>Inside the <tt class="docutils literal">'comment'</tt> state, we do the same thing again. Scan until the lexer
finds a star or slash. If it's the opening of a multiline comment, push the
<tt class="docutils literal">'comment'</tt> state on the stack and continue scanning, again in the
<tt class="docutils literal">'comment'</tt> state. Else, check if it's the end of the multiline comment. If
yes, pop one state from the stack.</p>
<p>Note: If you pop from an empty stack you'll get an <cite>IndexError</cite>. (There is an
easy way to prevent this from happening: don't <tt class="docutils literal">'#pop'</tt> in the root state).</p>
<p>If the <cite>RegexLexer</cite> encounters a newline that is flagged as an error token, the
stack is emptied and the lexer continues scanning in the <tt class="docutils literal">'root'</tt> state. This
helps producing error-tolerant highlighting for erroneous input, e.g. when a
single-line string is not closed.</p>
</div>
<div class="section" id="advanced-state-tricks">
<h3>Advanced state tricks</h3>
<p>There are a few more things you can do with states:</p>
<ul>
<li><p class="first">You can push multiple states onto the stack if you give a tuple instead of a
simple string as the third item in a rule tuple. For example, if you want to
match a comment containing a directive, something like:</p>
<pre class="literal-block">
/* <processing directive> rest of comment */
</pre>
<p>you can use this rule:</p>
<div class="syntax"><pre><span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'/\* <'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">,</span> <span class="p">(</span><span class="s">'comment'</span><span class="p">,</span> <span class="s">'directive'</span><span class="p">)),</span>
<span class="o">...</span>
<span class="p">],</span>
<span class="s">'directive'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'[^>]*'</span><span class="p">,</span> <span class="n">Comment</span><span class="o">.</span><span class="n">Directive</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'>'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">,</span> <span class="s">'#pop'</span><span class="p">),</span>
<span class="p">],</span>
<span class="s">'comment'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'[^*]+'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'\*/'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">,</span> <span class="s">'#pop'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'\*'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">),</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>When this encounters the above sample, first <tt class="docutils literal">'comment'</tt> and <tt class="docutils literal">'directive'</tt>
are pushed onto the stack, then the lexer continues in the directive state
until it finds the closing <tt class="docutils literal">></tt>, then it continues in the comment state until
the closing <tt class="docutils literal">*/</tt>. Then, both states are popped from the stack again and
lexing continues in the root state.</p>
<p><em>New in Pygments 0.9:</em> The tuple can contain the special <tt class="docutils literal">'#push'</tt> and
<tt class="docutils literal">'#pop'</tt> (but not <tt class="docutils literal">'#pop:n'</tt>) directives.</p>
</li>
<li><p class="first">You can include the rules of a state in the definition of another. This is
done by using <cite>include</cite> from <cite>pygments.lexer</cite>:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">RegexLexer</span><span class="p">,</span> <span class="n">bygroups</span><span class="p">,</span> <span class="n">include</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="o">*</span>
<span class="k">class</span> <span class="nc">ExampleLexer</span><span class="p">(</span><span class="n">RegexLexer</span><span class="p">):</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'comments'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'/\*.*?\*/'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'//.*?\n'</span><span class="p">,</span> <span class="n">Comment</span><span class="p">),</span>
<span class="p">],</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="n">include</span><span class="p">(</span><span class="s">'comments'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'(function )(\w+)( {)'</span><span class="p">,</span>
<span class="n">bygroups</span><span class="p">(</span><span class="n">Keyword</span><span class="p">,</span> <span class="n">Name</span><span class="p">,</span> <span class="n">Keyword</span><span class="p">),</span> <span class="s">'function'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'.'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="p">],</span>
<span class="s">'function'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'[^}/]+'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="n">include</span><span class="p">(</span><span class="s">'comments'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'/'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'}'</span><span class="p">,</span> <span class="n">Keyword</span><span class="p">,</span> <span class="s">'#pop'</span><span class="p">),</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>This is a hypothetical lexer for a language that consist of functions and
comments. Because comments can occur at toplevel and in functions, we need
rules for comments in both states. As you can see, the <cite>include</cite> helper saves
repeating rules that occur more than once (in this example, the state
<tt class="docutils literal">'comment'</tt> will never be entered by the lexer, as it's only there to be
included in <tt class="docutils literal">'root'</tt> and <tt class="docutils literal">'function'</tt>).</p>
</li>
<li><p class="first">Sometimes, you may want to "combine" a state from existing ones. This is
possible with the <cite>combine</cite> helper from <cite>pygments.lexer</cite>.</p>
<p>If you, instead of a new state, write <tt class="docutils literal"><span class="pre">combined('state1',</span> 'state2')</tt> as the
third item of a rule tuple, a new anonymous state will be formed from state1
and state2 and if the rule matches, the lexer will enter this state.</p>
<p>This is not used very often, but can be helpful in some cases, such as the
<cite>PythonLexer</cite>'s string literal processing.</p>
</li>
<li><p class="first">If you want your lexer to start lexing in a different state you can modify
the stack by overloading the <cite>get_tokens_unprocessed()</cite> method:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">RegexLexer</span>
<span class="k">class</span> <span class="nc">MyLexer</span><span class="p">(</span><span class="n">RegexLexer</span><span class="p">):</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span><span class="o">...</span><span class="p">}</span>
<span class="k">def</span> <span class="nf">get_tokens_unprocessed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">):</span>
<span class="n">stack</span> <span class="o">=</span> <span class="p">[</span><span class="s">'root'</span><span class="p">,</span> <span class="s">'otherstate'</span><span class="p">]</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">RegexLexer</span><span class="o">.</span><span class="n">get_tokens_unprocessed</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">stack</span><span class="p">):</span>
<span class="k">yield</span> <span class="n">item</span>
</pre></div>
<p>Some lexers like the <cite>PhpLexer</cite> use this to make the leading <tt class="docutils literal"><span class="pre"><?php</span></tt>
preprocessor comments optional. Note that you can crash the lexer easily
by putting values into the stack that don't exist in the token map. Also
removing <tt class="docutils literal">'root'</tt> from the stack can result in strange errors!</p>
</li>
<li><p class="first">An empty regex at the end of a state list, combined with <tt class="docutils literal">'#pop'</tt>, can
act as a return point from a state that doesn't have a clear end marker.</p>
</li>
</ul>
</div>
<div class="section" id="using-multiple-lexers">
<h3>Using multiple lexers</h3>
<p>Using multiple lexers for the same input can be tricky. One of the easiest
combination techniques is shown here: You can replace the token type entry in a
rule tuple (the second item) with a lexer class. The matched text will then be
lexed with that lexer, and the resulting tokens will be yielded.</p>
<p>For example, look at this stripped-down HTML lexer:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">RegexLexer</span><span class="p">,</span> <span class="n">bygroups</span><span class="p">,</span> <span class="n">using</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="o">*</span>
<span class="kn">from</span> <span class="nn">pygments.lexers.web</span> <span class="kn">import</span> <span class="n">JavascriptLexer</span>
<span class="k">class</span> <span class="nc">HtmlLexer</span><span class="p">(</span><span class="n">RegexLexer</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s">'HTML'</span>
<span class="n">aliases</span> <span class="o">=</span> <span class="p">[</span><span class="s">'html'</span><span class="p">]</span>
<span class="n">filenames</span> <span class="o">=</span> <span class="p">[</span><span class="s">'*.html'</span><span class="p">,</span> <span class="s">'*.htm'</span><span class="p">]</span>
<span class="n">flags</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">IGNORECASE</span> <span class="o">|</span> <span class="n">re</span><span class="o">.</span><span class="n">DOTALL</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">'[^<&]+'</span><span class="p">,</span> <span class="n">Text</span><span class="p">),</span>
<span class="p">(</span><span class="s">'&.*?;'</span><span class="p">,</span> <span class="n">Name</span><span class="o">.</span><span class="n">Entity</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'<\s*script\s*'</span><span class="p">,</span> <span class="n">Name</span><span class="o">.</span><span class="n">Tag</span><span class="p">,</span> <span class="p">(</span><span class="s">'script-content'</span><span class="p">,</span> <span class="s">'tag'</span><span class="p">)),</span>
<span class="p">(</span><span class="s">r'<\s*[a-zA-Z0-9:]+'</span><span class="p">,</span> <span class="n">Name</span><span class="o">.</span><span class="n">Tag</span><span class="p">,</span> <span class="s">'tag'</span><span class="p">),</span>
<span class="p">(</span><span class="s">r'<\s*/\s*[a-zA-Z0-9:]+\s*>'</span><span class="p">,</span> <span class="n">Name</span><span class="o">.</span><span class="n">Tag</span><span class="p">),</span>
<span class="p">],</span>
<span class="s">'script-content'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'(.+?)(<\s*/\s*script\s*>)'</span><span class="p">,</span>
<span class="n">bygroups</span><span class="p">(</span><span class="n">using</span><span class="p">(</span><span class="n">JavascriptLexer</span><span class="p">),</span> <span class="n">Name</span><span class="o">.</span><span class="n">Tag</span><span class="p">),</span>
<span class="s">'#pop'</span><span class="p">),</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>Here the content of a <tt class="docutils literal"><script></tt> tag is passed to a newly created instance of
a <cite>JavascriptLexer</cite> and not processed by the <cite>HtmlLexer</cite>. This is done using the
<cite>using</cite> helper that takes the other lexer class as its parameter.</p>
<p>Note the combination of <cite>bygroups</cite> and <cite>using</cite>. This makes sure that the content
up to the <tt class="docutils literal"></script></tt> end tag is processed by the <cite>JavascriptLexer</cite>, while the
end tag is yielded as a normal token with the <cite>Name.Tag</cite> type.</p>
<p>As an additional goodie, if the lexer class is replaced by <cite>this</cite> (imported from
<cite>pygments.lexer</cite>), the "other" lexer will be the current one (because you cannot
refer to the current class within the code that runs at class definition time).</p>
<p>Also note the <tt class="docutils literal"><span class="pre">(r'<\s*script\s*',</span> Name.Tag, <span class="pre">('script-content',</span> <span class="pre">'tag'))</span></tt> rule.
Here, two states are pushed onto the state stack, <tt class="docutils literal"><span class="pre">'script-content'</span></tt> and
<tt class="docutils literal">'tag'</tt>. That means that first <tt class="docutils literal">'tag'</tt> is processed, which will parse
attributes and the closing <tt class="docutils literal">></tt>, then the <tt class="docutils literal">'tag'</tt> state is popped and the
next state on top of the stack will be <tt class="docutils literal"><span class="pre">'script-content'</span></tt>.</p>
<p>The <cite>using()</cite> helper has a special keyword argument, <cite>state</cite>, which works as
follows: if given, the lexer to use initially is not in the <tt class="docutils literal">"root"</tt> state,
but in the state given by this argument. This <em>only</em> works with a <cite>RegexLexer</cite>.</p>
<p>Any other keywords arguments passed to <cite>using()</cite> are added to the keyword
arguments used to create the lexer.</p>
</div>
<div class="section" id="delegating-lexer">
<h3>Delegating Lexer</h3>
<p>Another approach for nested lexers is the <cite>DelegatingLexer</cite> which is for
example used for the template engine lexers. It takes two lexers as
arguments on initialisation: a <cite>root_lexer</cite> and a <cite>language_lexer</cite>.</p>
<p>The input is processed as follows: First, the whole text is lexed with the
<cite>language_lexer</cite>. All tokens yielded with a type of <tt class="docutils literal">Other</tt> are then
concatenated and given to the <cite>root_lexer</cite>. The language tokens of the
<cite>language_lexer</cite> are then inserted into the <cite>root_lexer</cite>'s token stream
at the appropriate positions.</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">DelegatingLexer</span>
<span class="kn">from</span> <span class="nn">pygments.lexers.web</span> <span class="kn">import</span> <span class="n">HtmlLexer</span><span class="p">,</span> <span class="n">PhpLexer</span>
<span class="k">class</span> <span class="nc">HtmlPhpLexer</span><span class="p">(</span><span class="n">DelegatingLexer</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">HtmlPhpLexer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">HtmlLexer</span><span class="p">,</span> <span class="n">PhpLexer</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">)</span>
</pre></div>
<p>This procedure ensures that e.g. HTML with template tags in it is highlighted
correctly even if the template tags are put into HTML tags or attributes.</p>
<p>If you want to change the needle token <tt class="docutils literal">Other</tt> to something else, you can
give the lexer another token type as the third parameter:</p>
<div class="syntax"><pre><span class="n">DelegatingLexer</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">MyLexer</span><span class="p">,</span> <span class="n">OtherLexer</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">)</span>
</pre></div>
</div>
<div class="section" id="callbacks">
<h3>Callbacks</h3>
<p>Sometimes the grammar of a language is so complex that a lexer would be unable
to parse it just by using regular expressions and stacks.</p>
<p>For this, the <cite>RegexLexer</cite> allows callbacks to be given in rule tuples, instead
of token types (<cite>bygroups</cite> and <cite>using</cite> are nothing else but preimplemented
callbacks). The callback must be a function taking two arguments:</p>
<ul class="simple">
<li>the lexer itself</li>
<li>the match object for the last matched rule</li>
</ul>
<p>The callback must then return an iterable of (or simply yield) <tt class="docutils literal">(index,
tokentype, value)</tt> tuples, which are then just passed through by
<cite>get_tokens_unprocessed()</cite>. The <tt class="docutils literal">index</tt> here is the position of the token in
the input string, <tt class="docutils literal">tokentype</tt> is the normal token type (like <cite>Name.Builtin</cite>),
and <tt class="docutils literal">value</tt> the associated part of the input string.</p>
<p>You can see an example here:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">RegexLexer</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="n">Generic</span>
<span class="k">class</span> <span class="nc">HypotheticLexer</span><span class="p">(</span><span class="n">RegexLexer</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">headline_callback</span><span class="p">(</span><span class="n">lexer</span><span class="p">,</span> <span class="n">match</span><span class="p">):</span>
<span class="n">equal_signs</span> <span class="o">=</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">text</span> <span class="o">=</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="k">yield</span> <span class="n">match</span><span class="o">.</span><span class="n">start</span><span class="p">(),</span> <span class="n">Generic</span><span class="o">.</span><span class="n">Headline</span><span class="p">,</span> <span class="n">equal_signs</span> <span class="o">+</span> <span class="n">text</span> <span class="o">+</span> <span class="n">equal_signs</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'(=+)(.*?)(\1)'</span><span class="p">,</span> <span class="n">headline_callback</span><span class="p">)</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>If the regex for the <cite>headline_callback</cite> matches, the function is called with the
match object. Note that after the callback is done, processing continues
normally, that is, after the end of the previous match. The callback has no
possibility to influence the position.</p>
<p>There are not really any simple examples for lexer callbacks, but you can see
them in action e.g. in the <a class="reference external" href="http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/compiled.py">compiled.py</a> source code in the <cite>CLexer</cite> and
<cite>JavaLexer</cite> classes.</p>
</div>
<div class="section" id="the-extendedregexlexer-class">
<h3>The ExtendedRegexLexer class</h3>
<p>The <cite>RegexLexer</cite>, even with callbacks, unfortunately isn't powerful enough for
the funky syntax rules of some languages that will go unnamed, such as Ruby.</p>
<p>But fear not; even then you don't have to abandon the regular expression
approach. For Pygments has a subclass of <cite>RegexLexer</cite>, the <cite>ExtendedRegexLexer</cite>.
All features known from RegexLexers are available here too, and the tokens are
specified in exactly the same way, <em>except</em> for one detail:</p>
<p>The <cite>get_tokens_unprocessed()</cite> method holds its internal state data not as local
variables, but in an instance of the <cite>pygments.lexer.LexerContext</cite> class, and
that instance is passed to callbacks as a third argument. This means that you
can modify the lexer state in callbacks.</p>
<p>The <cite>LexerContext</cite> class has the following members:</p>
<ul class="simple">
<li><cite>text</cite> -- the input text</li>
<li><cite>pos</cite> -- the current starting position that is used for matching regexes</li>
<li><cite>stack</cite> -- a list containing the state stack</li>
<li><cite>end</cite> -- the maximum position to which regexes are matched, this defaults to
the length of <cite>text</cite></li>
</ul>
<p>Additionally, the <cite>get_tokens_unprocessed()</cite> method can be given a
<cite>LexerContext</cite> instead of a string and will then process this context instead of
creating a new one for the string argument.</p>
<p>Note that because you can set the current position to anything in the callback,
it won't be automatically be set by the caller after the callback is finished.
For example, this is how the hypothetical lexer above would be written with the
<cite>ExtendedRegexLexer</cite>:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexer</span> <span class="kn">import</span> <span class="n">ExtendedRegexLexer</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="n">Generic</span>
<span class="k">class</span> <span class="nc">ExHypotheticLexer</span><span class="p">(</span><span class="n">ExtendedRegexLexer</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">headline_callback</span><span class="p">(</span><span class="n">lexer</span><span class="p">,</span> <span class="n">match</span><span class="p">,</span> <span class="n">ctx</span><span class="p">):</span>
<span class="n">equal_signs</span> <span class="o">=</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">text</span> <span class="o">=</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="k">yield</span> <span class="n">match</span><span class="o">.</span><span class="n">start</span><span class="p">(),</span> <span class="n">Generic</span><span class="o">.</span><span class="n">Headline</span><span class="p">,</span> <span class="n">equal_signs</span> <span class="o">+</span> <span class="n">text</span> <span class="o">+</span> <span class="n">equal_signs</span>
<span class="n">ctx</span><span class="o">.</span><span class="n">pos</span> <span class="o">=</span> <span class="n">match</span><span class="o">.</span><span class="n">end</span><span class="p">()</span>
<span class="n">tokens</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">'root'</span><span class="p">:</span> <span class="p">[</span>
<span class="p">(</span><span class="s">r'(=+)(.*?)(\1)'</span><span class="p">,</span> <span class="n">headline_callback</span><span class="p">)</span>
<span class="p">]</span>
<span class="p">}</span>
</pre></div>
<p>This might sound confusing (and it can really be). But it is needed, and for an
example look at the Ruby lexer in <a class="reference external" href="http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py">agile.py</a>.</p>
</div>
<div class="section" id="filtering-token-streams">
<h3>Filtering Token Streams</h3>
<p>Some languages ship a lot of builtin functions (for example PHP). The total
amount of those functions differs from system to system because not everybody
has every extension installed. In the case of PHP there are over 3000 builtin
functions. That's an incredible huge amount of functions, much more than you
can put into a regular expression.</p>
<p>But because only <cite>Name</cite> tokens can be function names it's solvable by overriding
the <tt class="docutils literal">get_tokens_unprocessed()</tt> method. The following lexer subclasses the
<cite>PythonLexer</cite> so that it highlights some additional names as pseudo keywords:</p>
<div class="syntax"><pre><span class="kn">from</span> <span class="nn">pygments.lexers.agile</span> <span class="kn">import</span> <span class="n">PythonLexer</span>
<span class="kn">from</span> <span class="nn">pygments.token</span> <span class="kn">import</span> <span class="n">Name</span><span class="p">,</span> <span class="n">Keyword</span>
<span class="k">class</span> <span class="nc">MyPythonLexer</span><span class="p">(</span><span class="n">PythonLexer</span><span class="p">):</span>
<span class="n">EXTRA_KEYWORDS</span> <span class="o">=</span> <span class="p">[</span><span class="s">'foo'</span><span class="p">,</span> <span class="s">'bar'</span><span class="p">,</span> <span class="s">'foobar'</span><span class="p">,</span> <span class="s">'barfoo'</span><span class="p">,</span> <span class="s">'spam'</span><span class="p">,</span> <span class="s">'eggs'</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">get_tokens_unprocessed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">):</span>
<span class="k">for</span> <span class="n">index</span><span class="p">,</span> <span class="n">token</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">PythonLexer</span><span class="o">.</span><span class="n">get_tokens_unprocessed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">):</span>
<span class="k">if</span> <span class="n">token</span> <span class="ow">is</span> <span class="n">Name</span> <span class="ow">and</span> <span class="n">value</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">EXTRA_KEYWORDS</span><span class="p">:</span>
<span class="k">yield</span> <span class="n">index</span><span class="p">,</span> <span class="n">Keyword</span><span class="o">.</span><span class="n">Pseudo</span><span class="p">,</span> <span class="n">value</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">yield</span> <span class="n">index</span><span class="p">,</span> <span class="n">token</span><span class="p">,</span> <span class="n">value</span>
</pre></div>
<p>The <cite>PhpLexer</cite> and <cite>LuaLexer</cite> use this method to resolve builtin functions.</p>
<p><strong>Note</strong> Do not confuse this with the <a class="reference external" href="./filters.html">filter</a> system.</p>
</div>
</div>
</body>
<!-- generated on: 2010-01-02 22:27:14.208796
file id: lexerdevelopment -->
</html>
|
