Debian Codegraph @

lib/Text/Hogan/Compiler.pm

raw
1 package Text::Hogan::Compiler;
2 $Text::Hogan::Compiler::VERSION = '2.04';
3 use Text::Hogan::Template;
4
5 use 5.10.0;
6 use strict;
7 use warnings;
8
9 use Ref::Util qw( is_arrayref );
10 use Text::Trim 'trim';
11
12 my $r_is_whitespace = qr/\S/;
13 my $r_quot = qr/"/;
14 my $r_newline = qr/\n/;
15 my $r_cr = qr/\r/;
16 my $r_slash = qr/\\/;
17
18 my $linesep = "\u{2028}";
19 my $paragraphsep = "\u{2029}";
20 my $r_linesep = qr/\Q$linesep\E/;
21 my $r_paragraphsep = qr/\Q$paragraphsep\E/;
22
23 my %tags = (
24 '#' => 1, '^' => 2, '<' => 3, '$' => 4,
25 '/' => 5, '!' => 6, '>' => 7, '=' => 8, '_v' => 9,
26 '{' => 10, '&' => 11, '_t' => 12
27 );
28
29 my $Template = Text::Hogan::Template->new();
30
31 sub new {
32 my $class = shift;
33 return bless {}, $class;
34 }
35
36 sub scan {
37 my ($self, $text_orig, $options) = @_;
38 my $text = [ split //, $text_orig ];
39
40 my $len = scalar(@$text);
41 my ($IN_TEXT, $IN_TAG_TYPE, $IN_TAG) = (0, 1, 2);
42 my $state = $IN_TEXT;
43 my $tag_type = undef;
44 my $tag = undef;
45 my $buf = "";
46 my @tokens;
47 my $seen_tag = 0;
48 my $i = 0;
49 my $line_start = 0;
50 my ($otag, $ctag) = ('{{', '}}');
51
52 my $add_buf = sub {
53 if (length $buf > 0) {
54 push @tokens, { 'tag' => '_t', 'text' => $buf };
55 $buf = "";
56 }
57 };
58
59 my $line_is_whitespace = sub {
60 my $is_all_whitespace = 1;
61 for (my $j = $line_start; $j < @tokens; $j++) {
62 $is_all_whitespace =
63 ($tags{$tokens[$j]{'tag'}} < $tags{'_v'}) ||
64 ($tokens[$j]{'tag'} eq '_t' && $tokens[$j]{'text'} !~ $r_is_whitespace);
65 if (!$is_all_whitespace) {
66 return 0;
67 }
68 }
69 return $is_all_whitespace;
70 };
71
72 my $filter_line = sub {
73 my ($have_seen_tag, $no_new_line) = @_;
74
75 $add_buf->();
76
77 if ($have_seen_tag && $line_is_whitespace->()) {
78 for (my $j = $line_start, my $next; $j < @tokens; $j++) {
79 if ($tokens[$j]{'text'}) {
80 if (($next = $tokens[$j+1]) && $next->{'tag'} eq '>') {
81 $next->{'indent'} = "".$tokens[$j]{'text'};
82 }
83 splice(@tokens,$j,1);
84 }
85 }
86 }
87 elsif (!$no_new_line) {
88 push @tokens, { 'tag' => "\n" };
89 }
90
91 $seen_tag = 0;
92 $line_start = scalar @tokens;
93 };
94
95 my $change_delimiters = sub {
96 my ($text_orig, $index) = @_;
97 my $text = join('', @$text_orig);
98
99 my $close = '=' . $ctag;
100 my $close_index = index($text, $close, $index);
101 my $offset = index($text, '=', $index) + 1;
102 my @delimiters = (
103 split ' ', trim(
104 # WARNING
105 #
106 # JavaScript substring and Perl substring functions differ!
107 #
108 # JavaScript's length parameter always goes from the beginning
109 # of the string, whereas Perl's goes from the offset parameter!
110 #
111 # node> "{{=<% %>=}}".substring(3, 8)
112 # '<% %>'
113 #
114 # perl> substr("{{=<% %>=}}", 3, 8);
115 # <% %>=}}
116 #
117 #
118 substr($text, $offset, $close_index - $offset)
119 )
120 );
121
122 $otag = $delimiters[0];
123 $ctag = $delimiters[-1];
124
125 return $close_index + length($close) - 1;
126 };
127
128 if ($options->{'delimiters'}) {
129 my @delimiters = split ' ', $options->{'delimiters'};
130 $otag = $delimiters[0];
131 $ctag = $delimiters[1];
132 }
133
134 for (my $i = 0; $i < $len; $i++) {
135 if ($state eq $IN_TEXT) {
136 if (tag_change($otag, $text, $i)) {
137 --$i;
138 $add_buf->();
139 $state = $IN_TAG_TYPE;
140 }
141 else {
142 if (char_at($text, $i) eq "\n") {
143 $filter_line->($seen_tag);
144 }
145 else {
146 $buf .= char_at($text, $i);
147 }
148 }
149 }
150 elsif ($state eq $IN_TAG_TYPE) {
151 $i += length($otag) - 1;
152 $i += 1 while($options->{'allow_whitespace_before_hashmark'} and (char_at($text, $i+1) eq ' '));
153 $tag = $tags{char_at($text,$i + 1)};
154 $tag_type = $tag ? char_at($text, $i + 1) : '_v';
155 if ($tag_type eq '=') {
156 $i = $change_delimiters->($text, $i);
157 $state = $IN_TEXT;
158 }
159 else {
160 if ($tag) {
161 $i++;
162 }
163 $state = $IN_TAG;
164 }
165 $seen_tag = $i;
166 }
167 else {
168 if (tag_change($ctag, $text, $i)) {
169 push @tokens, {
170 'tag' => $tag_type,
171 'n' => trim($buf),
172 'otag' => $otag,
173 'ctag' => $ctag,
174 'i' => (($tag_type eq '/') ? $seen_tag - length($otag) : $i + length($ctag)),
175 };
176 $buf = "";
177 $i += length($ctag) - 1;
178 $state = $IN_TEXT;
179 if ($tag_type eq '{') {
180 if ($ctag eq '}}') {
181 $i++;
182 }
183 else {
184 clean_triple_stache($tokens[-1]);
185 }
186 }
187 }
188 else {
189 $buf .= char_at($text, $i);
190 }
191 }
192 }
193
194 $filter_line->($seen_tag, 1);
195
196 return \@tokens;
197 }
198
199 sub clean_triple_stache {
200 my ($token) = @_;
201
202 if (substr($token->{'n'}, length($token->{'n'}) - 1) eq '}') {
203 $token->{'n'} = substr($token->{'n'}, 0, length($token->{'n'}) - 1);
204 }
205
206 return;
207 }
208
209 sub tag_change {
210 my ($tag, $text, $index) = @_;
211
212 if (char_at($text, $index) ne char_at($tag, 0)) {
213 return 0;
214 }
215
216 for (my $i = 1, my $l = length($tag); $i < $l; $i++) {
217 if (char_at($text, $index + $i) ne char_at($tag, $i)) {
218 return 0;
219 }
220 }
221
222 return 1;
223 }
224
225 my %allowed_in_super = (
226 '_t' => 1,
227 "\n" => 1,
228 '$' => 1,
229 '/' => 1,
230 );
231
232 sub build_tree {
233 my ($tokens, $kind, $stack, $custom_tags) = @_;
234 my (@instructions, $opener, $tail, $token);
235
236 $tail = $stack->[-1];
237
238 while (@$tokens > 0) {
239 $token = shift @$tokens;
240
241 if ($tail && ($tail->{'tag'} eq '<') && !$allowed_in_super{$token->{'tag'}}) {
242 die "Illegal content in < super tag.";
243 }
244
245 if ($tags{$token->{'tag'}} && ($tags{$token->{'tag'}} <= $tags{'$'} || is_opener($token, $custom_tags))) {
246 push @$stack, $token;
247 $token->{'nodes'} = build_tree($tokens, $token->{'tag'}, $stack, $custom_tags);
248 }
249 elsif ($token->{'tag'} eq '/') {
250 if (!@$stack) {
251 die "Closing tag without opener: /$token->{'n'}";
252 }
253 $opener = pop @$stack;
254 if ($token->{'n'} ne $opener->{'n'} && !is_closer($token->{'n'}, $opener->{'n'}, $custom_tags)) {
255 die "Nesting error: $opener->{'n'} vs $token->{'n'}";
256 }
257 $opener->{'end'} = $token->{'i'};
258 return \@instructions;
259 }
260 elsif ($token->{'tag'} eq "\n") {
261 $token->{'last'} = (@$tokens == 0) || ($tokens->[0]{'tag'} eq "\n");
262 }
263
264 push @instructions, $token;
265 }
266
267 if (@$stack) {
268 die "Missing closing tag: ", pop(@$stack)->{'n'};
269 }
270
271 return \@instructions;
272 }
273
274 sub is_opener {
275 my ($token, $tags) = @_;
276
277 for (my $i = 0, my $l = scalar(@$tags); $i < $l; $i++) {
278 if ($tags->[$i]{'o'} eq $token->{'n'}) {
279 $token->{'tag'} = '#';
280 return 1;
281 }
282 }
283
284 return 0;
285 }
286
287 sub is_closer {
288 my ($close, $open, $tags) = @_;
289
290 for (my $i = 0, my $l = scalar(@$tags); $i < $l; $i++) {
291 if (($tags->[$i]{'c'} eq $close) && ($tags->[$i]{'o'} eq $open)) {
292 return 1;
293 }
294 }
295
296 return 0;
297 }
298
299 sub stringify_substitutions {
300 my $obj = shift;
301
302 my @items;
303 for my $key (sort keys %$obj) {
304 push @items, sprintf('"%s" => sub { my ($self,$c,$p,$t,$i) = @_; %s }', esc($key), $obj->{$key});
305 }
306
307 return sprintf("{ %s }", join(', ', @items));
308 }
309
310 sub stringify_partials {
311 my $code_obj = shift;
312
313 my @partials;
314 for my $key (sort keys %{ $code_obj->{'partials'} }) {
315 push @partials, sprintf('"%s" => { "name" => "%s", %s }', $key,
316 esc($code_obj->{'partials'}{$key}{'name'}),
317 stringify_partials($code_obj->{'partials'}{$key})
318 );
319 }
320
321 return sprintf('"partials" => { %s }, "subs" => %s',
322 join(',', @partials),
323 stringify_substitutions($code_obj->{'subs'})
324 );
325 }
326
327 sub stringify {
328 my ($self,$code_obj, $text, $options) = @_;
329 return sprintf('{ code => sub { my ($t,$c,$p,$i) = @_; %s }, %s }',
330 wrap_main($code_obj->{'code'}),
331 stringify_partials($code_obj)
332 );
333 }
334
335 my $serial_no = 0;
336 sub generate {
337 my ($self, $tree, $text, $options) = @_;
338
339 $serial_no = 0;
340
341 my $context = { 'code' => "", 'subs' => {}, 'partials' => {} };
342 walk($tree, $context);
343
344 if ($options->{'as_string'}) {
345 return $self->stringify($context, $text, $options);
346 }
347
348 return $self->make_template($context, $text, $options);
349 }
350
351 sub wrap_main {
352 my ($code) = @_;
353 return sprintf('$t->b($i = $i || ""); %s return $t->fl();', $code);
354 }
355
356 sub make_template {
357 my $self = shift;
358 my ($code_obj, $text, $options) = @_;
359
360 my $template = make_partials($code_obj);
361 $template->{'code'} = eval sprintf("sub { my (\$t, \$c, \$p, \$i) = \@_; %s }", wrap_main($code_obj->{'code'}));
362 return $Template->new($template, $text, $self, $options);
363 }
364
365 sub make_partials {
366 my ($code_obj) = @_;
367
368 my $key;
369 my $template = {
370 'subs' => {},
371 'partials' => $code_obj->{'partials'},
372 'name' => $code_obj->{'name'},
373 };
374
375 for my $key (sort keys %{ $template->{'partials'} }) {
376 $template->{'partials'}{$key} = make_partials($template->{'partials'}{$key});
377 }
378
379 for my $key (sort keys %{ $code_obj->{'subs'} }) {
380 $template->{'subs'}{$key} = eval "sub { my (\$t, \$c, \$p, \$t, \$i) = \@_; $code_obj->{subs}{$key}; }";
381 }
382
383 return $template;
384 }
385
386 sub esc {
387 my $s = shift;
388
389 # standard from hogan.js
390 $s =~ s/$r_slash/\\\\/g;
391 $s =~ s/$r_quot/\\\"/g;
392 $s =~ s/$r_newline/\\n/g;
393 $s =~ s/$r_cr/\\r/g;
394 $s =~ s/$r_linesep/\\u2028/g;
395 $s =~ s/$r_paragraphsep/\\u2029/g;
396
397 # specific for Text::Hogan / Perl
398 $s =~ s/\$/\\\$/g;
399 $s =~ s/\@/\\@/g;
400 $s =~ s/\%/\\%/g;
401
402 return $s;
403 }
404
405 sub char_at {
406 my ($text, $index) = @_;
407 if (is_arrayref($text)) {
408 return $text->[$index];
409 }
410 return substr($text, $index, 1);
411 }
412
413 sub choose_method {
414 my ($s) = @_;
415 return $s =~ m/[.]/ ? "d" : "f";
416 }
417
418 sub create_partial {
419 my ($node, $context) = @_;
420
421 my $prefix = "<" . ($context->{'prefix'} || "");
422 my $sym = $prefix . $node->{'n'} . $serial_no++;
423 $context->{'partials'}{$sym} = {
424 'name' => $node->{'n'},
425 'partials' => {},
426 };
427 $context->{'code'} .= sprintf('$t->b($t->rp("%s",$c,$p,"%s"));',
428 esc($sym),
429 ($node->{'indent'} || "")
430 );
431
432 return $sym;
433 }
434
435 my %codegen = (
436 '#' => sub {
437 my ($node, $context) = @_;
438 $context->{'code'} .= sprintf('if($t->s($t->%s("%s",$c,$p,1),$c,$p,0,%s,%s,"%s %s")) { $t->rs($c,$p,sub { my ($c,$p,$t) = @_;',
439 choose_method($node->{'n'}),
440 esc($node->{'n'}),
441 @{$node}{qw/ i end otag ctag /},
442 );
443 walk($node->{'nodes'}, $context);
444 $context->{'code'} .= '}); pop @$c;}';
445 },
446 '^' => sub {
447 my ($node, $context) = @_;
448 $context->{'code'} .= sprintf('if (!$t->s($t->%s("%s",$c,$p,1),$c,$p,1,0,0,"")){',
449 choose_method($node->{'n'}),
450 esc($node->{'n'})
451 );
452 walk($node->{'nodes'}, $context);
453 $context->{'code'} .= "};";
454 },
455 '>' => \&create_partial,
456 '<' => sub {
457 my ($node, $context) = @_;
458 my $ctx = { 'partials' => {}, 'code' => "", 'subs' => {}, 'in_partial' => 1 };
459 walk($node->{'nodes'}, $ctx);
460 my $template = $context->{'partials'}{create_partial($node, $context)};
461 $template->{'subs'} = $ctx->{'subs'};
462 $template->{'partials'} = $ctx->{'partials'};
463 },
464 '$' => sub {
465 my ($node, $context) = @_;
466 my $ctx = { 'subs' => {}, 'code' => "", 'partials' => $context->{'partials'}, 'prefix' => $node->{'n'} };
467 walk($node->{'nodes'}, $ctx);
468 $context->{'subs'}{$node->{'n'}} = $ctx->{'code'};
469 if (!$context->{'in_partial'}) {
470 $context->{'code'} += sprintf('$t->sub("%s",$c,$p,$i);',
471 esc($node->{'n'})
472 );
473 }
474 },
475 "\n" => sub {
476 my ($node, $context) = @_;
477 $context->{'code'} .= twrite(sprintf('"\n"%s', ($node->{'last'} ? "" : ' . $i')));
478 },
479 '_v' => sub {
480 my ($node, $context) = @_;
481 $context->{'code'} .= twrite(sprintf('$t->v($t->%s("%s",$c,$p,0))',
482 choose_method($node->{'n'}),
483 esc($node->{'n'})
484 ));
485 },
486 '_t' => sub {
487 my ($node, $context) = @_;
488 $context->{'code'} .= twrite(sprintf('"%s"', esc($node->{'text'})));
489 },
490 '{' => \&triple_stache,
491 '&' => \&triple_stache,
492 );
493
494 sub triple_stache {
495 my ($node, $context) = @_;
496 $context->{'code'} .= sprintf('$t->b($t->t($t->%s("%s",$c,$p,0)));',
497 choose_method($node->{'n'}),
498 esc($node->{'n'})
499 );
500 }
501
502 sub twrite { sprintf '$t->b(%s);', @_ }
503
504 sub walk {
505 my ($nodelist, $context) = @_;
506
507 for my $node (@$nodelist) {
508 my $func = $codegen{$node->{'tag'}} or next;
509 $func->($node, $context);
510 }
511
512 return $context;
513 }
514
515 sub parse {
516 my ($self, $tokens, $text, $options) = @_;
517 $options ||= {};
518 return build_tree($tokens, "", [], $options->{'section_tags'} || []);
519 }
520
521 my %cache;
522
523 sub cache_key {
524 my ($text, $options) = @_;
525 return join('||', $text, !!$options->{'as_string'}, !!$options->{'numeric_string_as_string'}, !!$options->{'disable_lambda'}, ($options->{'delimiters'} || ""), ($options->{'allow_whitespace_before_hashmark'} || 0));
526 }
527
528 sub compile {
529 my ($self, $text, $options) = @_;
530 $options ||= {};
531
532 $text //= "";
533
534 my $key = cache_key($text, $options);
535 my $template = $cache{$key};
536
537 if ($template) {
538 my $partials = $template->{'partials'};
539 for my $name (sort keys %{ $template->{'partials'} }) {
540 delete $partials->{$name}{'instance'};
541 }
542 return $template;
543 }
544
545 $template = $self->generate(
546 $self->parse(
547 $self->scan($text, $options), $text, $options
548 ), $text, $options
549 );
550
551 return $cache{$key} = $template;
552 }
553
554 1;
555
556 __END__
557
558 =head1 NAME
559
560 Text::Hogan::Compiler - parse templates and output Perl code
561
562 =head1 VERSION
563
564 version 2.04
565
566 =head1 SYNOPSIS
567
568 use Text::Hogan::Compiler;
569
570 my $compiler = Text::Hogan::Compiler->new;
571
572 my $text = "Hello, {{name}}!";
573
574 my $tokens = $compiler->scan($text);
575 my $tree = $compiler->parse($tokens, $text);
576 my $template = $compiler->generate($tree, $text);
577
578 say $template->render({ name => "Alex" });
579
580 =head1 METHODS
581
582 =head2 new
583
584 Takes nothing, returns a Compiler object.
585
586 my $compiler = Text::Hogan::Compiler->new;
587
588 =cut
589
590 =head2 scan
591
592 Takes template text and returns an arrayref which is a list of tokens.
593
594 my $tokens = $compiler->scan("Hello, {{name}}!");
595
596 Optionally takes a hashref with options. 'delimiters' is a string which
597 represents different delimiters, split by white-space. You should never
598 need to pass this directly, it it used to implement the in-template
599 delimiter-switching functionality.
600
601 # equivalent to the above call with mustaches
602 my $tokens = Text::Hogan::Compiler->new->scan("Hello, <% name %>!", { delimiters => "<% %>" });
603
604 'allow_whitespace_before_hashmark' is a boolean. If true,tags are allowed
605 to have space(s) between the delimiters and the opening sigil ('#', '/', '^', '<', etc.).
606
607 my $tokens = Text::Hogan::Compiler->new->scan("Hello{{ # foo }}, again{{ / foo }}.", { allow_whitespace_before_hashmark => 1 });
608
609 =head2 parse
610
611 Takes the tokens returned by scan, along with the original text, and returns a
612 tree structure ready to be turned into Perl code.
613
614 my $tree = $compiler->parse($tokens, $text);
615
616 Optionally takes a hashref that can have a key called "section_tags" which
617 should be an arrayref. I don't know what it does. Probably something internal
618 related to recursive calls that you don't need to worry about.
619
620 Note that a lot of error checking on your input gets done in this method, and
621 it is pretty much the only place exceptions might be thrown. Exceptions which
622 may be thrown include: "Closing tag without opener", "Missing closing tag",
623 "Nesting error" and "Illegal content in < super tag".
624
625 =head2 generate
626
627 Takes the parsed tree and the original text and returns a Text::Hogan::Template
628 object that you can call render on.
629
630 my $template = $compiler->generate($tree, $text);
631
632 Optionally takes a hashref that can have
633
634 - a key "as_string". If that is passed then instead of getting a template object
635 back you get some stringified Perl code that you can cache somewhere on
636 disk as part of your build process.
637
638 my $perl_code_as_string = $compiler->generate($tree, $text, { 'as_string' => 1 });
639
640 - a key "numeric_string_as_string". If that is passed output that looks like a number
641 is NOT converted into a number (ie "01234" is NOT converted to "1234")
642
643 my $perl_code_as_string = $compiler->generate($tree, $text, { 'numeric_string_as_string' => 1 });
644
645
646 The options hashref can have other keys which will be passed to
647 Text::Hogan::Template::new among other places.
648
649 =head2 compile
650
651 Takes a template string and calls scan, parse and generate on it and returns
652 you the Text::Hogan::Template object.
653
654 my $template = $compiler->compile("Hello, {{name}}!");
655
656 Also caches templates by a sensible cache key, which can be useful if you're
657 not stringifying and storing on disk or in memory anyway.
658
659 Optionally takes a hashref that will be passed on to scan, parse, and generate.
660
661 my $perl_code_as_string = $compiler->compile(
662 $text,
663 {
664 delimiters => "<% %>",
665 as_string => 1,
666 allow_whitespace_before_hashmark => 1,
667 },
668 );
669
670 =head1 ENCODING
671
672 As long as you are consistent with your use of encoding in your template
673 variables and your context variables, everything should just work. You can use
674 byte strings or character strings and you'll get what you expect.
675
676 The only danger would be if you use byte strings of a multi-byte encoding and
677 you happen to get a clash with your delimiters, eg. if your 4 byte kanji
678 character happens to contain the ASCII '<' and '%' characters next to each
679 other. I have no idea what the likelihood of that is, but hopefully if you're
680 working with non-ASCII character sets you're also using Perl's unicode
681 character strings features.
682
683 Compiling long character string inputs with Text::Hogan used to be extremely
684 slow but an optimisation added in version 2.00 has made the overhead much more
685 manageable.
686
687 =head1 AUTHORS
688
689 Started out statement-for-statement copied from hogan.js by Twitter!
690
691 Initial translation by Alex Balhatchet ([email protected])
692
693 Further improvements from:
694
695 Ed Freyfogle
696 Mohammad S Anwar
697 Ricky Morse
698 Jerrad Pierce
699 Tom Hukins
700 Tony Finch
701 Yanick Champoux
702
703 =cut