| 1 |
.\" -*- mode: troff; coding: utf-8 -*- |
| 2 |
.\" Automatically generated by Pod::Man 5.0102 (Pod::Simple 3.45) |
| 3 |
.\" |
| 4 |
.\" Standard preamble: |
| 5 |
.\" ======================================================================== |
| 6 |
.de Sp \" Vertical space (when we can't use .PP) |
| 7 |
.if t .sp .5v |
| 8 |
.if n .sp |
| 9 |
.. |
| 10 |
.de Vb \" Begin verbatim text |
| 11 |
.ft CW |
| 12 |
.nf |
| 13 |
.ne \\$1 |
| 14 |
.. |
| 15 |
.de Ve \" End verbatim text |
| 16 |
.ft R |
| 17 |
.fi |
| 18 |
.. |
| 19 |
.\" \*(C` and \*(C' are quotes in nroff, nothing in troff, for use with C<>. |
| 20 |
.ie n \{\ |
| 21 |
. ds C` "" |
| 22 |
. ds C' "" |
| 23 |
'br\} |
| 24 |
.el\{\ |
| 25 |
. ds C` |
| 26 |
. ds C' |
| 27 |
'br\} |
| 28 |
.\" |
| 29 |
.\" Escape single quotes in literal strings from groff's Unicode transform. |
| 30 |
.ie \n(.g .ds Aq \(aq |
| 31 |
.el .ds Aq ' |
| 32 |
.\" |
| 33 |
.\" If the F register is >0, we'll generate index entries on stderr for |
| 34 |
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index |
| 35 |
.\" entries marked with X<> in POD. Of course, you'll have to process the |
| 36 |
.\" output yourself in some meaningful fashion. |
| 37 |
.\" |
| 38 |
.\" Avoid warning from groff about undefined register 'F'. |
| 39 |
.de IX |
| 40 |
.. |
| 41 |
.nr rF 0 |
| 42 |
.if \n(.g .if rF .nr rF 1 |
| 43 |
.if (\n(rF:(\n(.g==0)) \{\ |
| 44 |
. if \nF \{\ |
| 45 |
. de IX |
| 46 |
. tm Index:\\$1\t\\n%\t"\\$2" |
| 47 |
.. |
| 48 |
. if !\nF==2 \{\ |
| 49 |
. nr % 0 |
| 50 |
. nr F 2 |
| 51 |
. \} |
| 52 |
. \} |
| 53 |
.\} |
| 54 |
.rr rF |
| 55 |
.\" ======================================================================== |
| 56 |
.\" |
| 57 |
.IX Title "PERLFAQ6 1" |
| 58 |
.TH PERLFAQ6 1 2025-11-16 "perl v5.40.1" "Perl Programmers Reference Guide" |
| 59 |
.\" For nroff, turn off justification. Always turn off hyphenation; it makes |
| 60 |
.\" way too many mistakes in technical documents. |
| 61 |
.if n .ad l |
| 62 |
.nh |
| 63 |
.SH NAME |
| 64 |
perlfaq6 \- Regular Expressions |
| 65 |
.SH VERSION |
| 66 |
.IX Header "VERSION" |
| 67 |
version 5.20240218 |
| 68 |
.SH DESCRIPTION |
| 69 |
.IX Header "DESCRIPTION" |
| 70 |
This section is surprisingly small because the rest of the FAQ is |
| 71 |
littered with answers involving regular expressions. For example, |
| 72 |
decoding a URL and checking whether something is a number can be handled |
| 73 |
with regular expressions, but those answers are found elsewhere in |
| 74 |
this document (in perlfaq9: "How do I decode or create those %\-encodings |
| 75 |
on the web" and perlfaq4: "How do I determine whether a scalar is |
| 76 |
a number/whole/integer/float", to be precise). |
| 77 |
.SS "How can I hope to use regular expressions without creating illegible and unmaintainable code?" |
| 78 |
.IX Xref "regex, legibility regexp, legibility regular expression, legibility x" |
| 79 |
.IX Subsection "How can I hope to use regular expressions without creating illegible and unmaintainable code?" |
| 80 |
Three techniques can make regular expressions maintainable and |
| 81 |
understandable. |
| 82 |
.IP "Comments Outside the Regex" 4 |
| 83 |
.IX Item "Comments Outside the Regex" |
| 84 |
Describe what you're doing and how you're doing it, using normal Perl |
| 85 |
comments. |
| 86 |
.Sp |
| 87 |
.Vb 3 |
| 88 |
\& # turn the line into the first word, a colon, and the |
| 89 |
\& # number of characters on the rest of the line |
| 90 |
\& s/^(\ew+)(.*)/ lc($1) . ":" . length($2) /meg; |
| 91 |
.Ve |
| 92 |
.IP "Comments Inside the Regex" 4 |
| 93 |
.IX Item "Comments Inside the Regex" |
| 94 |
The \f(CW\*(C`/x\*(C'\fR modifier causes whitespace to be ignored in a regex pattern |
| 95 |
(except in a character class and a few other places), and also allows you to |
| 96 |
use normal comments there, too. As you can imagine, whitespace and comments |
| 97 |
help a lot. |
| 98 |
.Sp |
| 99 |
\&\f(CW\*(C`/x\*(C'\fR lets you turn this: |
| 100 |
.Sp |
| 101 |
.Vb 1 |
| 102 |
\& s{<(?:[^>\*(Aq"]*|".*?"|\*(Aq.*?\*(Aq)+>}{}gs; |
| 103 |
.Ve |
| 104 |
.Sp |
| 105 |
into this: |
| 106 |
.Sp |
| 107 |
.Vb 10 |
| 108 |
\& s{ < # opening angle bracket |
| 109 |
\& (?: # Non\-backreffing grouping paren |
| 110 |
\& [^>\*(Aq"] * # 0 or more things that are neither > nor \*(Aq nor " |
| 111 |
\& | # or else |
| 112 |
\& ".*?" # a section between double quotes (stingy match) |
| 113 |
\& | # or else |
| 114 |
\& \*(Aq.*?\*(Aq # a section between single quotes (stingy match) |
| 115 |
\& ) + # all occurring one or more times |
| 116 |
\& > # closing angle bracket |
| 117 |
\& }{}gsx; # replace with nothing, i.e. delete |
| 118 |
.Ve |
| 119 |
.Sp |
| 120 |
It's still not quite so clear as prose, but it is very useful for |
| 121 |
describing the meaning of each part of the pattern. |
| 122 |
.IP "Different Delimiters" 4 |
| 123 |
.IX Item "Different Delimiters" |
| 124 |
While we normally think of patterns as being delimited with \f(CW\*(C`/\*(C'\fR |
| 125 |
characters, they can be delimited by almost any character. perlre |
| 126 |
describes this. For example, the \f(CW\*(C`s///\*(C'\fR above uses braces as |
| 127 |
delimiters. Selecting another delimiter can avoid quoting the |
| 128 |
delimiter within the pattern: |
| 129 |
.Sp |
| 130 |
.Vb 2 |
| 131 |
\& s/\e/usr\e/local/\e/usr\e/share/g; # bad delimiter choice |
| 132 |
\& s#/usr/local#/usr/share#g; # better |
| 133 |
.Ve |
| 134 |
.Sp |
| 135 |
Using logically paired delimiters can be even more readable: |
| 136 |
.Sp |
| 137 |
.Vb 1 |
| 138 |
\& s{/usr/local/}{/usr/share}g; # better still |
| 139 |
.Ve |
| 140 |
.SS "I'm having trouble matching over more than one line. What's wrong?" |
| 141 |
.IX Xref "regex, multiline regexp, multiline regular expression, multiline" |
| 142 |
.IX Subsection "I'm having trouble matching over more than one line. What's wrong?" |
| 143 |
Either you don't have more than one line in the string you're looking |
| 144 |
at (probably), or else you aren't using the correct modifier(s) on |
| 145 |
your pattern (possibly). |
| 146 |
.PP |
| 147 |
There are many ways to get multiline data into a string. If you want |
| 148 |
it to happen automatically while reading input, you'll want to set $/ |
| 149 |
(probably to '' for paragraphs or \f(CW\*(C`undef\*(C'\fR for the whole file) to |
| 150 |
allow you to read more than one line at a time. |
| 151 |
.PP |
| 152 |
Read perlre to help you decide which of \f(CW\*(C`/s\*(C'\fR and \f(CW\*(C`/m\*(C'\fR (or both) |
| 153 |
you might want to use: \f(CW\*(C`/s\*(C'\fR allows dot to include newline, and \f(CW\*(C`/m\*(C'\fR |
| 154 |
allows caret and dollar to match next to a newline, not just at the |
| 155 |
end of the string. You do need to make sure that you've actually |
| 156 |
got a multiline string in there. |
| 157 |
.PP |
| 158 |
For example, this program detects duplicate words, even when they span |
| 159 |
line breaks (but not paragraph ones). For this example, we don't need |
| 160 |
\&\f(CW\*(C`/s\*(C'\fR because we aren't using dot in a regular expression that we want |
| 161 |
to cross line boundaries. Neither do we need \f(CW\*(C`/m\*(C'\fR because we don't |
| 162 |
want caret or dollar to match at any point inside the record next |
| 163 |
to newlines. But it's imperative that $/ be set to something other |
| 164 |
than the default, or else we won't actually ever have a multiline |
| 165 |
record read in. |
| 166 |
.PP |
| 167 |
.Vb 6 |
| 168 |
\& $/ = \*(Aq\*(Aq; # read in whole paragraph, not just one line |
| 169 |
\& while ( <> ) { |
| 170 |
\& while ( /\eb([\ew\*(Aq\-]+)(\es+\eg1)+\eb/gi ) { # word starts alpha |
| 171 |
\& print "Duplicate $1 at paragraph $.\en"; |
| 172 |
\& } |
| 173 |
\& } |
| 174 |
.Ve |
| 175 |
.PP |
| 176 |
Here's some code that finds sentences that begin with "From " (which would |
| 177 |
be mangled by many mailers): |
| 178 |
.PP |
| 179 |
.Vb 6 |
| 180 |
\& $/ = \*(Aq\*(Aq; # read in whole paragraph, not just one line |
| 181 |
\& while ( <> ) { |
| 182 |
\& while ( /^From /gm ) { # /m makes ^ match next to \en |
| 183 |
\& print "leading From in paragraph $.\en"; |
| 184 |
\& } |
| 185 |
\& } |
| 186 |
.Ve |
| 187 |
.PP |
| 188 |
Here's code that finds everything between START and END in a paragraph: |
| 189 |
.PP |
| 190 |
.Vb 6 |
| 191 |
\& undef $/; # read in whole file, not just one line or paragraph |
| 192 |
\& while ( <> ) { |
| 193 |
\& while ( /START(.*?)END/sgm ) { # /s makes . cross line boundaries |
| 194 |
\& print "$1\en"; |
| 195 |
\& } |
| 196 |
\& } |
| 197 |
.Ve |
| 198 |
.SS "How can I pull out lines between two patterns that are themselves on different lines?" |
| 199 |
.IX Xref ".." |
| 200 |
.IX Subsection "How can I pull out lines between two patterns that are themselves on different lines?" |
| 201 |
You can use Perl's somewhat exotic \f(CW\*(C`..\*(C'\fR operator (documented in |
| 202 |
perlop): |
| 203 |
.PP |
| 204 |
.Vb 1 |
| 205 |
\& perl \-ne \*(Aqprint if /START/ .. /END/\*(Aq file1 file2 ... |
| 206 |
.Ve |
| 207 |
.PP |
| 208 |
If you wanted text and not lines, you would use |
| 209 |
.PP |
| 210 |
.Vb 1 |
| 211 |
\& perl \-0777 \-ne \*(Aqprint "$1\en" while /START(.*?)END/gs\*(Aq file1 file2 ... |
| 212 |
.Ve |
| 213 |
.PP |
| 214 |
But if you want nested occurrences of \f(CW\*(C`START\*(C'\fR through \f(CW\*(C`END\*(C'\fR, you'll |
| 215 |
run up against the problem described in the question in this section |
| 216 |
on matching balanced text. |
| 217 |
.PP |
| 218 |
Here's another example of using \f(CW\*(C`..\*(C'\fR: |
| 219 |
.PP |
| 220 |
.Vb 7 |
| 221 |
\& while (<>) { |
| 222 |
\& my $in_header = 1 .. /^$/; |
| 223 |
\& my $in_body = /^$/ .. eof; |
| 224 |
\& # now choose between them |
| 225 |
\& } continue { |
| 226 |
\& $. = 0 if eof; # fix $. |
| 227 |
\& } |
| 228 |
.Ve |
| 229 |
.SS "How do I match XML, HTML, or other nasty, ugly things with a regex?" |
| 230 |
.IX Xref "regex, XML regex, HTML XML HTML pain frustration sucking out, will to live" |
| 231 |
.IX Subsection "How do I match XML, HTML, or other nasty, ugly things with a regex?" |
| 232 |
Do not use regexes. Use a module and forget about the |
| 233 |
regular expressions. The XML::LibXML, HTML::TokeParser and |
| 234 |
HTML::TreeBuilder modules are good starts, although each namespace |
| 235 |
has other parsing modules specialized for certain tasks and different |
| 236 |
ways of doing it. Start at CPAN Search ( <http://metacpan.org/> ) |
| 237 |
and wonder at all the work people have done for you already! :) |
| 238 |
.SS "I put a regular expression into $/ but it didn't work. What's wrong?" |
| 239 |
.IX Xref "$ , regexes in $INPUT_RECORD_SEPARATOR, regexes in $RS, regexes in" |
| 240 |
.IX Subsection "I put a regular expression into $/ but it didn't work. What's wrong?" |
| 241 |
$/ has to be a string. You can use these examples if you really need to |
| 242 |
do this. |
| 243 |
.PP |
| 244 |
If you have File::Stream, this is easy. |
| 245 |
.PP |
| 246 |
.Vb 1 |
| 247 |
\& use File::Stream; |
| 248 |
\& |
| 249 |
\& my $stream = File::Stream\->new( |
| 250 |
\& $filehandle, |
| 251 |
\& separator => qr/\es*,\es*/, |
| 252 |
\& ); |
| 253 |
\& |
| 254 |
\& print "$_\en" while <$stream>; |
| 255 |
.Ve |
| 256 |
.PP |
| 257 |
If you don't have File::Stream, you have to do a little more work. |
| 258 |
.PP |
| 259 |
You can use the four-argument form of sysread to continually add to |
| 260 |
a buffer. After you add to the buffer, you check if you have a |
| 261 |
complete line (using your regular expression). |
| 262 |
.PP |
| 263 |
.Vb 7 |
| 264 |
\& local $_ = ""; |
| 265 |
\& while( sysread FH, $_, 8192, length ) { |
| 266 |
\& while( s/^((?s).*?)your_pattern// ) { |
| 267 |
\& my $record = $1; |
| 268 |
\& # do stuff here. |
| 269 |
\& } |
| 270 |
\& } |
| 271 |
.Ve |
| 272 |
.PP |
| 273 |
You can do the same thing with foreach and a match using the |
| 274 |
c flag and the \eG anchor, if you do not mind your entire file |
| 275 |
being in memory at the end. |
| 276 |
.PP |
| 277 |
.Vb 7 |
| 278 |
\& local $_ = ""; |
| 279 |
\& while( sysread FH, $_, 8192, length ) { |
| 280 |
\& foreach my $record ( m/\eG((?s).*?)your_pattern/gc ) { |
| 281 |
\& # do stuff here. |
| 282 |
\& } |
| 283 |
\& substr( $_, 0, pos ) = "" if pos; |
| 284 |
\& } |
| 285 |
.Ve |
| 286 |
.SS "How do I substitute case-insensitively on the LHS while preserving case on the RHS?" |
| 287 |
.IX Xref "replace, case preserving substitute, case preserving substitution, case preserving s, case preserving" |
| 288 |
.IX Subsection "How do I substitute case-insensitively on the LHS while preserving case on the RHS?" |
| 289 |
Here's a lovely Perlish solution by Larry Rosler. It exploits |
| 290 |
properties of bitwise xor on ASCII strings. |
| 291 |
.PP |
| 292 |
.Vb 1 |
| 293 |
\& $_= "this is a TEsT case"; |
| 294 |
\& |
| 295 |
\& $old = \*(Aqtest\*(Aq; |
| 296 |
\& $new = \*(Aqsuccess\*(Aq; |
| 297 |
\& |
| 298 |
\& s{(\eQ$old\eE)} |
| 299 |
\& { uc $new | (uc $1 ^ $1) . |
| 300 |
\& (uc(substr $1, \-1) ^ substr $1, \-1) x |
| 301 |
\& (length($new) \- length $1) |
| 302 |
\& }egi; |
| 303 |
\& |
| 304 |
\& print; |
| 305 |
.Ve |
| 306 |
.PP |
| 307 |
And here it is as a subroutine, modeled after the above: |
| 308 |
.PP |
| 309 |
.Vb 3 |
| 310 |
\& sub preserve_case { |
| 311 |
\& my ($old, $new) = @_; |
| 312 |
\& my $mask = uc $old ^ $old; |
| 313 |
\& |
| 314 |
\& uc $new | $mask . |
| 315 |
\& substr($mask, \-1) x (length($new) \- length($old)) |
| 316 |
\& } |
| 317 |
\& |
| 318 |
\& $string = "this is a TEsT case"; |
| 319 |
\& $string =~ s/(test)/preserve_case($1, "success")/egi; |
| 320 |
\& print "$string\en"; |
| 321 |
.Ve |
| 322 |
.PP |
| 323 |
This prints: |
| 324 |
.PP |
| 325 |
.Vb 1 |
| 326 |
\& this is a SUcCESS case |
| 327 |
.Ve |
| 328 |
.PP |
| 329 |
As an alternative, to keep the case of the replacement word if it is |
| 330 |
longer than the original, you can use this code, by Jeff Pinyan: |
| 331 |
.PP |
| 332 |
.Vb 3 |
| 333 |
\& sub preserve_case { |
| 334 |
\& my ($from, $to) = @_; |
| 335 |
\& my ($lf, $lt) = map length, @_; |
| 336 |
\& |
| 337 |
\& if ($lt < $lf) { $from = substr $from, 0, $lt } |
| 338 |
\& else { $from .= substr $to, $lf } |
| 339 |
\& |
| 340 |
\& return uc $to | ($from ^ uc $from); |
| 341 |
\& } |
| 342 |
.Ve |
| 343 |
.PP |
| 344 |
This changes the sentence to "this is a SUcCess case." |
| 345 |
.PP |
| 346 |
Just to show that C programmers can write C in any programming language, |
| 347 |
if you prefer a more C\-like solution, the following script makes the |
| 348 |
substitution have the same case, letter by letter, as the original. |
| 349 |
(It also happens to run about 240% slower than the Perlish solution runs.) |
| 350 |
If the substitution has more characters than the string being substituted, |
| 351 |
the case of the last character is used for the rest of the substitution. |
| 352 |
.PP |
| 353 |
.Vb 8 |
| 354 |
\& # Original by Nathan Torkington, massaged by Jeffrey Friedl |
| 355 |
\& # |
| 356 |
\& sub preserve_case |
| 357 |
\& { |
| 358 |
\& my ($old, $new) = @_; |
| 359 |
\& my $state = 0; # 0 = no change; 1 = lc; 2 = uc |
| 360 |
\& my ($i, $oldlen, $newlen, $c) = (0, length($old), length($new)); |
| 361 |
\& my $len = $oldlen < $newlen ? $oldlen : $newlen; |
| 362 |
\& |
| 363 |
\& for ($i = 0; $i < $len; $i++) { |
| 364 |
\& if ($c = substr($old, $i, 1), $c =~ /[\eW\ed_]/) { |
| 365 |
\& $state = 0; |
| 366 |
\& } elsif (lc $c eq $c) { |
| 367 |
\& substr($new, $i, 1) = lc(substr($new, $i, 1)); |
| 368 |
\& $state = 1; |
| 369 |
\& } else { |
| 370 |
\& substr($new, $i, 1) = uc(substr($new, $i, 1)); |
| 371 |
\& $state = 2; |
| 372 |
\& } |
| 373 |
\& } |
| 374 |
\& # finish up with any remaining new (for when new is longer than old) |
| 375 |
\& if ($newlen > $oldlen) { |
| 376 |
\& if ($state == 1) { |
| 377 |
\& substr($new, $oldlen) = lc(substr($new, $oldlen)); |
| 378 |
\& } elsif ($state == 2) { |
| 379 |
\& substr($new, $oldlen) = uc(substr($new, $oldlen)); |
| 380 |
\& } |
| 381 |
\& } |
| 382 |
\& return $new; |
| 383 |
\& } |
| 384 |
.Ve |
| 385 |
.ie n .SS "How can I make ""\ew"" match national character sets?" |
| 386 |
.el .SS "How can I make \f(CW\ew\fP match national character sets?" |
| 387 |
.IX Xref "\\w" |
| 388 |
.IX Subsection "How can I make w match national character sets?" |
| 389 |
Put \f(CW\*(C`use locale;\*(C'\fR in your script. The \ew character class is taken |
| 390 |
from the current locale. |
| 391 |
.PP |
| 392 |
See perllocale for details. |
| 393 |
.ie n .SS "How can I match a locale-smart version of ""/[a\-zA\-Z]/""?" |
| 394 |
.el .SS "How can I match a locale-smart version of \f(CW/[a\-zA\-Z]/\fP?" |
| 395 |
.IX Xref "alpha" |
| 396 |
.IX Subsection "How can I match a locale-smart version of /[a-zA-Z]/?" |
| 397 |
You can use the POSIX character class syntax \f(CW\*(C`/[[:alpha:]]/\*(C'\fR |
| 398 |
documented in perlre. |
| 399 |
.PP |
| 400 |
No matter which locale you are in, the alphabetic characters are |
| 401 |
the characters in \ew without the digits and the underscore. |
| 402 |
As a regex, that looks like \f(CW\*(C`/[^\eW\ed_]/\*(C'\fR. Its complement, |
| 403 |
the non-alphabetics, is then everything in \eW along with |
| 404 |
the digits and the underscore, or \f(CW\*(C`/[\eW\ed_]/\*(C'\fR. |
| 405 |
.SS "How can I quote a variable to use in a regex?" |
| 406 |
.IX Xref "regex, escaping regexp, escaping regular expression, escaping" |
| 407 |
.IX Subsection "How can I quote a variable to use in a regex?" |
| 408 |
The Perl parser will expand \f(CW$variable\fR and \f(CW@variable\fR references in |
| 409 |
regular expressions unless the delimiter is a single quote. Remember, |
| 410 |
too, that the right-hand side of a \f(CW\*(C`s///\*(C'\fR substitution is considered |
| 411 |
a double-quoted string (see perlop for more details). Remember |
| 412 |
also that any regex special characters will be acted on unless you |
| 413 |
precede the substitution with \eQ. Here's an example: |
| 414 |
.PP |
| 415 |
.Vb 2 |
| 416 |
\& $string = "Placido P. Octopus"; |
| 417 |
\& $regex = "P."; |
| 418 |
\& |
| 419 |
\& $string =~ s/$regex/Polyp/; |
| 420 |
\& # $string is now "Polypacido P. Octopus" |
| 421 |
.Ve |
| 422 |
.PP |
| 423 |
Because \f(CW\*(C`.\*(C'\fR is special in regular expressions, and can match any |
| 424 |
single character, the regex \f(CW\*(C`P.\*(C'\fR here has matched the <Pl> in the |
| 425 |
original string. |
| 426 |
.PP |
| 427 |
To escape the special meaning of \f(CW\*(C`.\*(C'\fR, we use \f(CW\*(C`\eQ\*(C'\fR: |
| 428 |
.PP |
| 429 |
.Vb 2 |
| 430 |
\& $string = "Placido P. Octopus"; |
| 431 |
\& $regex = "P."; |
| 432 |
\& |
| 433 |
\& $string =~ s/\eQ$regex/Polyp/; |
| 434 |
\& # $string is now "Placido Polyp Octopus" |
| 435 |
.Ve |
| 436 |
.PP |
| 437 |
The use of \f(CW\*(C`\eQ\*(C'\fR causes the \f(CW\*(C`.\*(C'\fR in the regex to be treated as a |
| 438 |
regular character, so that \f(CW\*(C`P.\*(C'\fR matches a \f(CW\*(C`P\*(C'\fR followed by a dot. |
| 439 |
.ie n .SS "What is ""/o"" really for?" |
| 440 |
.el .SS "What is \f(CW/o\fP really for?" |
| 441 |
.IX Xref " o, regular expressions compile, regular expressions" |
| 442 |
.IX Subsection "What is /o really for?" |
| 443 |
(contributed by brian d foy) |
| 444 |
.PP |
| 445 |
The \f(CW\*(C`/o\*(C'\fR option for regular expressions (documented in perlop and |
| 446 |
perlreref) tells Perl to compile the regular expression only once. |
| 447 |
This is only useful when the pattern contains a variable. Perls 5.6 |
| 448 |
and later handle this automatically if the pattern does not change. |
| 449 |
.PP |
| 450 |
Since the match operator \f(CW\*(C`m//\*(C'\fR, the substitution operator \f(CW\*(C`s///\*(C'\fR, |
| 451 |
and the regular expression quoting operator \f(CW\*(C`qr//\*(C'\fR are double-quotish |
| 452 |
constructs, you can interpolate variables into the pattern. See the |
| 453 |
answer to "How can I quote a variable to use in a regex?" for more |
| 454 |
details. |
| 455 |
.PP |
| 456 |
This example takes a regular expression from the argument list and |
| 457 |
prints the lines of input that match it: |
| 458 |
.PP |
| 459 |
.Vb 1 |
| 460 |
\& my $pattern = shift @ARGV; |
| 461 |
\& |
| 462 |
\& while( <> ) { |
| 463 |
\& print if m/$pattern/; |
| 464 |
\& } |
| 465 |
.Ve |
| 466 |
.PP |
| 467 |
Versions of Perl prior to 5.6 would recompile the regular expression |
| 468 |
for each iteration, even if \f(CW$pattern\fR had not changed. The \f(CW\*(C`/o\*(C'\fR |
| 469 |
would prevent this by telling Perl to compile the pattern the first |
| 470 |
time, then reuse that for subsequent iterations: |
| 471 |
.PP |
| 472 |
.Vb 1 |
| 473 |
\& my $pattern = shift @ARGV; |
| 474 |
\& |
| 475 |
\& while( <> ) { |
| 476 |
\& print if m/$pattern/o; # useful for Perl < 5.6 |
| 477 |
\& } |
| 478 |
.Ve |
| 479 |
.PP |
| 480 |
In versions 5.6 and later, Perl won't recompile the regular expression |
| 481 |
if the variable hasn't changed, so you probably don't need the \f(CW\*(C`/o\*(C'\fR |
| 482 |
option. It doesn't hurt, but it doesn't help either. If you want any |
| 483 |
version of Perl to compile the regular expression only once even if |
| 484 |
the variable changes (thus, only using its initial value), you still |
| 485 |
need the \f(CW\*(C`/o\*(C'\fR. |
| 486 |
.PP |
| 487 |
You can watch Perl's regular expression engine at work to verify for |
| 488 |
yourself if Perl is recompiling a regular expression. The \f(CWuse re |
| 489 |
\&\*(Aqdebug\*(Aq\fR pragma (comes with Perl 5.005 and later) shows the details. |
| 490 |
With Perls before 5.6, you should see \f(CW\*(C`re\*(C'\fR reporting that its |
| 491 |
compiling the regular expression on each iteration. With Perl 5.6 or |
| 492 |
later, you should only see \f(CW\*(C`re\*(C'\fR report that for the first iteration. |
| 493 |
.PP |
| 494 |
.Vb 1 |
| 495 |
\& use re \*(Aqdebug\*(Aq; |
| 496 |
\& |
| 497 |
\& my $regex = \*(AqPerl\*(Aq; |
| 498 |
\& foreach ( qw(Perl Java Ruby Python) ) { |
| 499 |
\& print STDERR "\-" x 73, "\en"; |
| 500 |
\& print STDERR "Trying $_...\en"; |
| 501 |
\& print STDERR "\et$_ is good!\en" if m/$regex/; |
| 502 |
\& } |
| 503 |
.Ve |
| 504 |
.SS "How do I use a regular expression to strip C\-style comments from a file?" |
| 505 |
.IX Subsection "How do I use a regular expression to strip C-style comments from a file?" |
| 506 |
While this actually can be done, it's much harder than you'd think. |
| 507 |
For example, this one-liner |
| 508 |
.PP |
| 509 |
.Vb 1 |
| 510 |
\& perl \-0777 \-pe \*(Aqs{/\e*.*?\e*/}{}gs\*(Aq foo.c |
| 511 |
.Ve |
| 512 |
.PP |
| 513 |
will work in many but not all cases. You see, it's too simple-minded for |
| 514 |
certain kinds of C programs, in particular, those with what appear to be |
| 515 |
comments in quoted strings. For that, you'd need something like this, |
| 516 |
created by Jeffrey Friedl and later modified by Fred Curtis. |
| 517 |
.PP |
| 518 |
.Vb 4 |
| 519 |
\& $/ = undef; |
| 520 |
\& $_ = <>; |
| 521 |
\& s#/\e*[^*]*\e*+([^/*][^*]*\e*+)*/|("(\e\e.|[^"\e\e])*"|\*(Aq(\e\e.|[^\*(Aq\e\e])*\*(Aq|.[^/"\*(Aq\e\e]*)#defined $2 ? $2 : ""#gse; |
| 522 |
\& print; |
| 523 |
.Ve |
| 524 |
.PP |
| 525 |
This could, of course, be more legibly written with the \f(CW\*(C`/x\*(C'\fR modifier, adding |
| 526 |
whitespace and comments. Here it is expanded, courtesy of Fred Curtis. |
| 527 |
.PP |
| 528 |
.Vb 8 |
| 529 |
\& s{ |
| 530 |
\& /\e* ## Start of /* ... */ comment |
| 531 |
\& [^*]*\e*+ ## Non\-* followed by 1\-or\-more *\*(Aqs |
| 532 |
\& ( |
| 533 |
\& [^/*][^*]*\e*+ |
| 534 |
\& )* ## 0\-or\-more things which don\*(Aqt start with / |
| 535 |
\& ## but do end with \*(Aq*\*(Aq |
| 536 |
\& / ## End of /* ... */ comment |
| 537 |
\& |
| 538 |
\& | ## OR various things which aren\*(Aqt comments: |
| 539 |
\& |
| 540 |
\& ( |
| 541 |
\& " ## Start of " ... " string |
| 542 |
\& ( |
| 543 |
\& \e\e. ## Escaped char |
| 544 |
\& | ## OR |
| 545 |
\& [^"\e\e] ## Non "\e |
| 546 |
\& )* |
| 547 |
\& " ## End of " ... " string |
| 548 |
\& |
| 549 |
\& | ## OR |
| 550 |
\& |
| 551 |
\& \*(Aq ## Start of \*(Aq ... \*(Aq string |
| 552 |
\& ( |
| 553 |
\& \e\e. ## Escaped char |
| 554 |
\& | ## OR |
| 555 |
\& [^\*(Aq\e\e] ## Non \*(Aq\e |
| 556 |
\& )* |
| 557 |
\& \*(Aq ## End of \*(Aq ... \*(Aq string |
| 558 |
\& |
| 559 |
\& | ## OR |
| 560 |
\& |
| 561 |
\& . ## Anything other char |
| 562 |
\& [^/"\*(Aq\e\e]* ## Chars which doesn\*(Aqt start a comment, string or escape |
| 563 |
\& ) |
| 564 |
\& }{defined $2 ? $2 : ""}gxse; |
| 565 |
.Ve |
| 566 |
.PP |
| 567 |
A slight modification also removes C++ comments, possibly spanning multiple lines |
| 568 |
using a continuation character: |
| 569 |
.PP |
| 570 |
.Vb 1 |
| 571 |
\& s#/\e*[^*]*\e*+([^/*][^*]*\e*+)*/|//([^\e\e]|[^\en][\en]?)*?\en|("(\e\e.|[^"\e\e])*"|\*(Aq(\e\e.|[^\*(Aq\e\e])*\*(Aq|.[^/"\*(Aq\e\e]*)#defined $3 ? $3 : ""#gse; |
| 572 |
.Ve |
| 573 |
.SS "Can I use Perl regular expressions to match balanced text?" |
| 574 |
.IX Xref "regex, matching balanced test regexp, matching balanced test regular expression, matching balanced test possessive PARNO Text::Balanced Regexp::Common backtracking recursion" |
| 575 |
.IX Subsection "Can I use Perl regular expressions to match balanced text?" |
| 576 |
(contributed by brian d foy) |
| 577 |
.PP |
| 578 |
Your first try should probably be the Text::Balanced module, which |
| 579 |
is in the Perl standard library since Perl 5.8. It has a variety of |
| 580 |
functions to deal with tricky text. The Regexp::Common module can |
| 581 |
also help by providing canned patterns you can use. |
| 582 |
.PP |
| 583 |
As of Perl 5.10, you can match balanced text with regular expressions |
| 584 |
using recursive patterns. Before Perl 5.10, you had to resort to |
| 585 |
various tricks such as using Perl code in \f(CW\*(C`(??{})\*(C'\fR sequences. |
| 586 |
.PP |
| 587 |
Here's an example using a recursive regular expression. The goal is to |
| 588 |
capture all of the text within angle brackets, including the text in |
| 589 |
nested angle brackets. This sample text has two "major" groups: a |
| 590 |
group with one level of nesting and a group with two levels of |
| 591 |
nesting. There are five total groups in angle brackets: |
| 592 |
.PP |
| 593 |
.Vb 3 |
| 594 |
\& I have some <brackets in <nested brackets> > and |
| 595 |
\& <another group <nested once <nested twice> > > |
| 596 |
\& and that\*(Aqs it. |
| 597 |
.Ve |
| 598 |
.PP |
| 599 |
The regular expression to match the balanced text uses two new (to |
| 600 |
Perl 5.10) regular expression features. These are covered in perlre |
| 601 |
and this example is a modified version of one in that documentation. |
| 602 |
.PP |
| 603 |
First, adding the new possessive \f(CW\*(C`+\*(C'\fR to any quantifier finds the |
| 604 |
longest match and does not backtrack. That's important since you want |
| 605 |
to handle any angle brackets through the recursion, not backtracking. |
| 606 |
The group \f(CW\*(C`[^<>]++\*(C'\fR finds one or more non-angle brackets without |
| 607 |
backtracking. |
| 608 |
.PP |
| 609 |
Second, the new \f(CW\*(C`(?PARNO)\*(C'\fR refers to the sub-pattern in the |
| 610 |
particular capture group given by \f(CW\*(C`PARNO\*(C'\fR. In the following regex, |
| 611 |
the first capture group finds (and remembers) the balanced text, and |
| 612 |
you need that same pattern within the first buffer to get past the |
| 613 |
nested text. That's the recursive part. The \f(CW\*(C`(?1)\*(C'\fR uses the pattern |
| 614 |
in the outer capture group as an independent part of the regex. |
| 615 |
.PP |
| 616 |
Putting it all together, you have: |
| 617 |
.PP |
| 618 |
.Vb 1 |
| 619 |
\& #!/usr/local/bin/perl5.10.0 |
| 620 |
\& |
| 621 |
\& my $string =<<"HERE"; |
| 622 |
\& I have some <brackets in <nested brackets> > and |
| 623 |
\& <another group <nested once <nested twice> > > |
| 624 |
\& and that\*(Aqs it. |
| 625 |
\& HERE |
| 626 |
\& |
| 627 |
\& my @groups = $string =~ m/ |
| 628 |
\& ( # start of capture group 1 |
| 629 |
\& < # match an opening angle bracket |
| 630 |
\& (?: |
| 631 |
\& [^<>]++ # one or more non angle brackets, non backtracking |
| 632 |
\& | |
| 633 |
\& (?1) # found < or >, so recurse to capture group 1 |
| 634 |
\& )* |
| 635 |
\& > # match a closing angle bracket |
| 636 |
\& ) # end of capture group 1 |
| 637 |
\& /xg; |
| 638 |
\& |
| 639 |
\& $" = "\en\et"; |
| 640 |
\& print "Found:\en\et@groups\en"; |
| 641 |
.Ve |
| 642 |
.PP |
| 643 |
The output shows that Perl found the two major groups: |
| 644 |
.PP |
| 645 |
.Vb 3 |
| 646 |
\& Found: |
| 647 |
\& <brackets in <nested brackets> > |
| 648 |
\& <another group <nested once <nested twice> > > |
| 649 |
.Ve |
| 650 |
.PP |
| 651 |
With a little extra work, you can get all of the groups in angle |
| 652 |
brackets even if they are in other angle brackets too. Each time you |
| 653 |
get a balanced match, remove its outer delimiter (that's the one you |
| 654 |
just matched so don't match it again) and add it to a queue of strings |
| 655 |
to process. Keep doing that until you get no matches: |
| 656 |
.PP |
| 657 |
.Vb 1 |
| 658 |
\& #!/usr/local/bin/perl5.10.0 |
| 659 |
\& |
| 660 |
\& my @queue =<<"HERE"; |
| 661 |
\& I have some <brackets in <nested brackets> > and |
| 662 |
\& <another group <nested once <nested twice> > > |
| 663 |
\& and that\*(Aqs it. |
| 664 |
\& HERE |
| 665 |
\& |
| 666 |
\& my $regex = qr/ |
| 667 |
\& ( # start of bracket 1 |
| 668 |
\& < # match an opening angle bracket |
| 669 |
\& (?: |
| 670 |
\& [^<>]++ # one or more non angle brackets, non backtracking |
| 671 |
\& | |
| 672 |
\& (?1) # recurse to bracket 1 |
| 673 |
\& )* |
| 674 |
\& > # match a closing angle bracket |
| 675 |
\& ) # end of bracket 1 |
| 676 |
\& /x; |
| 677 |
\& |
| 678 |
\& $" = "\en\et"; |
| 679 |
\& |
| 680 |
\& while( @queue ) { |
| 681 |
\& my $string = shift @queue; |
| 682 |
\& |
| 683 |
\& my @groups = $string =~ m/$regex/g; |
| 684 |
\& print "Found:\en\et@groups\en\en" if @groups; |
| 685 |
\& |
| 686 |
\& unshift @queue, map { s/^<//; s/>$//; $_ } @groups; |
| 687 |
\& } |
| 688 |
.Ve |
| 689 |
.PP |
| 690 |
The output shows all of the groups. The outermost matches show up |
| 691 |
first and the nested matches show up later: |
| 692 |
.PP |
| 693 |
.Vb 3 |
| 694 |
\& Found: |
| 695 |
\& <brackets in <nested brackets> > |
| 696 |
\& <another group <nested once <nested twice> > > |
| 697 |
\& |
| 698 |
\& Found: |
| 699 |
\& <nested brackets> |
| 700 |
\& |
| 701 |
\& Found: |
| 702 |
\& <nested once <nested twice> > |
| 703 |
\& |
| 704 |
\& Found: |
| 705 |
\& <nested twice> |
| 706 |
.Ve |
| 707 |
.SS "What does it mean that regexes are greedy? How can I get around it?" |
| 708 |
.IX Xref "greedy greediness" |
| 709 |
.IX Subsection "What does it mean that regexes are greedy? How can I get around it?" |
| 710 |
Most people mean that greedy regexes match as much as they can. |
| 711 |
Technically speaking, it's actually the quantifiers (\f(CW\*(C`?\*(C'\fR, \f(CW\*(C`*\*(C'\fR, \f(CW\*(C`+\*(C'\fR, |
| 712 |
\&\f(CW\*(C`{}\*(C'\fR) that are greedy rather than the whole pattern; Perl prefers local |
| 713 |
greed and immediate gratification to overall greed. To get non-greedy |
| 714 |
versions of the same quantifiers, use (\f(CW\*(C`??\*(C'\fR, \f(CW\*(C`*?\*(C'\fR, \f(CW\*(C`+?\*(C'\fR, \f(CW\*(C`{}?\*(C'\fR). |
| 715 |
.PP |
| 716 |
An example: |
| 717 |
.PP |
| 718 |
.Vb 3 |
| 719 |
\& my $s1 = my $s2 = "I am very very cold"; |
| 720 |
\& $s1 =~ s/ve.*y //; # I am cold |
| 721 |
\& $s2 =~ s/ve.*?y //; # I am very cold |
| 722 |
.Ve |
| 723 |
.PP |
| 724 |
Notice how the second substitution stopped matching as soon as it |
| 725 |
encountered "y ". The \f(CW\*(C`*?\*(C'\fR quantifier effectively tells the regular |
| 726 |
expression engine to find a match as quickly as possible and pass |
| 727 |
control on to whatever is next in line, as you would if you were |
| 728 |
playing hot potato. |
| 729 |
.SS "How do I process each word on each line?" |
| 730 |
.IX Xref "word" |
| 731 |
.IX Subsection "How do I process each word on each line?" |
| 732 |
Use the split function: |
| 733 |
.PP |
| 734 |
.Vb 5 |
| 735 |
\& while (<>) { |
| 736 |
\& foreach my $word ( split ) { |
| 737 |
\& # do something with $word here |
| 738 |
\& } |
| 739 |
\& } |
| 740 |
.Ve |
| 741 |
.PP |
| 742 |
Note that this isn't really a word in the English sense; it's just |
| 743 |
chunks of consecutive non-whitespace characters. |
| 744 |
.PP |
| 745 |
To work with only alphanumeric sequences (including underscores), you |
| 746 |
might consider |
| 747 |
.PP |
| 748 |
.Vb 5 |
| 749 |
\& while (<>) { |
| 750 |
\& foreach $word (m/(\ew+)/g) { |
| 751 |
\& # do something with $word here |
| 752 |
\& } |
| 753 |
\& } |
| 754 |
.Ve |
| 755 |
.SS "How can I print out a word-frequency or line-frequency summary?" |
| 756 |
.IX Subsection "How can I print out a word-frequency or line-frequency summary?" |
| 757 |
To do this, you have to parse out each word in the input stream. We'll |
| 758 |
pretend that by word you mean chunk of alphabetics, hyphens, or |
| 759 |
apostrophes, rather than the non-whitespace chunk idea of a word given |
| 760 |
in the previous question: |
| 761 |
.PP |
| 762 |
.Vb 6 |
| 763 |
\& my (%seen); |
| 764 |
\& while (<>) { |
| 765 |
\& while ( /(\eb[^\eW_\ed][\ew\*(Aq\-]+\eb)/g ) { # misses "\`sheep\*(Aq" |
| 766 |
\& $seen{$1}++; |
| 767 |
\& } |
| 768 |
\& } |
| 769 |
\& |
| 770 |
\& while ( my ($word, $count) = each %seen ) { |
| 771 |
\& print "$count $word\en"; |
| 772 |
\& } |
| 773 |
.Ve |
| 774 |
.PP |
| 775 |
If you wanted to do the same thing for lines, you wouldn't need a |
| 776 |
regular expression: |
| 777 |
.PP |
| 778 |
.Vb 1 |
| 779 |
\& my (%seen); |
| 780 |
\& |
| 781 |
\& while (<>) { |
| 782 |
\& $seen{$_}++; |
| 783 |
\& } |
| 784 |
\& |
| 785 |
\& while ( my ($line, $count) = each %seen ) { |
| 786 |
\& print "$count $line"; |
| 787 |
\& } |
| 788 |
.Ve |
| 789 |
.PP |
| 790 |
If you want these output in a sorted order, see perlfaq4: "How do I |
| 791 |
sort a hash (optionally by value instead of key)?". |
| 792 |
.SS "How can I do approximate matching?" |
| 793 |
.IX Xref "match, approximate matching, approximate" |
| 794 |
.IX Subsection "How can I do approximate matching?" |
| 795 |
See the module String::Approx available from CPAN. |
| 796 |
.SS "How do I efficiently match many regular expressions at once?" |
| 797 |
.IX Xref "regex, efficiency regexp, efficiency regular expression, efficiency" |
| 798 |
.IX Subsection "How do I efficiently match many regular expressions at once?" |
| 799 |
(contributed by brian d foy) |
| 800 |
.PP |
| 801 |
You want to |
| 802 |
avoid compiling a regular expression every time you want to match it. |
| 803 |
In this example, perl must recompile the regular expression for every |
| 804 |
iteration of the \f(CW\*(C`foreach\*(C'\fR loop since \f(CW$pattern\fR can change: |
| 805 |
.PP |
| 806 |
.Vb 1 |
| 807 |
\& my @patterns = qw( fo+ ba[rz] ); |
| 808 |
\& |
| 809 |
\& LINE: while( my $line = <> ) { |
| 810 |
\& foreach my $pattern ( @patterns ) { |
| 811 |
\& if( $line =~ m/\eb$pattern\eb/i ) { |
| 812 |
\& print $line; |
| 813 |
\& next LINE; |
| 814 |
\& } |
| 815 |
\& } |
| 816 |
\& } |
| 817 |
.Ve |
| 818 |
.PP |
| 819 |
The \f(CW\*(C`qr//\*(C'\fR operator compiles a regular |
| 820 |
expression, but doesn't apply it. When you use the pre-compiled |
| 821 |
version of the regex, perl does less work. In this example, I inserted |
| 822 |
a \f(CW\*(C`map\*(C'\fR to turn each pattern into its pre-compiled form. The rest of |
| 823 |
the script is the same, but faster: |
| 824 |
.PP |
| 825 |
.Vb 1 |
| 826 |
\& my @patterns = map { qr/\eb$_\eb/i } qw( fo+ ba[rz] ); |
| 827 |
\& |
| 828 |
\& LINE: while( my $line = <> ) { |
| 829 |
\& foreach my $pattern ( @patterns ) { |
| 830 |
\& if( $line =~ m/$pattern/ ) { |
| 831 |
\& print $line; |
| 832 |
\& next LINE; |
| 833 |
\& } |
| 834 |
\& } |
| 835 |
\& } |
| 836 |
.Ve |
| 837 |
.PP |
| 838 |
In some cases, you may be able to make several patterns into a single |
| 839 |
regular expression. Beware of situations that require backtracking |
| 840 |
though. In this example, the regex is only compiled once because |
| 841 |
\&\f(CW$regex\fR doesn't change between iterations: |
| 842 |
.PP |
| 843 |
.Vb 1 |
| 844 |
\& my $regex = join \*(Aq|\*(Aq, qw( fo+ ba[rz] ); |
| 845 |
\& |
| 846 |
\& while( my $line = <> ) { |
| 847 |
\& print if $line =~ m/\eb(?:$regex)\eb/i; |
| 848 |
\& } |
| 849 |
.Ve |
| 850 |
.PP |
| 851 |
The function "list2re" in Data::Munge on CPAN can also be used to form |
| 852 |
a single regex that matches a list of literal strings (not regexes). |
| 853 |
.PP |
| 854 |
For more details on regular expression efficiency, see \fIMastering |
| 855 |
Regular Expressions\fR by Jeffrey Friedl. He explains how the regular |
| 856 |
expressions engine works and why some patterns are surprisingly |
| 857 |
inefficient. Once you understand how perl applies regular expressions, |
| 858 |
you can tune them for individual situations. |
| 859 |
.ie n .SS "Why don't word-boundary searches with ""\eb"" work for me?" |
| 860 |
.el .SS "Why don't word-boundary searches with \f(CW\eb\fP work for me?" |
| 861 |
.IX Xref "\\b" |
| 862 |
.IX Subsection "Why don't word-boundary searches with b work for me?" |
| 863 |
(contributed by brian d foy) |
| 864 |
.PP |
| 865 |
Ensure that you know what \eb really does: it's the boundary between a |
| 866 |
word character, \ew, and something that isn't a word character. That |
| 867 |
thing that isn't a word character might be \eW, but it can also be the |
| 868 |
start or end of the string. |
| 869 |
.PP |
| 870 |
It's not (not!) the boundary between whitespace and non-whitespace, |
| 871 |
and it's not the stuff between words we use to create sentences. |
| 872 |
.PP |
| 873 |
In regex speak, a word boundary (\eb) is a "zero width assertion", |
| 874 |
meaning that it doesn't represent a character in the string, but a |
| 875 |
condition at a certain position. |
| 876 |
.PP |
| 877 |
For the regular expression, /\ebPerl\eb/, there has to be a word |
| 878 |
boundary before the "P" and after the "l". As long as something other |
| 879 |
than a word character precedes the "P" and succeeds the "l", the |
| 880 |
pattern will match. These strings match /\ebPerl\eb/. |
| 881 |
.PP |
| 882 |
.Vb 4 |
| 883 |
\& "Perl" # no word char before "P" or after "l" |
| 884 |
\& "Perl " # same as previous (space is not a word char) |
| 885 |
\& "\*(AqPerl\*(Aq" # the "\*(Aq" char is not a word char |
| 886 |
\& "Perl\*(Aqs" # no word char before "P", non\-word char after "l" |
| 887 |
.Ve |
| 888 |
.PP |
| 889 |
These strings do not match /\ebPerl\eb/. |
| 890 |
.PP |
| 891 |
.Vb 2 |
| 892 |
\& "Perl_" # "_" is a word char! |
| 893 |
\& "Perler" # no word char before "P", but one after "l" |
| 894 |
.Ve |
| 895 |
.PP |
| 896 |
You don't have to use \eb to match words though. You can look for |
| 897 |
non-word characters surrounded by word characters. These strings |
| 898 |
match the pattern /\eb'\eb/. |
| 899 |
.PP |
| 900 |
.Vb 2 |
| 901 |
\& "don\*(Aqt" # the "\*(Aq" char is surrounded by "n" and "t" |
| 902 |
\& "qep\*(Aqa\*(Aq" # the "\*(Aq" char is surrounded by "p" and "a" |
| 903 |
.Ve |
| 904 |
.PP |
| 905 |
These strings do not match /\eb'\eb/. |
| 906 |
.PP |
| 907 |
.Vb 1 |
| 908 |
\& "foo\*(Aq" # there is no word char after non\-word "\*(Aq" |
| 909 |
.Ve |
| 910 |
.PP |
| 911 |
You can also use the complement of \eb, \eB, to specify that there |
| 912 |
should not be a word boundary. |
| 913 |
.PP |
| 914 |
In the pattern /\eBam\eB/, there must be a word character before the "a" |
| 915 |
and after the "m". These patterns match /\eBam\eB/: |
| 916 |
.PP |
| 917 |
.Vb 2 |
| 918 |
\& "llama" # "am" surrounded by word chars |
| 919 |
\& "Samuel" # same |
| 920 |
.Ve |
| 921 |
.PP |
| 922 |
These strings do not match /\eBam\eB/ |
| 923 |
.PP |
| 924 |
.Vb 2 |
| 925 |
\& "Sam" # no word boundary before "a", but one after "m" |
| 926 |
\& "I am Sam" # "am" surrounded by non\-word chars |
| 927 |
.Ve |
| 928 |
.SS "Why does using $&, $`, or $' slow my program down?" |
| 929 |
.IX Xref "$MATCH $& $POSTMATCH $' $PREMATCH $`" |
| 930 |
.IX Subsection "Why does using $&, $`, or $' slow my program down?" |
| 931 |
(contributed by Anno Siegel) |
| 932 |
.PP |
| 933 |
Once Perl sees that you need one of these variables anywhere in the |
| 934 |
program, it provides them on each and every pattern match. That means |
| 935 |
that on every pattern match the entire string will be copied, part of it |
| 936 |
to $`, part to $&, and part to $'. Thus the penalty is most severe with |
| 937 |
long strings and patterns that match often. Avoid $&, $', and $` if you |
| 938 |
can, but if you can't, once you've used them at all, use them at will |
| 939 |
because you've already paid the price. Remember that some algorithms |
| 940 |
really appreciate them. As of the 5.005 release, the $& variable is no |
| 941 |
longer "expensive" the way the other two are. |
| 942 |
.PP |
| 943 |
Since Perl 5.6.1 the special variables @\- and @+ can functionally replace |
| 944 |
$`, $& and $'. These arrays contain pointers to the beginning and end |
| 945 |
of each match (see perlvar for the full story), so they give you |
| 946 |
essentially the same information, but without the risk of excessive |
| 947 |
string copying. |
| 948 |
.PP |
| 949 |
Perl 5.10 added three specials, \f(CW\*(C`${^MATCH}\*(C'\fR, \f(CW\*(C`${^PREMATCH}\*(C'\fR, and |
| 950 |
\&\f(CW\*(C`${^POSTMATCH}\*(C'\fR to do the same job but without the global performance |
| 951 |
penalty. Perl 5.10 only sets these variables if you compile or execute the |
| 952 |
regular expression with the \f(CW\*(C`/p\*(C'\fR modifier. |
| 953 |
.ie n .SS "What good is ""\eG"" in a regular expression?" |
| 954 |
.el .SS "What good is \f(CW\eG\fP in a regular expression?" |
| 955 |
.IX Xref "\\G" |
| 956 |
.IX Subsection "What good is G in a regular expression?" |
| 957 |
You use the \f(CW\*(C`\eG\*(C'\fR anchor to start the next match on the same |
| 958 |
string where the last match left off. The regular |
| 959 |
expression engine cannot skip over any characters to find |
| 960 |
the next match with this anchor, so \f(CW\*(C`\eG\*(C'\fR is similar to the |
| 961 |
beginning of string anchor, \f(CW\*(C`^\*(C'\fR. The \f(CW\*(C`\eG\*(C'\fR anchor is typically |
| 962 |
used with the \f(CW\*(C`g\*(C'\fR modifier. It uses the value of \f(CWpos()\fR |
| 963 |
as the position to start the next match. As the match |
| 964 |
operator makes successive matches, it updates \f(CWpos()\fR with the |
| 965 |
position of the next character past the last match (or the |
| 966 |
first character of the next match, depending on how you like |
| 967 |
to look at it). Each string has its own \f(CWpos()\fR value. |
| 968 |
.PP |
| 969 |
Suppose you want to match all of consecutive pairs of digits |
| 970 |
in a string like "1122a44" and stop matching when you |
| 971 |
encounter non-digits. You want to match \f(CW11\fR and \f(CW22\fR but |
| 972 |
the letter \f(CW\*(C`a\*(C'\fR shows up between \f(CW22\fR and \f(CW44\fR and you want |
| 973 |
to stop at \f(CW\*(C`a\*(C'\fR. Simply matching pairs of digits skips over |
| 974 |
the \f(CW\*(C`a\*(C'\fR and still matches \f(CW44\fR. |
| 975 |
.PP |
| 976 |
.Vb 2 |
| 977 |
\& $_ = "1122a44"; |
| 978 |
\& my @pairs = m/(\ed\ed)/g; # qw( 11 22 44 ) |
| 979 |
.Ve |
| 980 |
.PP |
| 981 |
If you use the \f(CW\*(C`\eG\*(C'\fR anchor, you force the match after \f(CW22\fR to |
| 982 |
start with the \f(CW\*(C`a\*(C'\fR. The regular expression cannot match |
| 983 |
there since it does not find a digit, so the next match |
| 984 |
fails and the match operator returns the pairs it already |
| 985 |
found. |
| 986 |
.PP |
| 987 |
.Vb 2 |
| 988 |
\& $_ = "1122a44"; |
| 989 |
\& my @pairs = m/\eG(\ed\ed)/g; # qw( 11 22 ) |
| 990 |
.Ve |
| 991 |
.PP |
| 992 |
You can also use the \f(CW\*(C`\eG\*(C'\fR anchor in scalar context. You |
| 993 |
still need the \f(CW\*(C`g\*(C'\fR modifier. |
| 994 |
.PP |
| 995 |
.Vb 4 |
| 996 |
\& $_ = "1122a44"; |
| 997 |
\& while( m/\eG(\ed\ed)/g ) { |
| 998 |
\& print "Found $1\en"; |
| 999 |
\& } |
| 1000 |
.Ve |
| 1001 |
.PP |
| 1002 |
After the match fails at the letter \f(CW\*(C`a\*(C'\fR, perl resets \f(CWpos()\fR |
| 1003 |
and the next match on the same string starts at the beginning. |
| 1004 |
.PP |
| 1005 |
.Vb 4 |
| 1006 |
\& $_ = "1122a44"; |
| 1007 |
\& while( m/\eG(\ed\ed)/g ) { |
| 1008 |
\& print "Found $1\en"; |
| 1009 |
\& } |
| 1010 |
\& |
| 1011 |
\& print "Found $1 after while" if m/(\ed\ed)/g; # finds "11" |
| 1012 |
.Ve |
| 1013 |
.PP |
| 1014 |
You can disable \f(CWpos()\fR resets on fail with the \f(CW\*(C`c\*(C'\fR modifier, documented |
| 1015 |
in perlop and perlreref. Subsequent matches start where the last |
| 1016 |
successful match ended (the value of \f(CWpos()\fR) even if a match on the |
| 1017 |
same string has failed in the meantime. In this case, the match after |
| 1018 |
the \f(CWwhile()\fR loop starts at the \f(CW\*(C`a\*(C'\fR (where the last match stopped), |
| 1019 |
and since it does not use any anchor it can skip over the \f(CW\*(C`a\*(C'\fR to find |
| 1020 |
\&\f(CW44\fR. |
| 1021 |
.PP |
| 1022 |
.Vb 4 |
| 1023 |
\& $_ = "1122a44"; |
| 1024 |
\& while( m/\eG(\ed\ed)/gc ) { |
| 1025 |
\& print "Found $1\en"; |
| 1026 |
\& } |
| 1027 |
\& |
| 1028 |
\& print "Found $1 after while" if m/(\ed\ed)/g; # finds "44" |
| 1029 |
.Ve |
| 1030 |
.PP |
| 1031 |
Typically you use the \f(CW\*(C`\eG\*(C'\fR anchor with the \f(CW\*(C`c\*(C'\fR modifier |
| 1032 |
when you want to try a different match if one fails, |
| 1033 |
such as in a tokenizer. Jeffrey Friedl offers this example |
| 1034 |
which works in 5.004 or later. |
| 1035 |
.PP |
| 1036 |
.Vb 9 |
| 1037 |
\& while (<>) { |
| 1038 |
\& chomp; |
| 1039 |
\& PARSER: { |
| 1040 |
\& m/ \eG( \ed+\eb )/gcx && do { print "number: $1\en"; redo; }; |
| 1041 |
\& m/ \eG( \ew+ )/gcx && do { print "word: $1\en"; redo; }; |
| 1042 |
\& m/ \eG( \es+ )/gcx && do { print "space: $1\en"; redo; }; |
| 1043 |
\& m/ \eG( [^\ew\ed]+ )/gcx && do { print "other: $1\en"; redo; }; |
| 1044 |
\& } |
| 1045 |
\& } |
| 1046 |
.Ve |
| 1047 |
.PP |
| 1048 |
For each line, the \f(CW\*(C`PARSER\*(C'\fR loop first tries to match a series |
| 1049 |
of digits followed by a word boundary. This match has to |
| 1050 |
start at the place the last match left off (or the beginning |
| 1051 |
of the string on the first match). Since \f(CW\*(C`m/ \eG( \ed+\eb |
| 1052 |
)/gcx\*(C'\fR uses the \f(CW\*(C`c\*(C'\fR modifier, if the string does not match that |
| 1053 |
regular expression, perl does not reset \fBpos()\fR and the next |
| 1054 |
match starts at the same position to try a different |
| 1055 |
pattern. |
| 1056 |
.SS "Are Perl regexes DFAs or NFAs? Are they POSIX compliant?" |
| 1057 |
.IX Xref "DFA NFA POSIX" |
| 1058 |
.IX Subsection "Are Perl regexes DFAs or NFAs? Are they POSIX compliant?" |
| 1059 |
While it's true that Perl's regular expressions resemble the DFAs |
| 1060 |
(deterministic finite automata) of the \fBegrep\fR\|(1) program, they are in |
| 1061 |
fact implemented as NFAs (non-deterministic finite automata) to allow |
| 1062 |
backtracking and backreferencing. And they aren't POSIX-style either, |
| 1063 |
because those guarantee worst-case behavior for all cases. (It seems |
| 1064 |
that some people prefer guarantees of consistency, even when what's |
| 1065 |
guaranteed is slowness.) See the book "Mastering Regular Expressions" |
| 1066 |
(from O'Reilly) by Jeffrey Friedl for all the details you could ever |
| 1067 |
hope to know on these matters (a full citation appears in |
| 1068 |
perlfaq2). |
| 1069 |
.SS "What's wrong with using grep in a void context?" |
| 1070 |
.IX Xref "grep" |
| 1071 |
.IX Subsection "What's wrong with using grep in a void context?" |
| 1072 |
The problem is that grep builds a return list, regardless of the context. |
| 1073 |
This means you're making Perl go to the trouble of building a list that |
| 1074 |
you then just throw away. If the list is large, you waste both time and space. |
| 1075 |
If your intent is to iterate over the list, then use a for loop for this |
| 1076 |
purpose. |
| 1077 |
.PP |
| 1078 |
In perls older than 5.8.1, map suffers from this problem as well. |
| 1079 |
But since 5.8.1, this has been fixed, and map is context aware \- in void |
| 1080 |
context, no lists are constructed. |
| 1081 |
.SS "How can I match strings with multibyte characters?" |
| 1082 |
.IX Xref "regex, and multibyte characters regexp, and multibyte characters regular expression, and multibyte characters martian encoding, Martian" |
| 1083 |
.IX Subsection "How can I match strings with multibyte characters?" |
| 1084 |
Starting from Perl 5.6 Perl has had some level of multibyte character |
| 1085 |
support. Perl 5.8 or later is recommended. Supported multibyte |
| 1086 |
character repertoires include Unicode, and legacy encodings |
| 1087 |
through the Encode module. See perluniintro, perlunicode, |
| 1088 |
and Encode. |
| 1089 |
.PP |
| 1090 |
If you are stuck with older Perls, you can do Unicode with the |
| 1091 |
Unicode::String module, and character conversions using the |
| 1092 |
Unicode::Map8 and Unicode::Map modules. If you are using |
| 1093 |
Japanese encodings, you might try using the jperl 5.005_03. |
| 1094 |
.PP |
| 1095 |
Finally, the following set of approaches was offered by Jeffrey |
| 1096 |
Friedl, whose article in issue #5 of The Perl Journal talks about |
| 1097 |
this very matter. |
| 1098 |
.PP |
| 1099 |
Let's suppose you have some weird Martian encoding where pairs of |
| 1100 |
ASCII uppercase letters encode single Martian letters (i.e. the two |
| 1101 |
bytes "CV" make a single Martian letter, as do the two bytes "SG", |
| 1102 |
"VS", "XX", etc.). Other bytes represent single characters, just like |
| 1103 |
ASCII. |
| 1104 |
.PP |
| 1105 |
So, the string of Martian "I am CVSGXX!" uses 12 bytes to encode the |
| 1106 |
nine characters 'I', ' ', 'a', 'm', ' ', 'CV', 'SG', 'XX', '!'. |
| 1107 |
.PP |
| 1108 |
Now, say you want to search for the single character \f(CW\*(C`/GX/\*(C'\fR. Perl |
| 1109 |
doesn't know about Martian, so it'll find the two bytes "GX" in the "I |
| 1110 |
am CVSGXX!" string, even though that character isn't there: it just |
| 1111 |
looks like it is because "SG" is next to "XX", but there's no real |
| 1112 |
"GX". This is a big problem. |
| 1113 |
.PP |
| 1114 |
Here are a few ways, all painful, to deal with it: |
| 1115 |
.PP |
| 1116 |
.Vb 2 |
| 1117 |
\& # Make sure adjacent "martian" bytes are no longer adjacent. |
| 1118 |
\& $martian =~ s/([A\-Z][A\-Z])/ $1 /g; |
| 1119 |
\& |
| 1120 |
\& print "found GX!\en" if $martian =~ /GX/; |
| 1121 |
.Ve |
| 1122 |
.PP |
| 1123 |
Or like this: |
| 1124 |
.PP |
| 1125 |
.Vb 6 |
| 1126 |
\& my @chars = $martian =~ m/([A\-Z][A\-Z]|[^A\-Z])/g; |
| 1127 |
\& # above is conceptually similar to: my @chars = $text =~ m/(.)/g; |
| 1128 |
\& # |
| 1129 |
\& foreach my $char (@chars) { |
| 1130 |
\& print "found GX!\en", last if $char eq \*(AqGX\*(Aq; |
| 1131 |
\& } |
| 1132 |
.Ve |
| 1133 |
.PP |
| 1134 |
Or like this: |
| 1135 |
.PP |
| 1136 |
.Vb 6 |
| 1137 |
\& while ($martian =~ m/\eG([A\-Z][A\-Z]|.)/gs) { # \eG probably unneeded |
| 1138 |
\& if ($1 eq \*(AqGX\*(Aq) { |
| 1139 |
\& print "found GX!\en"; |
| 1140 |
\& last; |
| 1141 |
\& } |
| 1142 |
\& } |
| 1143 |
.Ve |
| 1144 |
.PP |
| 1145 |
Here's another, slightly less painful, way to do it from Benjamin |
| 1146 |
Goldberg, who uses a zero-width negative look-behind assertion. |
| 1147 |
.PP |
| 1148 |
.Vb 5 |
| 1149 |
\& print "found GX!\en" if $martian =~ m/ |
| 1150 |
\& (?<![A\-Z]) |
| 1151 |
\& (?:[A\-Z][A\-Z])*? |
| 1152 |
\& GX |
| 1153 |
\& /x; |
| 1154 |
.Ve |
| 1155 |
.PP |
| 1156 |
This succeeds if the "martian" character GX is in the string, and fails |
| 1157 |
otherwise. If you don't like using (?<!), a zero-width negative |
| 1158 |
look-behind assertion, you can replace (?<![A\-Z]) with (?:^|[^A\-Z]). |
| 1159 |
.PP |
| 1160 |
It does have the drawback of putting the wrong thing in $\-[0] and $+[0], |
| 1161 |
but this usually can be worked around. |
| 1162 |
.SS "How do I match a regular expression that's in a variable?" |
| 1163 |
.IX Xref "regex, in variable eval regex quotemeta \\Q, regex \\E, regex qr" |
| 1164 |
.IX Subsection "How do I match a regular expression that's in a variable?" |
| 1165 |
(contributed by brian d foy) |
| 1166 |
.PP |
| 1167 |
We don't have to hard-code patterns into the match operator (or |
| 1168 |
anything else that works with regular expressions). We can put the |
| 1169 |
pattern in a variable for later use. |
| 1170 |
.PP |
| 1171 |
The match operator is a double quote context, so you can interpolate |
| 1172 |
your variable just like a double quoted string. In this case, you |
| 1173 |
read the regular expression as user input and store it in \f(CW$regex\fR. |
| 1174 |
Once you have the pattern in \f(CW$regex\fR, you use that variable in the |
| 1175 |
match operator. |
| 1176 |
.PP |
| 1177 |
.Vb 1 |
| 1178 |
\& chomp( my $regex = <STDIN> ); |
| 1179 |
\& |
| 1180 |
\& if( $string =~ m/$regex/ ) { ... } |
| 1181 |
.Ve |
| 1182 |
.PP |
| 1183 |
Any regular expression special characters in \f(CW$regex\fR are still |
| 1184 |
special, and the pattern still has to be valid or Perl will complain. |
| 1185 |
For instance, in this pattern there is an unpaired parenthesis. |
| 1186 |
.PP |
| 1187 |
.Vb 1 |
| 1188 |
\& my $regex = "Unmatched ( paren"; |
| 1189 |
\& |
| 1190 |
\& "Two parens to bind them all" =~ m/$regex/; |
| 1191 |
.Ve |
| 1192 |
.PP |
| 1193 |
When Perl compiles the regular expression, it treats the parenthesis |
| 1194 |
as the start of a memory match. When it doesn't find the closing |
| 1195 |
parenthesis, it complains: |
| 1196 |
.PP |
| 1197 |
.Vb 1 |
| 1198 |
\& Unmatched ( in regex; marked by <\-\- HERE in m/Unmatched ( <\-\- HERE paren/ at script line 3. |
| 1199 |
.Ve |
| 1200 |
.PP |
| 1201 |
You can get around this in several ways depending on our situation. |
| 1202 |
First, if you don't want any of the characters in the string to be |
| 1203 |
special, you can escape them with \f(CW\*(C`quotemeta\*(C'\fR before you use the string. |
| 1204 |
.PP |
| 1205 |
.Vb 2 |
| 1206 |
\& chomp( my $regex = <STDIN> ); |
| 1207 |
\& $regex = quotemeta( $regex ); |
| 1208 |
\& |
| 1209 |
\& if( $string =~ m/$regex/ ) { ... } |
| 1210 |
.Ve |
| 1211 |
.PP |
| 1212 |
You can also do this directly in the match operator using the \f(CW\*(C`\eQ\*(C'\fR |
| 1213 |
and \f(CW\*(C`\eE\*(C'\fR sequences. The \f(CW\*(C`\eQ\*(C'\fR tells Perl where to start escaping |
| 1214 |
special characters, and the \f(CW\*(C`\eE\*(C'\fR tells it where to stop (see perlop |
| 1215 |
for more details). |
| 1216 |
.PP |
| 1217 |
.Vb 1 |
| 1218 |
\& chomp( my $regex = <STDIN> ); |
| 1219 |
\& |
| 1220 |
\& if( $string =~ m/\eQ$regex\eE/ ) { ... } |
| 1221 |
.Ve |
| 1222 |
.PP |
| 1223 |
Alternately, you can use \f(CW\*(C`qr//\*(C'\fR, the regular expression quote operator (see |
| 1224 |
perlop for more details). It quotes and perhaps compiles the pattern, |
| 1225 |
and you can apply regular expression flags to the pattern. |
| 1226 |
.PP |
| 1227 |
.Vb 1 |
| 1228 |
\& chomp( my $input = <STDIN> ); |
| 1229 |
\& |
| 1230 |
\& my $regex = qr/$input/is; |
| 1231 |
\& |
| 1232 |
\& $string =~ m/$regex/ # same as m/$input/is; |
| 1233 |
.Ve |
| 1234 |
.PP |
| 1235 |
You might also want to trap any errors by wrapping an \f(CW\*(C`eval\*(C'\fR block |
| 1236 |
around the whole thing. |
| 1237 |
.PP |
| 1238 |
.Vb 1 |
| 1239 |
\& chomp( my $input = <STDIN> ); |
| 1240 |
\& |
| 1241 |
\& eval { |
| 1242 |
\& if( $string =~ m/\eQ$input\eE/ ) { ... } |
| 1243 |
\& }; |
| 1244 |
\& warn $@ if $@; |
| 1245 |
.Ve |
| 1246 |
.PP |
| 1247 |
Or... |
| 1248 |
.PP |
| 1249 |
.Vb 7 |
| 1250 |
\& my $regex = eval { qr/$input/is }; |
| 1251 |
\& if( defined $regex ) { |
| 1252 |
\& $string =~ m/$regex/; |
| 1253 |
\& } |
| 1254 |
\& else { |
| 1255 |
\& warn $@; |
| 1256 |
\& } |
| 1257 |
.Ve |
| 1258 |
.SH "AUTHOR AND COPYRIGHT" |
| 1259 |
.IX Header "AUTHOR AND COPYRIGHT" |
| 1260 |
Copyright (c) 1997\-2010 Tom Christiansen, Nathan Torkington, and |
| 1261 |
other authors as noted. All rights reserved. |
| 1262 |
.PP |
| 1263 |
This documentation is free; you can redistribute it and/or modify it |
| 1264 |
under the same terms as Perl itself. |
| 1265 |
.PP |
| 1266 |
Irrespective of its distribution, all code examples in this file |
| 1267 |
are hereby placed into the public domain. You are permitted and |
| 1268 |
encouraged to use this code in your own programs for fun |
| 1269 |
or for profit as you see fit. A simple comment in the code giving |
| 1270 |
credit would be courteous but is not required. |