5 * 'A fair jaw-cracker dwarf-language must be.' --Samwise Gamgee
7 * [p.285 of _The Lord of the Rings_, II/iii: "The Ring Goes South"]
10 /* This file contains functions for compiling a regular expression. See
11 * also regexec.c which funnily enough, contains functions for executing
12 * a regular expression.
14 * This file is also copied at build time to ext/re/re_comp.c, where
15 * it's built with -DPERL_EXT_RE_BUILD -DPERL_EXT_RE_DEBUG -DPERL_EXT.
16 * This causes the main functions to be compiled under new names and with
17 * debugging support added, which makes "use re 'debug'" work.
20 /* NOTE: this is derived from Henry Spencer's regexp code, and should not
21 * confused with the original package (see point 3 below). Thanks, Henry!
24 /* Additional note: this code is very heavily munged from Henry's version
25 * in places. In some spots I've traded clarity for efficiency, so don't
26 * blame Henry for some of the lack of readability.
29 /* The names of the functions have been changed from regcomp and
30 * regexec to pregcomp and pregexec in order to avoid conflicts
31 * with the POSIX routines of the same names.
34 #ifdef PERL_EXT_RE_BUILD
39 * pregcomp and pregexec -- regsub and regerror are not used in perl
41 * Copyright (c) 1986 by University of Toronto.
42 * Written by Henry Spencer. Not derived from licensed software.
44 * Permission is granted to anyone to use this software for any
45 * purpose on any computer system, and to redistribute it freely,
46 * subject to the following restrictions:
48 * 1. The author is not responsible for the consequences of use of
49 * this software, no matter how awful, even if they arise
52 * 2. The origin of this software must not be misrepresented, either
53 * by explicit claim or by omission.
55 * 3. Altered versions must be plainly marked as such, and must not
56 * be misrepresented as being the original software.
59 **** Alterations to Henry's code are...
61 **** Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
62 **** 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008
63 **** by Larry Wall and others
65 **** You may distribute under the terms of either the GNU General Public
66 **** License or the Artistic License, as specified in the README file.
69 * Beware that some of this code is subtly aware of the way operator
70 * precedence is structured in regular expressions. Serious changes in
71 * regular-expression syntax might require a total rethink.
74 #define PERL_IN_REGCOMP_C
77 #ifndef PERL_IN_XSUB_RE
82 #ifdef PERL_IN_XSUB_RE
88 #include "dquote_static.c"
89 #ifndef PERL_IN_XSUB_RE
90 # include "charclass_invlists.h"
98 # if defined(BUGGY_MSC6)
99 /* MSC 6.00A breaks on op/regexp.t test 85 unless we turn this off */
100 # pragma optimize("a",off)
101 /* But MSC 6.00A is happy with 'w', for aliases only across function calls*/
102 # pragma optimize("w",on )
103 # endif /* BUGGY_MSC6 */
107 #define STATIC static
110 typedef struct RExC_state_t {
111 U32 flags; /* are we folding, multilining? */
112 char *precomp; /* uncompiled string. */
113 REGEXP *rx_sv; /* The SV that is the regexp. */
114 regexp *rx; /* perl core regexp structure */
115 regexp_internal *rxi; /* internal data for regexp object pprivate field */
116 char *start; /* Start of input for compile */
117 char *end; /* End of input for compile */
118 char *parse; /* Input-scan pointer. */
119 I32 whilem_seen; /* number of WHILEM in this expr */
120 regnode *emit_start; /* Start of emitted-code area */
121 regnode *emit_bound; /* First regnode outside of the allocated space */
122 regnode *emit; /* Code-emit pointer; ®dummy = don't = compiling */
123 I32 naughty; /* How bad is this pattern? */
124 I32 sawback; /* Did we see \1, ...? */
126 I32 size; /* Code size. */
127 I32 npar; /* Capture buffer count, (OPEN). */
128 I32 cpar; /* Capture buffer count, (CLOSE). */
129 I32 nestroot; /* root parens we are in - used by accept */
133 regnode **open_parens; /* pointers to open parens */
134 regnode **close_parens; /* pointers to close parens */
135 regnode *opend; /* END node in program */
136 I32 utf8; /* whether the pattern is utf8 or not */
137 I32 orig_utf8; /* whether the pattern was originally in utf8 */
138 /* XXX use this for future optimisation of case
139 * where pattern must be upgraded to utf8. */
140 I32 uni_semantics; /* If a d charset modifier should use unicode
141 rules, even if the pattern is not in
143 HV *paren_names; /* Paren names */
145 regnode **recurse; /* Recurse regops */
146 I32 recurse_count; /* Number of recurse regops */
149 I32 override_recoding;
151 char *starttry; /* -Dr: where regtry was called. */
152 #define RExC_starttry (pRExC_state->starttry)
155 const char *lastparse;
157 AV *paren_name_list; /* idx -> name */
158 #define RExC_lastparse (pRExC_state->lastparse)
159 #define RExC_lastnum (pRExC_state->lastnum)
160 #define RExC_paren_name_list (pRExC_state->paren_name_list)
164 #define RExC_flags (pRExC_state->flags)
165 #define RExC_precomp (pRExC_state->precomp)
166 #define RExC_rx_sv (pRExC_state->rx_sv)
167 #define RExC_rx (pRExC_state->rx)
168 #define RExC_rxi (pRExC_state->rxi)
169 #define RExC_start (pRExC_state->start)
170 #define RExC_end (pRExC_state->end)
171 #define RExC_parse (pRExC_state->parse)
172 #define RExC_whilem_seen (pRExC_state->whilem_seen)
173 #ifdef RE_TRACK_PATTERN_OFFSETS
174 #define RExC_offsets (pRExC_state->rxi->u.offsets) /* I am not like the others */
176 #define RExC_emit (pRExC_state->emit)
177 #define RExC_emit_start (pRExC_state->emit_start)
178 #define RExC_emit_bound (pRExC_state->emit_bound)
179 #define RExC_naughty (pRExC_state->naughty)
180 #define RExC_sawback (pRExC_state->sawback)
181 #define RExC_seen (pRExC_state->seen)
182 #define RExC_size (pRExC_state->size)
183 #define RExC_npar (pRExC_state->npar)
184 #define RExC_nestroot (pRExC_state->nestroot)
185 #define RExC_extralen (pRExC_state->extralen)
186 #define RExC_seen_zerolen (pRExC_state->seen_zerolen)
187 #define RExC_seen_evals (pRExC_state->seen_evals)
188 #define RExC_utf8 (pRExC_state->utf8)
189 #define RExC_uni_semantics (pRExC_state->uni_semantics)
190 #define RExC_orig_utf8 (pRExC_state->orig_utf8)
191 #define RExC_open_parens (pRExC_state->open_parens)
192 #define RExC_close_parens (pRExC_state->close_parens)
193 #define RExC_opend (pRExC_state->opend)
194 #define RExC_paren_names (pRExC_state->paren_names)
195 #define RExC_recurse (pRExC_state->recurse)
196 #define RExC_recurse_count (pRExC_state->recurse_count)
197 #define RExC_in_lookbehind (pRExC_state->in_lookbehind)
198 #define RExC_contains_locale (pRExC_state->contains_locale)
199 #define RExC_override_recoding (pRExC_state->override_recoding)
202 #define ISMULT1(c) ((c) == '*' || (c) == '+' || (c) == '?')
203 #define ISMULT2(s) ((*s) == '*' || (*s) == '+' || (*s) == '?' || \
204 ((*s) == '{' && regcurly(s)))
207 #undef SPSTART /* dratted cpp namespace... */
210 * Flags to be passed up and down.
212 #define WORST 0 /* Worst case. */
213 #define HASWIDTH 0x01 /* Known to match non-null strings. */
215 /* Simple enough to be STAR/PLUS operand, in an EXACT node must be a single
216 * character, and if utf8, must be invariant. Note that this is not the same thing as REGNODE_SIMPLE */
218 #define SPSTART 0x04 /* Starts with * or +. */
219 #define TRYAGAIN 0x08 /* Weeded out a declaration. */
220 #define POSTPONED 0x10 /* (?1),(?&name), (??{...}) or similar */
222 #define REG_NODE_NUM(x) ((x) ? (int)((x)-RExC_emit_start) : -1)
224 /* whether trie related optimizations are enabled */
225 #if PERL_ENABLE_EXTENDED_TRIE_OPTIMISATION
226 #define TRIE_STUDY_OPT
227 #define FULL_TRIE_STUDY
233 #define PBYTE(u8str,paren) ((U8*)(u8str))[(paren) >> 3]
234 #define PBITVAL(paren) (1 << ((paren) & 7))
235 #define PAREN_TEST(u8str,paren) ( PBYTE(u8str,paren) & PBITVAL(paren))
236 #define PAREN_SET(u8str,paren) PBYTE(u8str,paren) |= PBITVAL(paren)
237 #define PAREN_UNSET(u8str,paren) PBYTE(u8str,paren) &= (~PBITVAL(paren))
239 /* If not already in utf8, do a longjmp back to the beginning */
240 #define UTF8_LONGJMP 42 /* Choose a value not likely to ever conflict */
241 #define REQUIRE_UTF8 STMT_START { \
242 if (! UTF) JMPENV_JUMP(UTF8_LONGJMP); \
245 /* About scan_data_t.
247 During optimisation we recurse through the regexp program performing
248 various inplace (keyhole style) optimisations. In addition study_chunk
249 and scan_commit populate this data structure with information about
250 what strings MUST appear in the pattern. We look for the longest
251 string that must appear at a fixed location, and we look for the
252 longest string that may appear at a floating location. So for instance
257 Both 'FOO' and 'A' are fixed strings. Both 'B' and 'BAR' are floating
258 strings (because they follow a .* construct). study_chunk will identify
259 both FOO and BAR as being the longest fixed and floating strings respectively.
261 The strings can be composites, for instance
265 will result in a composite fixed substring 'foo'.
267 For each string some basic information is maintained:
269 - offset or min_offset
270 This is the position the string must appear at, or not before.
271 It also implicitly (when combined with minlenp) tells us how many
272 characters must match before the string we are searching for.
273 Likewise when combined with minlenp and the length of the string it
274 tells us how many characters must appear after the string we have
278 Only used for floating strings. This is the rightmost point that
279 the string can appear at. If set to I32 max it indicates that the
280 string can occur infinitely far to the right.
283 A pointer to the minimum length of the pattern that the string
284 was found inside. This is important as in the case of positive
285 lookahead or positive lookbehind we can have multiple patterns
290 The minimum length of the pattern overall is 3, the minimum length
291 of the lookahead part is 3, but the minimum length of the part that
292 will actually match is 1. So 'FOO's minimum length is 3, but the
293 minimum length for the F is 1. This is important as the minimum length
294 is used to determine offsets in front of and behind the string being
295 looked for. Since strings can be composites this is the length of the
296 pattern at the time it was committed with a scan_commit. Note that
297 the length is calculated by study_chunk, so that the minimum lengths
298 are not known until the full pattern has been compiled, thus the
299 pointer to the value.
303 In the case of lookbehind the string being searched for can be
304 offset past the start point of the final matching string.
305 If this value was just blithely removed from the min_offset it would
306 invalidate some of the calculations for how many chars must match
307 before or after (as they are derived from min_offset and minlen and
308 the length of the string being searched for).
309 When the final pattern is compiled and the data is moved from the
310 scan_data_t structure into the regexp structure the information
311 about lookbehind is factored in, with the information that would
312 have been lost precalculated in the end_shift field for the
315 The fields pos_min and pos_delta are used to store the minimum offset
316 and the delta to the maximum offset at the current point in the pattern.
320 typedef struct scan_data_t {
321 /*I32 len_min; unused */
322 /*I32 len_delta; unused */
326 I32 last_end; /* min value, <0 unless valid. */
329 SV **longest; /* Either &l_fixed, or &l_float. */
330 SV *longest_fixed; /* longest fixed string found in pattern */
331 I32 offset_fixed; /* offset where it starts */
332 I32 *minlen_fixed; /* pointer to the minlen relevant to the string */
333 I32 lookbehind_fixed; /* is the position of the string modfied by LB */
334 SV *longest_float; /* longest floating string found in pattern */
335 I32 offset_float_min; /* earliest point in string it can appear */
336 I32 offset_float_max; /* latest point in string it can appear */
337 I32 *minlen_float; /* pointer to the minlen relevant to the string */
338 I32 lookbehind_float; /* is the position of the string modified by LB */
342 struct regnode_charclass_class *start_class;
346 * Forward declarations for pregcomp()'s friends.
349 static const scan_data_t zero_scan_data =
350 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ,0};
352 #define SF_BEFORE_EOL (SF_BEFORE_SEOL|SF_BEFORE_MEOL)
353 #define SF_BEFORE_SEOL 0x0001
354 #define SF_BEFORE_MEOL 0x0002
355 #define SF_FIX_BEFORE_EOL (SF_FIX_BEFORE_SEOL|SF_FIX_BEFORE_MEOL)
356 #define SF_FL_BEFORE_EOL (SF_FL_BEFORE_SEOL|SF_FL_BEFORE_MEOL)
359 # define SF_FIX_SHIFT_EOL (0+2)
360 # define SF_FL_SHIFT_EOL (0+4)
362 # define SF_FIX_SHIFT_EOL (+2)
363 # define SF_FL_SHIFT_EOL (+4)
366 #define SF_FIX_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FIX_SHIFT_EOL)
367 #define SF_FIX_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FIX_SHIFT_EOL)
369 #define SF_FL_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FL_SHIFT_EOL)
370 #define SF_FL_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FL_SHIFT_EOL) /* 0x20 */
371 #define SF_IS_INF 0x0040
372 #define SF_HAS_PAR 0x0080
373 #define SF_IN_PAR 0x0100
374 #define SF_HAS_EVAL 0x0200
375 #define SCF_DO_SUBSTR 0x0400
376 #define SCF_DO_STCLASS_AND 0x0800
377 #define SCF_DO_STCLASS_OR 0x1000
378 #define SCF_DO_STCLASS (SCF_DO_STCLASS_AND|SCF_DO_STCLASS_OR)
379 #define SCF_WHILEM_VISITED_POS 0x2000
381 #define SCF_TRIE_RESTUDY 0x4000 /* Do restudy? */
382 #define SCF_SEEN_ACCEPT 0x8000
384 #define UTF cBOOL(RExC_utf8)
386 /* The enums for all these are ordered so things work out correctly */
387 #define LOC (get_regex_charset(RExC_flags) == REGEX_LOCALE_CHARSET)
388 #define DEPENDS_SEMANTICS (get_regex_charset(RExC_flags) == REGEX_DEPENDS_CHARSET)
389 #define UNI_SEMANTICS (get_regex_charset(RExC_flags) == REGEX_UNICODE_CHARSET)
390 #define AT_LEAST_UNI_SEMANTICS (get_regex_charset(RExC_flags) >= REGEX_UNICODE_CHARSET)
391 #define ASCII_RESTRICTED (get_regex_charset(RExC_flags) == REGEX_ASCII_RESTRICTED_CHARSET)
392 #define MORE_ASCII_RESTRICTED (get_regex_charset(RExC_flags) == REGEX_ASCII_MORE_RESTRICTED_CHARSET)
393 #define AT_LEAST_ASCII_RESTRICTED (get_regex_charset(RExC_flags) >= REGEX_ASCII_RESTRICTED_CHARSET)
395 #define FOLD cBOOL(RExC_flags & RXf_PMf_FOLD)
397 #define OOB_UNICODE 12345678
398 #define OOB_NAMEDCLASS -1
400 #define CHR_SVLEN(sv) (UTF ? sv_len_utf8(sv) : SvCUR(sv))
401 #define CHR_DIST(a,b) (UTF ? utf8_distance(a,b) : a - b)
404 /* length of regex to show in messages that don't mark a position within */
405 #define RegexLengthToShowInErrorMessages 127
408 * If MARKER[12] are adjusted, be sure to adjust the constants at the top
409 * of t/op/regmesg.t, the tests in t/op/re_tests, and those in
410 * op/pragma/warn/regcomp.
412 #define MARKER1 "<-- HERE" /* marker as it appears in the description */
413 #define MARKER2 " <-- HERE " /* marker as it appears within the regex */
415 #define REPORT_LOCATION " in regex; marked by " MARKER1 " in m/%.*s" MARKER2 "%s/"
418 * Calls SAVEDESTRUCTOR_X if needed, then calls Perl_croak with the given
419 * arg. Show regex, up to a maximum length. If it's too long, chop and add
422 #define _FAIL(code) STMT_START { \
423 const char *ellipses = ""; \
424 IV len = RExC_end - RExC_precomp; \
427 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx_sv); \
428 if (len > RegexLengthToShowInErrorMessages) { \
429 /* chop 10 shorter than the max, to ensure meaning of "..." */ \
430 len = RegexLengthToShowInErrorMessages - 10; \
436 #define FAIL(msg) _FAIL( \
437 Perl_croak(aTHX_ "%s in regex m/%.*s%s/", \
438 msg, (int)len, RExC_precomp, ellipses))
440 #define FAIL2(msg,arg) _FAIL( \
441 Perl_croak(aTHX_ msg " in regex m/%.*s%s/", \
442 arg, (int)len, RExC_precomp, ellipses))
445 * Simple_vFAIL -- like FAIL, but marks the current location in the scan
447 #define Simple_vFAIL(m) STMT_START { \
448 const IV offset = RExC_parse - RExC_precomp; \
449 Perl_croak(aTHX_ "%s" REPORT_LOCATION, \
450 m, (int)offset, RExC_precomp, RExC_precomp + offset); \
454 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL()
456 #define vFAIL(m) STMT_START { \
458 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx_sv); \
463 * Like Simple_vFAIL(), but accepts two arguments.
465 #define Simple_vFAIL2(m,a1) STMT_START { \
466 const IV offset = RExC_parse - RExC_precomp; \
467 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, \
468 (int)offset, RExC_precomp, RExC_precomp + offset); \
472 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL2().
474 #define vFAIL2(m,a1) STMT_START { \
476 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx_sv); \
477 Simple_vFAIL2(m, a1); \
482 * Like Simple_vFAIL(), but accepts three arguments.
484 #define Simple_vFAIL3(m, a1, a2) STMT_START { \
485 const IV offset = RExC_parse - RExC_precomp; \
486 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, \
487 (int)offset, RExC_precomp, RExC_precomp + offset); \
491 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL3().
493 #define vFAIL3(m,a1,a2) STMT_START { \
495 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx_sv); \
496 Simple_vFAIL3(m, a1, a2); \
500 * Like Simple_vFAIL(), but accepts four arguments.
502 #define Simple_vFAIL4(m, a1, a2, a3) STMT_START { \
503 const IV offset = RExC_parse - RExC_precomp; \
504 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, a3, \
505 (int)offset, RExC_precomp, RExC_precomp + offset); \
508 #define ckWARNreg(loc,m) STMT_START { \
509 const IV offset = loc - RExC_precomp; \
510 Perl_ck_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
511 (int)offset, RExC_precomp, RExC_precomp + offset); \
514 #define ckWARNregdep(loc,m) STMT_START { \
515 const IV offset = loc - RExC_precomp; \
516 Perl_ck_warner_d(aTHX_ packWARN2(WARN_DEPRECATED, WARN_REGEXP), \
518 (int)offset, RExC_precomp, RExC_precomp + offset); \
521 #define ckWARN2regdep(loc,m, a1) STMT_START { \
522 const IV offset = loc - RExC_precomp; \
523 Perl_ck_warner_d(aTHX_ packWARN2(WARN_DEPRECATED, WARN_REGEXP), \
525 a1, (int)offset, RExC_precomp, RExC_precomp + offset); \
528 #define ckWARN2reg(loc, m, a1) STMT_START { \
529 const IV offset = loc - RExC_precomp; \
530 Perl_ck_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
531 a1, (int)offset, RExC_precomp, RExC_precomp + offset); \
534 #define vWARN3(loc, m, a1, a2) STMT_START { \
535 const IV offset = loc - RExC_precomp; \
536 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
537 a1, a2, (int)offset, RExC_precomp, RExC_precomp + offset); \
540 #define ckWARN3reg(loc, m, a1, a2) STMT_START { \
541 const IV offset = loc - RExC_precomp; \
542 Perl_ck_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
543 a1, a2, (int)offset, RExC_precomp, RExC_precomp + offset); \
546 #define vWARN4(loc, m, a1, a2, a3) STMT_START { \
547 const IV offset = loc - RExC_precomp; \
548 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
549 a1, a2, a3, (int)offset, RExC_precomp, RExC_precomp + offset); \
552 #define ckWARN4reg(loc, m, a1, a2, a3) STMT_START { \
553 const IV offset = loc - RExC_precomp; \
554 Perl_ck_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
555 a1, a2, a3, (int)offset, RExC_precomp, RExC_precomp + offset); \
558 #define vWARN5(loc, m, a1, a2, a3, a4) STMT_START { \
559 const IV offset = loc - RExC_precomp; \
560 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
561 a1, a2, a3, a4, (int)offset, RExC_precomp, RExC_precomp + offset); \
565 /* Allow for side effects in s */
566 #define REGC(c,s) STMT_START { \
567 if (!SIZE_ONLY) *(s) = (c); else (void)(s); \
570 /* Macros for recording node offsets. 20001227 mjd@plover.com
571 * Nodes are numbered 1, 2, 3, 4. Node #n's position is recorded in
572 * element 2*n-1 of the array. Element #2n holds the byte length node #n.
573 * Element 0 holds the number n.
574 * Position is 1 indexed.
576 #ifndef RE_TRACK_PATTERN_OFFSETS
577 #define Set_Node_Offset_To_R(node,byte)
578 #define Set_Node_Offset(node,byte)
579 #define Set_Cur_Node_Offset
580 #define Set_Node_Length_To_R(node,len)
581 #define Set_Node_Length(node,len)
582 #define Set_Node_Cur_Length(node)
583 #define Node_Offset(n)
584 #define Node_Length(n)
585 #define Set_Node_Offset_Length(node,offset,len)
586 #define ProgLen(ri) ri->u.proglen
587 #define SetProgLen(ri,x) ri->u.proglen = x
589 #define ProgLen(ri) ri->u.offsets[0]
590 #define SetProgLen(ri,x) ri->u.offsets[0] = x
591 #define Set_Node_Offset_To_R(node,byte) STMT_START { \
593 MJD_OFFSET_DEBUG(("** (%d) offset of node %d is %d.\n", \
594 __LINE__, (int)(node), (int)(byte))); \
596 Perl_croak(aTHX_ "value of node is %d in Offset macro", (int)(node)); \
598 RExC_offsets[2*(node)-1] = (byte); \
603 #define Set_Node_Offset(node,byte) \
604 Set_Node_Offset_To_R((node)-RExC_emit_start, (byte)-RExC_start)
605 #define Set_Cur_Node_Offset Set_Node_Offset(RExC_emit, RExC_parse)
607 #define Set_Node_Length_To_R(node,len) STMT_START { \
609 MJD_OFFSET_DEBUG(("** (%d) size of node %d is %d.\n", \
610 __LINE__, (int)(node), (int)(len))); \
612 Perl_croak(aTHX_ "value of node is %d in Length macro", (int)(node)); \
614 RExC_offsets[2*(node)] = (len); \
619 #define Set_Node_Length(node,len) \
620 Set_Node_Length_To_R((node)-RExC_emit_start, len)
621 #define Set_Cur_Node_Length(len) Set_Node_Length(RExC_emit, len)
622 #define Set_Node_Cur_Length(node) \
623 Set_Node_Length(node, RExC_parse - parse_start)
625 /* Get offsets and lengths */
626 #define Node_Offset(n) (RExC_offsets[2*((n)-RExC_emit_start)-1])
627 #define Node_Length(n) (RExC_offsets[2*((n)-RExC_emit_start)])
629 #define Set_Node_Offset_Length(node,offset,len) STMT_START { \
630 Set_Node_Offset_To_R((node)-RExC_emit_start, (offset)); \
631 Set_Node_Length_To_R((node)-RExC_emit_start, (len)); \
635 #if PERL_ENABLE_EXPERIMENTAL_REGEX_OPTIMISATIONS
636 #define EXPERIMENTAL_INPLACESCAN
637 #endif /*PERL_ENABLE_EXPERIMENTAL_REGEX_OPTIMISATIONS*/
639 #define DEBUG_STUDYDATA(str,data,depth) \
640 DEBUG_OPTIMISE_MORE_r(if(data){ \
641 PerlIO_printf(Perl_debug_log, \
642 "%*s" str "Pos:%"IVdf"/%"IVdf \
643 " Flags: 0x%"UVXf" Whilem_c: %"IVdf" Lcp: %"IVdf" %s", \
644 (int)(depth)*2, "", \
645 (IV)((data)->pos_min), \
646 (IV)((data)->pos_delta), \
647 (UV)((data)->flags), \
648 (IV)((data)->whilem_c), \
649 (IV)((data)->last_closep ? *((data)->last_closep) : -1), \
650 is_inf ? "INF " : "" \
652 if ((data)->last_found) \
653 PerlIO_printf(Perl_debug_log, \
654 "Last:'%s' %"IVdf":%"IVdf"/%"IVdf" %sFixed:'%s' @ %"IVdf \
655 " %sFloat: '%s' @ %"IVdf"/%"IVdf"", \
656 SvPVX_const((data)->last_found), \
657 (IV)((data)->last_end), \
658 (IV)((data)->last_start_min), \
659 (IV)((data)->last_start_max), \
660 ((data)->longest && \
661 (data)->longest==&((data)->longest_fixed)) ? "*" : "", \
662 SvPVX_const((data)->longest_fixed), \
663 (IV)((data)->offset_fixed), \
664 ((data)->longest && \
665 (data)->longest==&((data)->longest_float)) ? "*" : "", \
666 SvPVX_const((data)->longest_float), \
667 (IV)((data)->offset_float_min), \
668 (IV)((data)->offset_float_max) \
670 PerlIO_printf(Perl_debug_log,"\n"); \
673 static void clear_re(pTHX_ void *r);
675 /* Mark that we cannot extend a found fixed substring at this point.
676 Update the longest found anchored substring and the longest found
677 floating substrings if needed. */
680 S_scan_commit(pTHX_ const RExC_state_t *pRExC_state, scan_data_t *data, I32 *minlenp, int is_inf)
682 const STRLEN l = CHR_SVLEN(data->last_found);
683 const STRLEN old_l = CHR_SVLEN(*data->longest);
684 GET_RE_DEBUG_FLAGS_DECL;
686 PERL_ARGS_ASSERT_SCAN_COMMIT;
688 if ((l >= old_l) && ((l > old_l) || (data->flags & SF_BEFORE_EOL))) {
689 SvSetMagicSV(*data->longest, data->last_found);
690 if (*data->longest == data->longest_fixed) {
691 data->offset_fixed = l ? data->last_start_min : data->pos_min;
692 if (data->flags & SF_BEFORE_EOL)
694 |= ((data->flags & SF_BEFORE_EOL) << SF_FIX_SHIFT_EOL);
696 data->flags &= ~SF_FIX_BEFORE_EOL;
697 data->minlen_fixed=minlenp;
698 data->lookbehind_fixed=0;
700 else { /* *data->longest == data->longest_float */
701 data->offset_float_min = l ? data->last_start_min : data->pos_min;
702 data->offset_float_max = (l
703 ? data->last_start_max
704 : data->pos_min + data->pos_delta);
705 if (is_inf || (U32)data->offset_float_max > (U32)I32_MAX)
706 data->offset_float_max = I32_MAX;
707 if (data->flags & SF_BEFORE_EOL)
709 |= ((data->flags & SF_BEFORE_EOL) << SF_FL_SHIFT_EOL);
711 data->flags &= ~SF_FL_BEFORE_EOL;
712 data->minlen_float=minlenp;
713 data->lookbehind_float=0;
716 SvCUR_set(data->last_found, 0);
718 SV * const sv = data->last_found;
719 if (SvUTF8(sv) && SvMAGICAL(sv)) {
720 MAGIC * const mg = mg_find(sv, PERL_MAGIC_utf8);
726 data->flags &= ~SF_BEFORE_EOL;
727 DEBUG_STUDYDATA("commit: ",data,0);
730 /* Can match anything (initialization) */
732 S_cl_anything(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
734 PERL_ARGS_ASSERT_CL_ANYTHING;
736 ANYOF_BITMAP_SETALL(cl);
737 cl->flags = ANYOF_CLASS|ANYOF_EOS|ANYOF_UNICODE_ALL
738 |ANYOF_LOC_NONBITMAP_FOLD|ANYOF_NON_UTF8_LATIN1_ALL;
740 /* If any portion of the regex is to operate under locale rules,
741 * initialization includes it. The reason this isn't done for all regexes
742 * is that the optimizer was written under the assumption that locale was
743 * all-or-nothing. Given the complexity and lack of documentation in the
744 * optimizer, and that there are inadequate test cases for locale, so many
745 * parts of it may not work properly, it is safest to avoid locale unless
747 if (RExC_contains_locale) {
748 ANYOF_CLASS_SETALL(cl); /* /l uses class */
749 cl->flags |= ANYOF_LOCALE;
752 ANYOF_CLASS_ZERO(cl); /* Only /l uses class now */
756 /* Can match anything (initialization) */
758 S_cl_is_anything(const struct regnode_charclass_class *cl)
762 PERL_ARGS_ASSERT_CL_IS_ANYTHING;
764 for (value = 0; value <= ANYOF_MAX; value += 2)
765 if (ANYOF_CLASS_TEST(cl, value) && ANYOF_CLASS_TEST(cl, value + 1))
767 if (!(cl->flags & ANYOF_UNICODE_ALL))
769 if (!ANYOF_BITMAP_TESTALLSET((const void*)cl))
774 /* Can match anything (initialization) */
776 S_cl_init(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
778 PERL_ARGS_ASSERT_CL_INIT;
780 Zero(cl, 1, struct regnode_charclass_class);
782 cl_anything(pRExC_state, cl);
783 ARG_SET(cl, ANYOF_NONBITMAP_EMPTY);
786 /* These two functions currently do the exact same thing */
787 #define cl_init_zero S_cl_init
789 /* 'AND' a given class with another one. Can create false positives. 'cl'
790 * should not be inverted. 'and_with->flags & ANYOF_CLASS' should be 0 if
791 * 'and_with' is a regnode_charclass instead of a regnode_charclass_class. */
793 S_cl_and(struct regnode_charclass_class *cl,
794 const struct regnode_charclass_class *and_with)
796 PERL_ARGS_ASSERT_CL_AND;
798 assert(and_with->type == ANYOF);
800 /* I (khw) am not sure all these restrictions are necessary XXX */
801 if (!(ANYOF_CLASS_TEST_ANY_SET(and_with))
802 && !(ANYOF_CLASS_TEST_ANY_SET(cl))
803 && (and_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
804 && !(and_with->flags & ANYOF_LOC_NONBITMAP_FOLD)
805 && !(cl->flags & ANYOF_LOC_NONBITMAP_FOLD)) {
808 if (and_with->flags & ANYOF_INVERT)
809 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
810 cl->bitmap[i] &= ~and_with->bitmap[i];
812 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
813 cl->bitmap[i] &= and_with->bitmap[i];
814 } /* XXXX: logic is complicated otherwise, leave it along for a moment. */
816 if (and_with->flags & ANYOF_INVERT) {
818 /* Here, the and'ed node is inverted. Get the AND of the flags that
819 * aren't affected by the inversion. Those that are affected are
820 * handled individually below */
821 U8 affected_flags = cl->flags & ~INVERSION_UNAFFECTED_FLAGS;
822 cl->flags &= (and_with->flags & INVERSION_UNAFFECTED_FLAGS);
823 cl->flags |= affected_flags;
825 /* We currently don't know how to deal with things that aren't in the
826 * bitmap, but we know that the intersection is no greater than what
827 * is already in cl, so let there be false positives that get sorted
828 * out after the synthetic start class succeeds, and the node is
829 * matched for real. */
831 /* The inversion of these two flags indicate that the resulting
832 * intersection doesn't have them */
833 if (and_with->flags & ANYOF_UNICODE_ALL) {
834 cl->flags &= ~ANYOF_UNICODE_ALL;
836 if (and_with->flags & ANYOF_NON_UTF8_LATIN1_ALL) {
837 cl->flags &= ~ANYOF_NON_UTF8_LATIN1_ALL;
840 else { /* and'd node is not inverted */
841 U8 outside_bitmap_but_not_utf8; /* Temp variable */
843 if (! ANYOF_NONBITMAP(and_with)) {
845 /* Here 'and_with' doesn't match anything outside the bitmap
846 * (except possibly ANYOF_UNICODE_ALL), which means the
847 * intersection can't either, except for ANYOF_UNICODE_ALL, in
848 * which case we don't know what the intersection is, but it's no
849 * greater than what cl already has, so can just leave it alone,
850 * with possible false positives */
851 if (! (and_with->flags & ANYOF_UNICODE_ALL)) {
852 ARG_SET(cl, ANYOF_NONBITMAP_EMPTY);
853 cl->flags &= ~ANYOF_NONBITMAP_NON_UTF8;
856 else if (! ANYOF_NONBITMAP(cl)) {
858 /* Here, 'and_with' does match something outside the bitmap, and cl
859 * doesn't have a list of things to match outside the bitmap. If
860 * cl can match all code points above 255, the intersection will
861 * be those above-255 code points that 'and_with' matches. If cl
862 * can't match all Unicode code points, it means that it can't
863 * match anything outside the bitmap (since the 'if' that got us
864 * into this block tested for that), so we leave the bitmap empty.
866 if (cl->flags & ANYOF_UNICODE_ALL) {
867 ARG_SET(cl, ARG(and_with));
869 /* and_with's ARG may match things that don't require UTF8.
870 * And now cl's will too, in spite of this being an 'and'. See
871 * the comments below about the kludge */
872 cl->flags |= and_with->flags & ANYOF_NONBITMAP_NON_UTF8;
876 /* Here, both 'and_with' and cl match something outside the
877 * bitmap. Currently we do not do the intersection, so just match
878 * whatever cl had at the beginning. */
882 /* Take the intersection of the two sets of flags. However, the
883 * ANYOF_NONBITMAP_NON_UTF8 flag is treated as an 'or'. This is a
884 * kludge around the fact that this flag is not treated like the others
885 * which are initialized in cl_anything(). The way the optimizer works
886 * is that the synthetic start class (SSC) is initialized to match
887 * anything, and then the first time a real node is encountered, its
888 * values are AND'd with the SSC's with the result being the values of
889 * the real node. However, there are paths through the optimizer where
890 * the AND never gets called, so those initialized bits are set
891 * inappropriately, which is not usually a big deal, as they just cause
892 * false positives in the SSC, which will just mean a probably
893 * imperceptible slow down in execution. However this bit has a
894 * higher false positive consequence in that it can cause utf8.pm,
895 * utf8_heavy.pl ... to be loaded when not necessary, which is a much
896 * bigger slowdown and also causes significant extra memory to be used.
897 * In order to prevent this, the code now takes a different tack. The
898 * bit isn't set unless some part of the regular expression needs it,
899 * but once set it won't get cleared. This means that these extra
900 * modules won't get loaded unless there was some path through the
901 * pattern that would have required them anyway, and so any false
902 * positives that occur by not ANDing them out when they could be
903 * aren't as severe as they would be if we treated this bit like all
905 outside_bitmap_but_not_utf8 = (cl->flags | and_with->flags)
906 & ANYOF_NONBITMAP_NON_UTF8;
907 cl->flags &= and_with->flags;
908 cl->flags |= outside_bitmap_but_not_utf8;
912 /* 'OR' a given class with another one. Can create false positives. 'cl'
913 * should not be inverted. 'or_with->flags & ANYOF_CLASS' should be 0 if
914 * 'or_with' is a regnode_charclass instead of a regnode_charclass_class. */
916 S_cl_or(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl, const struct regnode_charclass_class *or_with)
918 PERL_ARGS_ASSERT_CL_OR;
920 if (or_with->flags & ANYOF_INVERT) {
922 /* Here, the or'd node is to be inverted. This means we take the
923 * complement of everything not in the bitmap, but currently we don't
924 * know what that is, so give up and match anything */
925 if (ANYOF_NONBITMAP(or_with)) {
926 cl_anything(pRExC_state, cl);
929 * (B1 | CL1) | (!B2 & !CL2) = (B1 | !B2 & !CL2) | (CL1 | (!B2 & !CL2))
930 * <= (B1 | !B2) | (CL1 | !CL2)
931 * which is wasteful if CL2 is small, but we ignore CL2:
932 * (B1 | CL1) | (!B2 & !CL2) <= (B1 | CL1) | !B2 = (B1 | !B2) | CL1
933 * XXXX Can we handle case-fold? Unclear:
934 * (OK1(i) | OK1(i')) | !(OK1(i) | OK1(i')) =
935 * (OK1(i) | OK1(i')) | (!OK1(i) & !OK1(i'))
937 else if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
938 && !(or_with->flags & ANYOF_LOC_NONBITMAP_FOLD)
939 && !(cl->flags & ANYOF_LOC_NONBITMAP_FOLD) ) {
942 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
943 cl->bitmap[i] |= ~or_with->bitmap[i];
944 } /* XXXX: logic is complicated otherwise */
946 cl_anything(pRExC_state, cl);
949 /* And, we can just take the union of the flags that aren't affected
950 * by the inversion */
951 cl->flags |= or_with->flags & INVERSION_UNAFFECTED_FLAGS;
953 /* For the remaining flags:
954 ANYOF_UNICODE_ALL and inverted means to not match anything above
955 255, which means that the union with cl should just be
956 what cl has in it, so can ignore this flag
957 ANYOF_NON_UTF8_LATIN1_ALL and inverted means if not utf8 and ord
958 is 127-255 to match them, but then invert that, so the
959 union with cl should just be what cl has in it, so can
962 } else { /* 'or_with' is not inverted */
963 /* (B1 | CL1) | (B2 | CL2) = (B1 | B2) | (CL1 | CL2)) */
964 if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
965 && (!(or_with->flags & ANYOF_LOC_NONBITMAP_FOLD)
966 || (cl->flags & ANYOF_LOC_NONBITMAP_FOLD)) ) {
969 /* OR char bitmap and class bitmap separately */
970 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
971 cl->bitmap[i] |= or_with->bitmap[i];
972 if (ANYOF_CLASS_TEST_ANY_SET(or_with)) {
973 for (i = 0; i < ANYOF_CLASSBITMAP_SIZE; i++)
974 cl->classflags[i] |= or_with->classflags[i];
975 cl->flags |= ANYOF_CLASS;
978 else { /* XXXX: logic is complicated, leave it along for a moment. */
979 cl_anything(pRExC_state, cl);
982 if (ANYOF_NONBITMAP(or_with)) {
984 /* Use the added node's outside-the-bit-map match if there isn't a
985 * conflict. If there is a conflict (both nodes match something
986 * outside the bitmap, but what they match outside is not the same
987 * pointer, and hence not easily compared until XXX we extend
988 * inversion lists this far), give up and allow the start class to
989 * match everything outside the bitmap. If that stuff is all above
990 * 255, can just set UNICODE_ALL, otherwise caould be anything. */
991 if (! ANYOF_NONBITMAP(cl)) {
992 ARG_SET(cl, ARG(or_with));
994 else if (ARG(cl) != ARG(or_with)) {
996 if ((or_with->flags & ANYOF_NONBITMAP_NON_UTF8)) {
997 cl_anything(pRExC_state, cl);
1000 cl->flags |= ANYOF_UNICODE_ALL;
1005 /* Take the union */
1006 cl->flags |= or_with->flags;
1010 #define TRIE_LIST_ITEM(state,idx) (trie->states[state].trans.list)[ idx ]
1011 #define TRIE_LIST_CUR(state) ( TRIE_LIST_ITEM( state, 0 ).forid )
1012 #define TRIE_LIST_LEN(state) ( TRIE_LIST_ITEM( state, 0 ).newstate )
1013 #define TRIE_LIST_USED(idx) ( trie->states[state].trans.list ? (TRIE_LIST_CUR( idx ) - 1) : 0 )
1018 dump_trie(trie,widecharmap,revcharmap)
1019 dump_trie_interim_list(trie,widecharmap,revcharmap,next_alloc)
1020 dump_trie_interim_table(trie,widecharmap,revcharmap,next_alloc)
1022 These routines dump out a trie in a somewhat readable format.
1023 The _interim_ variants are used for debugging the interim
1024 tables that are used to generate the final compressed
1025 representation which is what dump_trie expects.
1027 Part of the reason for their existence is to provide a form
1028 of documentation as to how the different representations function.
1033 Dumps the final compressed table form of the trie to Perl_debug_log.
1034 Used for debugging make_trie().
1038 S_dump_trie(pTHX_ const struct _reg_trie_data *trie, HV *widecharmap,
1039 AV *revcharmap, U32 depth)
1042 SV *sv=sv_newmortal();
1043 int colwidth= widecharmap ? 6 : 4;
1045 GET_RE_DEBUG_FLAGS_DECL;
1047 PERL_ARGS_ASSERT_DUMP_TRIE;
1049 PerlIO_printf( Perl_debug_log, "%*sChar : %-6s%-6s%-4s ",
1050 (int)depth * 2 + 2,"",
1051 "Match","Base","Ofs" );
1053 for( state = 0 ; state < trie->uniquecharcount ; state++ ) {
1054 SV ** const tmp = av_fetch( revcharmap, state, 0);
1056 PerlIO_printf( Perl_debug_log, "%*s",
1058 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
1059 PL_colors[0], PL_colors[1],
1060 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
1061 PERL_PV_ESCAPE_FIRSTCHAR
1066 PerlIO_printf( Perl_debug_log, "\n%*sState|-----------------------",
1067 (int)depth * 2 + 2,"");
1069 for( state = 0 ; state < trie->uniquecharcount ; state++ )
1070 PerlIO_printf( Perl_debug_log, "%.*s", colwidth, "--------");
1071 PerlIO_printf( Perl_debug_log, "\n");
1073 for( state = 1 ; state < trie->statecount ; state++ ) {
1074 const U32 base = trie->states[ state ].trans.base;
1076 PerlIO_printf( Perl_debug_log, "%*s#%4"UVXf"|", (int)depth * 2 + 2,"", (UV)state);
1078 if ( trie->states[ state ].wordnum ) {
1079 PerlIO_printf( Perl_debug_log, " W%4X", trie->states[ state ].wordnum );
1081 PerlIO_printf( Perl_debug_log, "%6s", "" );
1084 PerlIO_printf( Perl_debug_log, " @%4"UVXf" ", (UV)base );
1089 while( ( base + ofs < trie->uniquecharcount ) ||
1090 ( base + ofs - trie->uniquecharcount < trie->lasttrans
1091 && trie->trans[ base + ofs - trie->uniquecharcount ].check != state))
1094 PerlIO_printf( Perl_debug_log, "+%2"UVXf"[ ", (UV)ofs);
1096 for ( ofs = 0 ; ofs < trie->uniquecharcount ; ofs++ ) {
1097 if ( ( base + ofs >= trie->uniquecharcount ) &&
1098 ( base + ofs - trie->uniquecharcount < trie->lasttrans ) &&
1099 trie->trans[ base + ofs - trie->uniquecharcount ].check == state )
1101 PerlIO_printf( Perl_debug_log, "%*"UVXf,
1103 (UV)trie->trans[ base + ofs - trie->uniquecharcount ].next );
1105 PerlIO_printf( Perl_debug_log, "%*s",colwidth," ." );
1109 PerlIO_printf( Perl_debug_log, "]");
1112 PerlIO_printf( Perl_debug_log, "\n" );
1114 PerlIO_printf(Perl_debug_log, "%*sword_info N:(prev,len)=", (int)depth*2, "");
1115 for (word=1; word <= trie->wordcount; word++) {
1116 PerlIO_printf(Perl_debug_log, " %d:(%d,%d)",
1117 (int)word, (int)(trie->wordinfo[word].prev),
1118 (int)(trie->wordinfo[word].len));
1120 PerlIO_printf(Perl_debug_log, "\n" );
1123 Dumps a fully constructed but uncompressed trie in list form.
1124 List tries normally only are used for construction when the number of
1125 possible chars (trie->uniquecharcount) is very high.
1126 Used for debugging make_trie().
1129 S_dump_trie_interim_list(pTHX_ const struct _reg_trie_data *trie,
1130 HV *widecharmap, AV *revcharmap, U32 next_alloc,
1134 SV *sv=sv_newmortal();
1135 int colwidth= widecharmap ? 6 : 4;
1136 GET_RE_DEBUG_FLAGS_DECL;
1138 PERL_ARGS_ASSERT_DUMP_TRIE_INTERIM_LIST;
1140 /* print out the table precompression. */
1141 PerlIO_printf( Perl_debug_log, "%*sState :Word | Transition Data\n%*s%s",
1142 (int)depth * 2 + 2,"", (int)depth * 2 + 2,"",
1143 "------:-----+-----------------\n" );
1145 for( state=1 ; state < next_alloc ; state ++ ) {
1148 PerlIO_printf( Perl_debug_log, "%*s %4"UVXf" :",
1149 (int)depth * 2 + 2,"", (UV)state );
1150 if ( ! trie->states[ state ].wordnum ) {
1151 PerlIO_printf( Perl_debug_log, "%5s| ","");
1153 PerlIO_printf( Perl_debug_log, "W%4x| ",
1154 trie->states[ state ].wordnum
1157 for( charid = 1 ; charid <= TRIE_LIST_USED( state ) ; charid++ ) {
1158 SV ** const tmp = av_fetch( revcharmap, TRIE_LIST_ITEM(state,charid).forid, 0);
1160 PerlIO_printf( Perl_debug_log, "%*s:%3X=%4"UVXf" | ",
1162 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
1163 PL_colors[0], PL_colors[1],
1164 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
1165 PERL_PV_ESCAPE_FIRSTCHAR
1167 TRIE_LIST_ITEM(state,charid).forid,
1168 (UV)TRIE_LIST_ITEM(state,charid).newstate
1171 PerlIO_printf(Perl_debug_log, "\n%*s| ",
1172 (int)((depth * 2) + 14), "");
1175 PerlIO_printf( Perl_debug_log, "\n");
1180 Dumps a fully constructed but uncompressed trie in table form.
1181 This is the normal DFA style state transition table, with a few
1182 twists to facilitate compression later.
1183 Used for debugging make_trie().
1186 S_dump_trie_interim_table(pTHX_ const struct _reg_trie_data *trie,
1187 HV *widecharmap, AV *revcharmap, U32 next_alloc,
1192 SV *sv=sv_newmortal();
1193 int colwidth= widecharmap ? 6 : 4;
1194 GET_RE_DEBUG_FLAGS_DECL;
1196 PERL_ARGS_ASSERT_DUMP_TRIE_INTERIM_TABLE;
1199 print out the table precompression so that we can do a visual check
1200 that they are identical.
1203 PerlIO_printf( Perl_debug_log, "%*sChar : ",(int)depth * 2 + 2,"" );
1205 for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) {
1206 SV ** const tmp = av_fetch( revcharmap, charid, 0);
1208 PerlIO_printf( Perl_debug_log, "%*s",
1210 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
1211 PL_colors[0], PL_colors[1],
1212 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
1213 PERL_PV_ESCAPE_FIRSTCHAR
1219 PerlIO_printf( Perl_debug_log, "\n%*sState+-",(int)depth * 2 + 2,"" );
1221 for( charid=0 ; charid < trie->uniquecharcount ; charid++ ) {
1222 PerlIO_printf( Perl_debug_log, "%.*s", colwidth,"--------");
1225 PerlIO_printf( Perl_debug_log, "\n" );
1227 for( state=1 ; state < next_alloc ; state += trie->uniquecharcount ) {
1229 PerlIO_printf( Perl_debug_log, "%*s%4"UVXf" : ",
1230 (int)depth * 2 + 2,"",
1231 (UV)TRIE_NODENUM( state ) );
1233 for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) {
1234 UV v=(UV)SAFE_TRIE_NODENUM( trie->trans[ state + charid ].next );
1236 PerlIO_printf( Perl_debug_log, "%*"UVXf, colwidth, v );
1238 PerlIO_printf( Perl_debug_log, "%*s", colwidth, "." );
1240 if ( ! trie->states[ TRIE_NODENUM( state ) ].wordnum ) {
1241 PerlIO_printf( Perl_debug_log, " (%4"UVXf")\n", (UV)trie->trans[ state ].check );
1243 PerlIO_printf( Perl_debug_log, " (%4"UVXf") W%4X\n", (UV)trie->trans[ state ].check,
1244 trie->states[ TRIE_NODENUM( state ) ].wordnum );
1252 /* make_trie(startbranch,first,last,tail,word_count,flags,depth)
1253 startbranch: the first branch in the whole branch sequence
1254 first : start branch of sequence of branch-exact nodes.
1255 May be the same as startbranch
1256 last : Thing following the last branch.
1257 May be the same as tail.
1258 tail : item following the branch sequence
1259 count : words in the sequence
1260 flags : currently the OP() type we will be building one of /EXACT(|F|Fl)/
1261 depth : indent depth
1263 Inplace optimizes a sequence of 2 or more Branch-Exact nodes into a TRIE node.
1265 A trie is an N'ary tree where the branches are determined by digital
1266 decomposition of the key. IE, at the root node you look up the 1st character and
1267 follow that branch repeat until you find the end of the branches. Nodes can be
1268 marked as "accepting" meaning they represent a complete word. Eg:
1272 would convert into the following structure. Numbers represent states, letters
1273 following numbers represent valid transitions on the letter from that state, if
1274 the number is in square brackets it represents an accepting state, otherwise it
1275 will be in parenthesis.
1277 +-h->+-e->[3]-+-r->(8)-+-s->[9]
1281 (1) +-i->(6)-+-s->[7]
1283 +-s->(3)-+-h->(4)-+-e->[5]
1285 Accept Word Mapping: 3=>1 (he),5=>2 (she), 7=>3 (his), 9=>4 (hers)
1287 This shows that when matching against the string 'hers' we will begin at state 1
1288 read 'h' and move to state 2, read 'e' and move to state 3 which is accepting,
1289 then read 'r' and go to state 8 followed by 's' which takes us to state 9 which
1290 is also accepting. Thus we know that we can match both 'he' and 'hers' with a
1291 single traverse. We store a mapping from accepting to state to which word was
1292 matched, and then when we have multiple possibilities we try to complete the
1293 rest of the regex in the order in which they occured in the alternation.
1295 The only prior NFA like behaviour that would be changed by the TRIE support is
1296 the silent ignoring of duplicate alternations which are of the form:
1298 / (DUPE|DUPE) X? (?{ ... }) Y /x
1300 Thus EVAL blocks following a trie may be called a different number of times with
1301 and without the optimisation. With the optimisations dupes will be silently
1302 ignored. This inconsistent behaviour of EVAL type nodes is well established as
1303 the following demonstrates:
1305 'words'=~/(word|word|word)(?{ print $1 })[xyz]/
1307 which prints out 'word' three times, but
1309 'words'=~/(word|word|word)(?{ print $1 })S/
1311 which doesnt print it out at all. This is due to other optimisations kicking in.
1313 Example of what happens on a structural level:
1315 The regexp /(ac|ad|ab)+/ will produce the following debug output:
1317 1: CURLYM[1] {1,32767}(18)
1328 This would be optimizable with startbranch=5, first=5, last=16, tail=16
1329 and should turn into:
1331 1: CURLYM[1] {1,32767}(18)
1333 [Words:3 Chars Stored:6 Unique Chars:4 States:5 NCP:1]
1341 Cases where tail != last would be like /(?foo|bar)baz/:
1351 which would be optimizable with startbranch=1, first=1, last=7, tail=8
1352 and would end up looking like:
1355 [Words:2 Chars Stored:6 Unique Chars:5 States:7 NCP:1]
1362 d = uvuni_to_utf8_flags(d, uv, 0);
1364 is the recommended Unicode-aware way of saying
1369 #define TRIE_STORE_REVCHAR(val) \
1372 SV *zlopp = newSV(7); /* XXX: optimize me */ \
1373 unsigned char *flrbbbbb = (unsigned char *) SvPVX(zlopp); \
1374 unsigned const char *const kapow = uvuni_to_utf8(flrbbbbb, val); \
1375 SvCUR_set(zlopp, kapow - flrbbbbb); \
1378 av_push(revcharmap, zlopp); \
1380 char ooooff = (char)val; \
1381 av_push(revcharmap, newSVpvn(&ooooff, 1)); \
1385 #define TRIE_READ_CHAR STMT_START { \
1388 /* if it is UTF then it is either already folded, or does not need folding */ \
1389 uvc = utf8n_to_uvuni( (const U8*) uc, UTF8_MAXLEN, &len, uniflags); \
1391 else if (folder == PL_fold_latin1) { \
1392 /* if we use this folder we have to obey unicode rules on latin-1 data */ \
1393 if ( foldlen > 0 ) { \
1394 uvc = utf8n_to_uvuni( (const U8*) scan, UTF8_MAXLEN, &len, uniflags ); \
1400 uvc = _to_fold_latin1( (U8) *uc, foldbuf, &foldlen, 1); \
1401 skiplen = UNISKIP(uvc); \
1402 foldlen -= skiplen; \
1403 scan = foldbuf + skiplen; \
1406 /* raw data, will be folded later if needed */ \
1414 #define TRIE_LIST_PUSH(state,fid,ns) STMT_START { \
1415 if ( TRIE_LIST_CUR( state ) >=TRIE_LIST_LEN( state ) ) { \
1416 U32 ging = TRIE_LIST_LEN( state ) *= 2; \
1417 Renew( trie->states[ state ].trans.list, ging, reg_trie_trans_le ); \
1419 TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).forid = fid; \
1420 TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).newstate = ns; \
1421 TRIE_LIST_CUR( state )++; \
1424 #define TRIE_LIST_NEW(state) STMT_START { \
1425 Newxz( trie->states[ state ].trans.list, \
1426 4, reg_trie_trans_le ); \
1427 TRIE_LIST_CUR( state ) = 1; \
1428 TRIE_LIST_LEN( state ) = 4; \
1431 #define TRIE_HANDLE_WORD(state) STMT_START { \
1432 U16 dupe= trie->states[ state ].wordnum; \
1433 regnode * const noper_next = regnext( noper ); \
1436 /* store the word for dumping */ \
1438 if (OP(noper) != NOTHING) \
1439 tmp = newSVpvn_utf8(STRING(noper), STR_LEN(noper), UTF); \
1441 tmp = newSVpvn_utf8( "", 0, UTF ); \
1442 av_push( trie_words, tmp ); \
1446 trie->wordinfo[curword].prev = 0; \
1447 trie->wordinfo[curword].len = wordlen; \
1448 trie->wordinfo[curword].accept = state; \
1450 if ( noper_next < tail ) { \
1452 trie->jump = (U16 *) PerlMemShared_calloc( word_count + 1, sizeof(U16) ); \
1453 trie->jump[curword] = (U16)(noper_next - convert); \
1455 jumper = noper_next; \
1457 nextbranch= regnext(cur); \
1461 /* It's a dupe. Pre-insert into the wordinfo[].prev */\
1462 /* chain, so that when the bits of chain are later */\
1463 /* linked together, the dups appear in the chain */\
1464 trie->wordinfo[curword].prev = trie->wordinfo[dupe].prev; \
1465 trie->wordinfo[dupe].prev = curword; \
1467 /* we haven't inserted this word yet. */ \
1468 trie->states[ state ].wordnum = curword; \
1473 #define TRIE_TRANS_STATE(state,base,ucharcount,charid,special) \
1474 ( ( base + charid >= ucharcount \
1475 && base + charid < ubound \
1476 && state == trie->trans[ base - ucharcount + charid ].check \
1477 && trie->trans[ base - ucharcount + charid ].next ) \
1478 ? trie->trans[ base - ucharcount + charid ].next \
1479 : ( state==1 ? special : 0 ) \
1483 #define MADE_JUMP_TRIE 2
1484 #define MADE_EXACT_TRIE 4
1487 S_make_trie(pTHX_ RExC_state_t *pRExC_state, regnode *startbranch, regnode *first, regnode *last, regnode *tail, U32 word_count, U32 flags, U32 depth)
1490 /* first pass, loop through and scan words */
1491 reg_trie_data *trie;
1492 HV *widecharmap = NULL;
1493 AV *revcharmap = newAV();
1495 const U32 uniflags = UTF8_ALLOW_DEFAULT;
1500 regnode *jumper = NULL;
1501 regnode *nextbranch = NULL;
1502 regnode *convert = NULL;
1503 U32 *prev_states; /* temp array mapping each state to previous one */
1504 /* we just use folder as a flag in utf8 */
1505 const U8 * folder = NULL;
1508 const U32 data_slot = add_data( pRExC_state, 4, "tuuu" );
1509 AV *trie_words = NULL;
1510 /* along with revcharmap, this only used during construction but both are
1511 * useful during debugging so we store them in the struct when debugging.
1514 const U32 data_slot = add_data( pRExC_state, 2, "tu" );
1515 STRLEN trie_charcount=0;
1517 SV *re_trie_maxbuff;
1518 GET_RE_DEBUG_FLAGS_DECL;
1520 PERL_ARGS_ASSERT_MAKE_TRIE;
1522 PERL_UNUSED_ARG(depth);
1529 case EXACTFU_TRICKYFOLD:
1530 case EXACTFU: folder = PL_fold_latin1; break;
1531 case EXACTF: folder = PL_fold; break;
1532 case EXACTFL: folder = PL_fold_locale; break;
1533 default: Perl_croak( aTHX_ "panic! In trie construction, unknown node type %u %s", (unsigned) flags, PL_reg_name[flags] );
1536 trie = (reg_trie_data *) PerlMemShared_calloc( 1, sizeof(reg_trie_data) );
1538 trie->startstate = 1;
1539 trie->wordcount = word_count;
1540 RExC_rxi->data->data[ data_slot ] = (void*)trie;
1541 trie->charmap = (U16 *) PerlMemShared_calloc( 256, sizeof(U16) );
1543 trie->bitmap = (char *) PerlMemShared_calloc( ANYOF_BITMAP_SIZE, 1 );
1544 trie->wordinfo = (reg_trie_wordinfo *) PerlMemShared_calloc(
1545 trie->wordcount+1, sizeof(reg_trie_wordinfo));
1548 trie_words = newAV();
1551 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1);
1552 if (!SvIOK(re_trie_maxbuff)) {
1553 sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT);
1556 PerlIO_printf( Perl_debug_log,
1557 "%*smake_trie start==%d, first==%d, last==%d, tail==%d depth=%d\n",
1558 (int)depth * 2 + 2, "",
1559 REG_NODE_NUM(startbranch),REG_NODE_NUM(first),
1560 REG_NODE_NUM(last), REG_NODE_NUM(tail),
1564 /* Find the node we are going to overwrite */
1565 if ( first == startbranch && OP( last ) != BRANCH ) {
1566 /* whole branch chain */
1569 /* branch sub-chain */
1570 convert = NEXTOPER( first );
1573 /* -- First loop and Setup --
1575 We first traverse the branches and scan each word to determine if it
1576 contains widechars, and how many unique chars there are, this is
1577 important as we have to build a table with at least as many columns as we
1580 We use an array of integers to represent the character codes 0..255
1581 (trie->charmap) and we use a an HV* to store Unicode characters. We use the
1582 native representation of the character value as the key and IV's for the
1585 *TODO* If we keep track of how many times each character is used we can
1586 remap the columns so that the table compression later on is more
1587 efficient in terms of memory by ensuring the most common value is in the
1588 middle and the least common are on the outside. IMO this would be better
1589 than a most to least common mapping as theres a decent chance the most
1590 common letter will share a node with the least common, meaning the node
1591 will not be compressible. With a middle is most common approach the worst
1592 case is when we have the least common nodes twice.
1596 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
1597 regnode * const noper = NEXTOPER( cur );
1598 const U8 *uc = (U8*)STRING( noper );
1599 const U8 * const e = uc + STR_LEN( noper );
1601 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
1603 const U8 *scan = (U8*)NULL;
1604 U32 wordlen = 0; /* required init */
1606 bool set_bit = trie->bitmap ? 1 : 0; /*store the first char in the bitmap?*/
1608 if (OP(noper) == NOTHING) {
1612 if ( set_bit ) { /* bitmap only alloced when !(UTF&&Folding) */
1613 TRIE_BITMAP_SET(trie,*uc); /* store the raw first byte
1614 regardless of encoding */
1615 if (OP( noper ) == EXACTFU_SS) {
1616 /* false positives are ok, so just set this */
1617 TRIE_BITMAP_SET(trie,0xDF);
1620 for ( ; uc < e ; uc += len ) {
1621 TRIE_CHARCOUNT(trie)++;
1626 U8 folded= folder[ (U8) uvc ];
1627 if ( !trie->charmap[ folded ] ) {
1628 trie->charmap[ folded ]=( ++trie->uniquecharcount );
1629 TRIE_STORE_REVCHAR( folded );
1632 if ( !trie->charmap[ uvc ] ) {
1633 trie->charmap[ uvc ]=( ++trie->uniquecharcount );
1634 TRIE_STORE_REVCHAR( uvc );
1637 /* store the codepoint in the bitmap, and its folded
1639 TRIE_BITMAP_SET(trie, uvc);
1641 /* store the folded codepoint */
1642 if ( folder ) TRIE_BITMAP_SET(trie, folder[(U8) uvc ]);
1645 /* store first byte of utf8 representation of
1646 variant codepoints */
1647 if (! UNI_IS_INVARIANT(uvc)) {
1648 TRIE_BITMAP_SET(trie, UTF8_TWO_BYTE_HI(uvc));
1651 set_bit = 0; /* We've done our bit :-) */
1656 widecharmap = newHV();
1658 svpp = hv_fetch( widecharmap, (char*)&uvc, sizeof( UV ), 1 );
1661 Perl_croak( aTHX_ "error creating/fetching widecharmap entry for 0x%"UVXf, uvc );
1663 if ( !SvTRUE( *svpp ) ) {
1664 sv_setiv( *svpp, ++trie->uniquecharcount );
1665 TRIE_STORE_REVCHAR(uvc);
1669 if( cur == first ) {
1670 trie->minlen = chars;
1671 trie->maxlen = chars;
1672 } else if (chars < trie->minlen) {
1673 trie->minlen = chars;
1674 } else if (chars > trie->maxlen) {
1675 trie->maxlen = chars;
1677 if (OP( noper ) == EXACTFU_SS) {
1678 /* XXX: workaround - 'ss' could match "\x{DF}" so minlen could be 1 and not 2*/
1679 if (trie->minlen > 1)
1682 if (OP( noper ) == EXACTFU_TRICKYFOLD) {
1683 /* XXX: workround - things like "\x{1FBE}\x{0308}\x{0301}" can match "\x{0390}"
1684 * - We assume that any such sequence might match a 2 byte string */
1685 if (trie->minlen > 2 )
1689 } /* end first pass */
1690 DEBUG_TRIE_COMPILE_r(
1691 PerlIO_printf( Perl_debug_log, "%*sTRIE(%s): W:%d C:%d Uq:%d Min:%d Max:%d\n",
1692 (int)depth * 2 + 2,"",
1693 ( widecharmap ? "UTF8" : "NATIVE" ), (int)word_count,
1694 (int)TRIE_CHARCOUNT(trie), trie->uniquecharcount,
1695 (int)trie->minlen, (int)trie->maxlen )
1699 We now know what we are dealing with in terms of unique chars and
1700 string sizes so we can calculate how much memory a naive
1701 representation using a flat table will take. If it's over a reasonable
1702 limit (as specified by ${^RE_TRIE_MAXBUF}) we use a more memory
1703 conservative but potentially much slower representation using an array
1706 At the end we convert both representations into the same compressed
1707 form that will be used in regexec.c for matching with. The latter
1708 is a form that cannot be used to construct with but has memory
1709 properties similar to the list form and access properties similar
1710 to the table form making it both suitable for fast searches and
1711 small enough that its feasable to store for the duration of a program.
1713 See the comment in the code where the compressed table is produced
1714 inplace from the flat tabe representation for an explanation of how
1715 the compression works.
1720 Newx(prev_states, TRIE_CHARCOUNT(trie) + 2, U32);
1723 if ( (IV)( ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1) > SvIV(re_trie_maxbuff) ) {
1725 Second Pass -- Array Of Lists Representation
1727 Each state will be represented by a list of charid:state records
1728 (reg_trie_trans_le) the first such element holds the CUR and LEN
1729 points of the allocated array. (See defines above).
1731 We build the initial structure using the lists, and then convert
1732 it into the compressed table form which allows faster lookups
1733 (but cant be modified once converted).
1736 STRLEN transcount = 1;
1738 DEBUG_TRIE_COMPILE_MORE_r( PerlIO_printf( Perl_debug_log,
1739 "%*sCompiling trie using list compiler\n",
1740 (int)depth * 2 + 2, ""));
1742 trie->states = (reg_trie_state *)
1743 PerlMemShared_calloc( TRIE_CHARCOUNT(trie) + 2,
1744 sizeof(reg_trie_state) );
1748 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
1750 regnode * const noper = NEXTOPER( cur );
1751 U8 *uc = (U8*)STRING( noper );
1752 const U8 * const e = uc + STR_LEN( noper );
1753 U32 state = 1; /* required init */
1754 U16 charid = 0; /* sanity init */
1755 U8 *scan = (U8*)NULL; /* sanity init */
1756 STRLEN foldlen = 0; /* required init */
1757 U32 wordlen = 0; /* required init */
1758 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
1761 if (OP(noper) != NOTHING) {
1762 for ( ; uc < e ; uc += len ) {
1767 charid = trie->charmap[ uvc ];
1769 SV** const svpp = hv_fetch( widecharmap, (char*)&uvc, sizeof( UV ), 0);
1773 charid=(U16)SvIV( *svpp );
1776 /* charid is now 0 if we dont know the char read, or nonzero if we do */
1783 if ( !trie->states[ state ].trans.list ) {
1784 TRIE_LIST_NEW( state );
1786 for ( check = 1; check <= TRIE_LIST_USED( state ); check++ ) {
1787 if ( TRIE_LIST_ITEM( state, check ).forid == charid ) {
1788 newstate = TRIE_LIST_ITEM( state, check ).newstate;
1793 newstate = next_alloc++;
1794 prev_states[newstate] = state;
1795 TRIE_LIST_PUSH( state, charid, newstate );
1800 Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc );
1804 TRIE_HANDLE_WORD(state);
1806 } /* end second pass */
1808 /* next alloc is the NEXT state to be allocated */
1809 trie->statecount = next_alloc;
1810 trie->states = (reg_trie_state *)
1811 PerlMemShared_realloc( trie->states,
1813 * sizeof(reg_trie_state) );
1815 /* and now dump it out before we compress it */
1816 DEBUG_TRIE_COMPILE_MORE_r(dump_trie_interim_list(trie, widecharmap,
1817 revcharmap, next_alloc,
1821 trie->trans = (reg_trie_trans *)
1822 PerlMemShared_calloc( transcount, sizeof(reg_trie_trans) );
1829 for( state=1 ; state < next_alloc ; state ++ ) {
1833 DEBUG_TRIE_COMPILE_MORE_r(
1834 PerlIO_printf( Perl_debug_log, "tp: %d zp: %d ",tp,zp)
1838 if (trie->states[state].trans.list) {
1839 U16 minid=TRIE_LIST_ITEM( state, 1).forid;
1843 for( idx = 2 ; idx <= TRIE_LIST_USED( state ) ; idx++ ) {
1844 const U16 forid = TRIE_LIST_ITEM( state, idx).forid;
1845 if ( forid < minid ) {
1847 } else if ( forid > maxid ) {
1851 if ( transcount < tp + maxid - minid + 1) {
1853 trie->trans = (reg_trie_trans *)
1854 PerlMemShared_realloc( trie->trans,
1856 * sizeof(reg_trie_trans) );
1857 Zero( trie->trans + (transcount / 2), transcount / 2 , reg_trie_trans );
1859 base = trie->uniquecharcount + tp - minid;
1860 if ( maxid == minid ) {
1862 for ( ; zp < tp ; zp++ ) {
1863 if ( ! trie->trans[ zp ].next ) {
1864 base = trie->uniquecharcount + zp - minid;
1865 trie->trans[ zp ].next = TRIE_LIST_ITEM( state, 1).newstate;
1866 trie->trans[ zp ].check = state;
1872 trie->trans[ tp ].next = TRIE_LIST_ITEM( state, 1).newstate;
1873 trie->trans[ tp ].check = state;
1878 for ( idx=1; idx <= TRIE_LIST_USED( state ) ; idx++ ) {
1879 const U32 tid = base - trie->uniquecharcount + TRIE_LIST_ITEM( state, idx ).forid;
1880 trie->trans[ tid ].next = TRIE_LIST_ITEM( state, idx ).newstate;
1881 trie->trans[ tid ].check = state;
1883 tp += ( maxid - minid + 1 );
1885 Safefree(trie->states[ state ].trans.list);
1888 DEBUG_TRIE_COMPILE_MORE_r(
1889 PerlIO_printf( Perl_debug_log, " base: %d\n",base);
1892 trie->states[ state ].trans.base=base;
1894 trie->lasttrans = tp + 1;
1898 Second Pass -- Flat Table Representation.
1900 we dont use the 0 slot of either trans[] or states[] so we add 1 to each.
1901 We know that we will need Charcount+1 trans at most to store the data
1902 (one row per char at worst case) So we preallocate both structures
1903 assuming worst case.
1905 We then construct the trie using only the .next slots of the entry
1908 We use the .check field of the first entry of the node temporarily to
1909 make compression both faster and easier by keeping track of how many non
1910 zero fields are in the node.
1912 Since trans are numbered from 1 any 0 pointer in the table is a FAIL
1915 There are two terms at use here: state as a TRIE_NODEIDX() which is a
1916 number representing the first entry of the node, and state as a
1917 TRIE_NODENUM() which is the trans number. state 1 is TRIE_NODEIDX(1) and
1918 TRIE_NODENUM(1), state 2 is TRIE_NODEIDX(2) and TRIE_NODENUM(3) if there
1919 are 2 entrys per node. eg:
1927 The table is internally in the right hand, idx form. However as we also
1928 have to deal with the states array which is indexed by nodenum we have to
1929 use TRIE_NODENUM() to convert.
1932 DEBUG_TRIE_COMPILE_MORE_r( PerlIO_printf( Perl_debug_log,
1933 "%*sCompiling trie using table compiler\n",
1934 (int)depth * 2 + 2, ""));
1936 trie->trans = (reg_trie_trans *)
1937 PerlMemShared_calloc( ( TRIE_CHARCOUNT(trie) + 1 )
1938 * trie->uniquecharcount + 1,
1939 sizeof(reg_trie_trans) );
1940 trie->states = (reg_trie_state *)
1941 PerlMemShared_calloc( TRIE_CHARCOUNT(trie) + 2,
1942 sizeof(reg_trie_state) );
1943 next_alloc = trie->uniquecharcount + 1;
1946 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
1948 regnode * const noper = NEXTOPER( cur );
1949 const U8 *uc = (U8*)STRING( noper );
1950 const U8 * const e = uc + STR_LEN( noper );
1952 U32 state = 1; /* required init */
1954 U16 charid = 0; /* sanity init */
1955 U32 accept_state = 0; /* sanity init */
1956 U8 *scan = (U8*)NULL; /* sanity init */
1958 STRLEN foldlen = 0; /* required init */
1959 U32 wordlen = 0; /* required init */
1961 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
1964 if ( OP(noper) != NOTHING ) {
1965 for ( ; uc < e ; uc += len ) {
1970 charid = trie->charmap[ uvc ];
1972 SV* const * const svpp = hv_fetch( widecharmap, (char*)&uvc, sizeof( UV ), 0);
1973 charid = svpp ? (U16)SvIV(*svpp) : 0;
1977 if ( !trie->trans[ state + charid ].next ) {
1978 trie->trans[ state + charid ].next = next_alloc;
1979 trie->trans[ state ].check++;
1980 prev_states[TRIE_NODENUM(next_alloc)]
1981 = TRIE_NODENUM(state);
1982 next_alloc += trie->uniquecharcount;
1984 state = trie->trans[ state + charid ].next;
1986 Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc );
1988 /* charid is now 0 if we dont know the char read, or nonzero if we do */
1991 accept_state = TRIE_NODENUM( state );
1992 TRIE_HANDLE_WORD(accept_state);
1994 } /* end second pass */
1996 /* and now dump it out before we compress it */
1997 DEBUG_TRIE_COMPILE_MORE_r(dump_trie_interim_table(trie, widecharmap,
1999 next_alloc, depth+1));
2003 * Inplace compress the table.*
2005 For sparse data sets the table constructed by the trie algorithm will
2006 be mostly 0/FAIL transitions or to put it another way mostly empty.
2007 (Note that leaf nodes will not contain any transitions.)
2009 This algorithm compresses the tables by eliminating most such
2010 transitions, at the cost of a modest bit of extra work during lookup:
2012 - Each states[] entry contains a .base field which indicates the
2013 index in the state[] array wheres its transition data is stored.
2015 - If .base is 0 there are no valid transitions from that node.
2017 - If .base is nonzero then charid is added to it to find an entry in
2020 -If trans[states[state].base+charid].check!=state then the
2021 transition is taken to be a 0/Fail transition. Thus if there are fail
2022 transitions at the front of the node then the .base offset will point
2023 somewhere inside the previous nodes data (or maybe even into a node
2024 even earlier), but the .check field determines if the transition is
2028 The following process inplace converts the table to the compressed
2029 table: We first do not compress the root node 1,and mark all its
2030 .check pointers as 1 and set its .base pointer as 1 as well. This
2031 allows us to do a DFA construction from the compressed table later,
2032 and ensures that any .base pointers we calculate later are greater
2035 - We set 'pos' to indicate the first entry of the second node.
2037 - We then iterate over the columns of the node, finding the first and
2038 last used entry at l and m. We then copy l..m into pos..(pos+m-l),
2039 and set the .check pointers accordingly, and advance pos
2040 appropriately and repreat for the next node. Note that when we copy
2041 the next pointers we have to convert them from the original
2042 NODEIDX form to NODENUM form as the former is not valid post
2045 - If a node has no transitions used we mark its base as 0 and do not
2046 advance the pos pointer.
2048 - If a node only has one transition we use a second pointer into the
2049 structure to fill in allocated fail transitions from other states.
2050 This pointer is independent of the main pointer and scans forward
2051 looking for null transitions that are allocated to a state. When it
2052 finds one it writes the single transition into the "hole". If the
2053 pointer doesnt find one the single transition is appended as normal.
2055 - Once compressed we can Renew/realloc the structures to release the
2058 See "Table-Compression Methods" in sec 3.9 of the Red Dragon,
2059 specifically Fig 3.47 and the associated pseudocode.
2063 const U32 laststate = TRIE_NODENUM( next_alloc );
2066 trie->statecount = laststate;
2068 for ( state = 1 ; state < laststate ; state++ ) {
2070 const U32 stateidx = TRIE_NODEIDX( state );
2071 const U32 o_used = trie->trans[ stateidx ].check;
2072 U32 used = trie->trans[ stateidx ].check;
2073 trie->trans[ stateidx ].check = 0;
2075 for ( charid = 0 ; used && charid < trie->uniquecharcount ; charid++ ) {
2076 if ( flag || trie->trans[ stateidx + charid ].next ) {
2077 if ( trie->trans[ stateidx + charid ].next ) {
2079 for ( ; zp < pos ; zp++ ) {
2080 if ( ! trie->trans[ zp ].next ) {
2084 trie->states[ state ].trans.base = zp + trie->uniquecharcount - charid ;
2085 trie->trans[ zp ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next );
2086 trie->trans[ zp ].check = state;
2087 if ( ++zp > pos ) pos = zp;
2094 trie->states[ state ].trans.base = pos + trie->uniquecharcount - charid ;
2096 trie->trans[ pos ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next );
2097 trie->trans[ pos ].check = state;
2102 trie->lasttrans = pos + 1;
2103 trie->states = (reg_trie_state *)
2104 PerlMemShared_realloc( trie->states, laststate
2105 * sizeof(reg_trie_state) );
2106 DEBUG_TRIE_COMPILE_MORE_r(
2107 PerlIO_printf( Perl_debug_log,
2108 "%*sAlloc: %d Orig: %"IVdf" elements, Final:%"IVdf". Savings of %%%5.2f\n",
2109 (int)depth * 2 + 2,"",
2110 (int)( ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1 ),
2113 ( ( next_alloc - pos ) * 100 ) / (double)next_alloc );
2116 } /* end table compress */
2118 DEBUG_TRIE_COMPILE_MORE_r(
2119 PerlIO_printf(Perl_debug_log, "%*sStatecount:%"UVxf" Lasttrans:%"UVxf"\n",
2120 (int)depth * 2 + 2, "",
2121 (UV)trie->statecount,
2122 (UV)trie->lasttrans)
2124 /* resize the trans array to remove unused space */
2125 trie->trans = (reg_trie_trans *)
2126 PerlMemShared_realloc( trie->trans, trie->lasttrans
2127 * sizeof(reg_trie_trans) );
2129 { /* Modify the program and insert the new TRIE node */
2130 U8 nodetype =(U8)(flags & 0xFF);
2134 regnode *optimize = NULL;
2135 #ifdef RE_TRACK_PATTERN_OFFSETS
2138 U32 mjd_nodelen = 0;
2139 #endif /* RE_TRACK_PATTERN_OFFSETS */
2140 #endif /* DEBUGGING */
2142 This means we convert either the first branch or the first Exact,
2143 depending on whether the thing following (in 'last') is a branch
2144 or not and whther first is the startbranch (ie is it a sub part of
2145 the alternation or is it the whole thing.)
2146 Assuming its a sub part we convert the EXACT otherwise we convert
2147 the whole branch sequence, including the first.
2149 /* Find the node we are going to overwrite */
2150 if ( first != startbranch || OP( last ) == BRANCH ) {
2151 /* branch sub-chain */
2152 NEXT_OFF( first ) = (U16)(last - first);
2153 #ifdef RE_TRACK_PATTERN_OFFSETS
2155 mjd_offset= Node_Offset((convert));
2156 mjd_nodelen= Node_Length((convert));
2159 /* whole branch chain */
2161 #ifdef RE_TRACK_PATTERN_OFFSETS
2164 const regnode *nop = NEXTOPER( convert );
2165 mjd_offset= Node_Offset((nop));
2166 mjd_nodelen= Node_Length((nop));
2170 PerlIO_printf(Perl_debug_log, "%*sMJD offset:%"UVuf" MJD length:%"UVuf"\n",
2171 (int)depth * 2 + 2, "",
2172 (UV)mjd_offset, (UV)mjd_nodelen)
2175 /* But first we check to see if there is a common prefix we can
2176 split out as an EXACT and put in front of the TRIE node. */
2177 trie->startstate= 1;
2178 if ( trie->bitmap && !widecharmap && !trie->jump ) {
2180 for ( state = 1 ; state < trie->statecount-1 ; state++ ) {
2184 const U32 base = trie->states[ state ].trans.base;
2186 if ( trie->states[state].wordnum )
2189 for ( ofs = 0 ; ofs < trie->uniquecharcount ; ofs++ ) {
2190 if ( ( base + ofs >= trie->uniquecharcount ) &&
2191 ( base + ofs - trie->uniquecharcount < trie->lasttrans ) &&
2192 trie->trans[ base + ofs - trie->uniquecharcount ].check == state )
2194 if ( ++count > 1 ) {
2195 SV **tmp = av_fetch( revcharmap, ofs, 0);
2196 const U8 *ch = (U8*)SvPV_nolen_const( *tmp );
2197 if ( state == 1 ) break;
2199 Zero(trie->bitmap, ANYOF_BITMAP_SIZE, char);
2201 PerlIO_printf(Perl_debug_log,
2202 "%*sNew Start State=%"UVuf" Class: [",
2203 (int)depth * 2 + 2, "",
2206 SV ** const tmp = av_fetch( revcharmap, idx, 0);
2207 const U8 * const ch = (U8*)SvPV_nolen_const( *tmp );
2209 TRIE_BITMAP_SET(trie,*ch);
2211 TRIE_BITMAP_SET(trie, folder[ *ch ]);
2213 PerlIO_printf(Perl_debug_log, "%s", (char*)ch)
2217 TRIE_BITMAP_SET(trie,*ch);
2219 TRIE_BITMAP_SET(trie,folder[ *ch ]);
2220 DEBUG_OPTIMISE_r(PerlIO_printf( Perl_debug_log,"%s", ch));
2226 SV **tmp = av_fetch( revcharmap, idx, 0);
2228 char *ch = SvPV( *tmp, len );
2230 SV *sv=sv_newmortal();
2231 PerlIO_printf( Perl_debug_log,
2232 "%*sPrefix State: %"UVuf" Idx:%"UVuf" Char='%s'\n",
2233 (int)depth * 2 + 2, "",
2235 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), 6,
2236 PL_colors[0], PL_colors[1],
2237 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
2238 PERL_PV_ESCAPE_FIRSTCHAR
2243 OP( convert ) = nodetype;
2244 str=STRING(convert);
2247 STR_LEN(convert) += len;
2253 DEBUG_OPTIMISE_r(PerlIO_printf( Perl_debug_log,"]\n"));
2258 trie->prefixlen = (state-1);
2260 regnode *n = convert+NODE_SZ_STR(convert);
2261 NEXT_OFF(convert) = NODE_SZ_STR(convert);
2262 trie->startstate = state;
2263 trie->minlen -= (state - 1);
2264 trie->maxlen -= (state - 1);
2266 /* At least the UNICOS C compiler choked on this
2267 * being argument to DEBUG_r(), so let's just have
2270 #ifdef PERL_EXT_RE_BUILD
2276 regnode *fix = convert;
2277 U32 word = trie->wordcount;
2279 Set_Node_Offset_Length(convert, mjd_offset, state - 1);
2280 while( ++fix < n ) {
2281 Set_Node_Offset_Length(fix, 0, 0);
2284 SV ** const tmp = av_fetch( trie_words, word, 0 );
2286 if ( STR_LEN(convert) <= SvCUR(*tmp) )
2287 sv_chop(*tmp, SvPV_nolen(*tmp) + STR_LEN(convert));
2289 sv_chop(*tmp, SvPV_nolen(*tmp) + SvCUR(*tmp));
2297 NEXT_OFF(convert) = (U16)(tail - convert);
2298 DEBUG_r(optimize= n);
2304 if ( trie->maxlen ) {
2305 NEXT_OFF( convert ) = (U16)(tail - convert);
2306 ARG_SET( convert, data_slot );
2307 /* Store the offset to the first unabsorbed branch in
2308 jump[0], which is otherwise unused by the jump logic.
2309 We use this when dumping a trie and during optimisation. */
2311 trie->jump[0] = (U16)(nextbranch - convert);
2313 /* If the start state is not accepting (meaning there is no empty string/NOTHING)
2314 * and there is a bitmap
2315 * and the first "jump target" node we found leaves enough room
2316 * then convert the TRIE node into a TRIEC node, with the bitmap
2317 * embedded inline in the opcode - this is hypothetically faster.
2319 if ( !trie->states[trie->startstate].wordnum
2321 && ( (char *)jumper - (char *)convert) >= (int)sizeof(struct regnode_charclass) )
2323 OP( convert ) = TRIEC;
2324 Copy(trie->bitmap, ((struct regnode_charclass *)convert)->bitmap, ANYOF_BITMAP_SIZE, char);
2325 PerlMemShared_free(trie->bitmap);
2328 OP( convert ) = TRIE;
2330 /* store the type in the flags */
2331 convert->flags = nodetype;
2335 + regarglen[ OP( convert ) ];
2337 /* XXX We really should free up the resource in trie now,
2338 as we won't use them - (which resources?) dmq */
2340 /* needed for dumping*/
2341 DEBUG_r(if (optimize) {
2342 regnode *opt = convert;
2344 while ( ++opt < optimize) {
2345 Set_Node_Offset_Length(opt,0,0);
2348 Try to clean up some of the debris left after the
2351 while( optimize < jumper ) {
2352 mjd_nodelen += Node_Length((optimize));
2353 OP( optimize ) = OPTIMIZED;
2354 Set_Node_Offset_Length(optimize,0,0);
2357 Set_Node_Offset_Length(convert,mjd_offset,mjd_nodelen);
2359 } /* end node insert */
2360 REH_CALL_COMP_NODE_HOOK(pRExC_state->rx, convert);
2362 /* Finish populating the prev field of the wordinfo array. Walk back
2363 * from each accept state until we find another accept state, and if
2364 * so, point the first word's .prev field at the second word. If the
2365 * second already has a .prev field set, stop now. This will be the
2366 * case either if we've already processed that word's accept state,
2367 * or that state had multiple words, and the overspill words were
2368 * already linked up earlier.
2375 for (word=1; word <= trie->wordcount; word++) {
2377 if (trie->wordinfo[word].prev)
2379 state = trie->wordinfo[word].accept;
2381 state = prev_states[state];
2384 prev = trie->states[state].wordnum;
2388 trie->wordinfo[word].prev = prev;
2390 Safefree(prev_states);
2394 /* and now dump out the compressed format */
2395 DEBUG_TRIE_COMPILE_r(dump_trie(trie, widecharmap, revcharmap, depth+1));
2397 RExC_rxi->data->data[ data_slot + 1 ] = (void*)widecharmap;
2399 RExC_rxi->data->data[ data_slot + TRIE_WORDS_OFFSET ] = (void*)trie_words;
2400 RExC_rxi->data->data[ data_slot + 3 ] = (void*)revcharmap;
2402 SvREFCNT_dec(revcharmap);
2406 : trie->startstate>1
2412 S_make_trie_failtable(pTHX_ RExC_state_t *pRExC_state, regnode *source, regnode *stclass, U32 depth)
2414 /* The Trie is constructed and compressed now so we can build a fail array if it's needed
2416 This is basically the Aho-Corasick algorithm. Its from exercise 3.31 and 3.32 in the
2417 "Red Dragon" -- Compilers, principles, techniques, and tools. Aho, Sethi, Ullman 1985/88
2420 We find the fail state for each state in the trie, this state is the longest proper
2421 suffix of the current state's 'word' that is also a proper prefix of another word in our
2422 trie. State 1 represents the word '' and is thus the default fail state. This allows
2423 the DFA not to have to restart after its tried and failed a word at a given point, it
2424 simply continues as though it had been matching the other word in the first place.
2426 'abcdgu'=~/abcdefg|cdgu/
2427 When we get to 'd' we are still matching the first word, we would encounter 'g' which would
2428 fail, which would bring us to the state representing 'd' in the second word where we would
2429 try 'g' and succeed, proceeding to match 'cdgu'.
2431 /* add a fail transition */
2432 const U32 trie_offset = ARG(source);
2433 reg_trie_data *trie=(reg_trie_data *)RExC_rxi->data->data[trie_offset];
2435 const U32 ucharcount = trie->uniquecharcount;
2436 const U32 numstates = trie->statecount;
2437 const U32 ubound = trie->lasttrans + ucharcount;
2441 U32 base = trie->states[ 1 ].trans.base;
2444 const U32 data_slot = add_data( pRExC_state, 1, "T" );
2445 GET_RE_DEBUG_FLAGS_DECL;
2447 PERL_ARGS_ASSERT_MAKE_TRIE_FAILTABLE;
2449 PERL_UNUSED_ARG(depth);
2453 ARG_SET( stclass, data_slot );
2454 aho = (reg_ac_data *) PerlMemShared_calloc( 1, sizeof(reg_ac_data) );
2455 RExC_rxi->data->data[ data_slot ] = (void*)aho;
2456 aho->trie=trie_offset;
2457 aho->states=(reg_trie_state *)PerlMemShared_malloc( numstates * sizeof(reg_trie_state) );
2458 Copy( trie->states, aho->states, numstates, reg_trie_state );
2459 Newxz( q, numstates, U32);
2460 aho->fail = (U32 *) PerlMemShared_calloc( numstates, sizeof(U32) );
2463 /* initialize fail[0..1] to be 1 so that we always have
2464 a valid final fail state */
2465 fail[ 0 ] = fail[ 1 ] = 1;
2467 for ( charid = 0; charid < ucharcount ; charid++ ) {
2468 const U32 newstate = TRIE_TRANS_STATE( 1, base, ucharcount, charid, 0 );
2470 q[ q_write ] = newstate;
2471 /* set to point at the root */
2472 fail[ q[ q_write++ ] ]=1;
2475 while ( q_read < q_write) {
2476 const U32 cur = q[ q_read++ % numstates ];
2477 base = trie->states[ cur ].trans.base;
2479 for ( charid = 0 ; charid < ucharcount ; charid++ ) {
2480 const U32 ch_state = TRIE_TRANS_STATE( cur, base, ucharcount, charid, 1 );
2482 U32 fail_state = cur;
2485 fail_state = fail[ fail_state ];
2486 fail_base = aho->states[ fail_state ].trans.base;
2487 } while ( !TRIE_TRANS_STATE( fail_state, fail_base, ucharcount, charid, 1 ) );
2489 fail_state = TRIE_TRANS_STATE( fail_state, fail_base, ucharcount, charid, 1 );
2490 fail[ ch_state ] = fail_state;
2491 if ( !aho->states[ ch_state ].wordnum && aho->states[ fail_state ].wordnum )
2493 aho->states[ ch_state ].wordnum = aho->states[ fail_state ].wordnum;
2495 q[ q_write++ % numstates] = ch_state;
2499 /* restore fail[0..1] to 0 so that we "fall out" of the AC loop
2500 when we fail in state 1, this allows us to use the
2501 charclass scan to find a valid start char. This is based on the principle
2502 that theres a good chance the string being searched contains lots of stuff
2503 that cant be a start char.
2505 fail[ 0 ] = fail[ 1 ] = 0;
2506 DEBUG_TRIE_COMPILE_r({
2507 PerlIO_printf(Perl_debug_log,
2508 "%*sStclass Failtable (%"UVuf" states): 0",
2509 (int)(depth * 2), "", (UV)numstates
2511 for( q_read=1; q_read<numstates; q_read++ ) {
2512 PerlIO_printf(Perl_debug_log, ", %"UVuf, (UV)fail[q_read]);
2514 PerlIO_printf(Perl_debug_log, "\n");
2517 /*RExC_seen |= REG_SEEN_TRIEDFA;*/
2522 * There are strange code-generation bugs caused on sparc64 by gcc-2.95.2.
2523 * These need to be revisited when a newer toolchain becomes available.
2525 #if defined(__sparc64__) && defined(__GNUC__)
2526 # if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
2527 # undef SPARC64_GCC_WORKAROUND
2528 # define SPARC64_GCC_WORKAROUND 1
2532 #define DEBUG_PEEP(str,scan,depth) \
2533 DEBUG_OPTIMISE_r({if (scan){ \
2534 SV * const mysv=sv_newmortal(); \
2535 regnode *Next = regnext(scan); \
2536 regprop(RExC_rx, mysv, scan); \
2537 PerlIO_printf(Perl_debug_log, "%*s" str ">%3d: %s (%d)\n", \
2538 (int)depth*2, "", REG_NODE_NUM(scan), SvPV_nolen_const(mysv),\
2539 Next ? (REG_NODE_NUM(Next)) : 0 ); \
2543 /* The below joins as many adjacent EXACTish nodes as possible into a single
2544 * one, and looks for problematic sequences of characters whose folds vs.
2545 * non-folds have sufficiently different lengths, that the optimizer would be
2546 * fooled into rejecting legitimate matches of them, and the trie construction
2547 * code can't cope with them. The joining is only done if:
2548 * 1) there is room in the current conglomerated node to entirely contain the
2550 * 2) they are the exact same node type
2552 * The adjacent nodes actually may be separated by NOTHING kind nodes, and
2553 * these get optimized out
2555 * If there are problematic code sequences, *min_subtract is set to the delta
2556 * that the minimum size of the node can be less than its actual size. And,
2557 * the node type of the result is changed to reflect that it contains these
2560 * And *has_exactf_sharp_s is set to indicate whether or not the node is EXACTF
2561 * and contains LATIN SMALL LETTER SHARP S
2563 * This is as good a place as any to discuss the design of handling these
2564 * problematic sequences. It's been wrong in Perl for a very long time. There
2565 * are three code points in Unicode whose folded lengths differ so much from
2566 * the un-folded lengths that it causes problems for the optimizer and trie
2567 * construction. Why only these are problematic, and not others where lengths
2568 * also differ is something I (khw) do not understand. New versions of Unicode
2569 * might add more such code points. Hopefully the logic in fold_grind.t that
2570 * figures out what to test (in part by verifying that each size-combination
2571 * gets tested) will catch any that do come along, so they can be added to the
2572 * special handling below. The chances of new ones are actually rather small,
2573 * as most, if not all, of the world's scripts that have casefolding have
2574 * already been encoded by Unicode. Also, a number of Unicode's decisions were
2575 * made to allow compatibility with pre-existing standards, and almost all of
2576 * those have already been dealt with. These would otherwise be the most
2577 * likely candidates for generating further tricky sequences. In other words,
2578 * Unicode by itself is unlikely to add new ones unless it is for compatibility
2579 * with pre-existing standards, and there aren't many of those left.
2581 * The previous designs for dealing with these involved assigning a special
2582 * node for them. This approach doesn't work, as evidenced by this example:
2583 * "\xDFs" =~ /s\xDF/ui # Used to fail before these patches
2584 * Both these fold to "sss", but if the pattern is parsed to create a node of
2585 * that would match just the \xDF, it won't be able to handle the case where a
2586 * successful match would have to cross the node's boundary. The new approach
2587 * that hopefully generally solves the problem generates an EXACTFU_SS node
2590 * There are a number of components to the approach (a lot of work for just
2591 * three code points!):
2592 * 1) This routine examines each EXACTFish node that could contain the
2593 * problematic sequences. It returns in *min_subtract how much to
2594 * subtract from the the actual length of the string to get a real minimum
2595 * for one that could match it. This number is usually 0 except for the
2596 * problematic sequences. This delta is used by the caller to adjust the
2597 * min length of the match, and the delta between min and max, so that the
2598 * optimizer doesn't reject these possibilities based on size constraints.
2599 * 2) These sequences are not currently correctly handled by the trie code
2600 * either, so it changes the joined node type to ops that are not handled
2601 * by trie's, those new ops being EXACTFU_SS and EXACTFU_TRICKYFOLD.
2602 * 3) This is sufficient for the two Greek sequences (described below), but
2603 * the one involving the Sharp s (\xDF) needs more. The node type
2604 * EXACTFU_SS is used for an EXACTFU node that contains at least one "ss"
2605 * sequence in it. For non-UTF-8 patterns and strings, this is the only
2606 * case where there is a possible fold length change. That means that a
2607 * regular EXACTFU node without UTF-8 involvement doesn't have to concern
2608 * itself with length changes, and so can be processed faster. regexec.c
2609 * takes advantage of this. Generally, an EXACTFish node that is in UTF-8
2610 * is pre-folded by regcomp.c. This saves effort in regex matching.
2611 * However, probably mostly for historical reasons, the pre-folding isn't
2612 * done for non-UTF8 patterns (and it can't be for EXACTF and EXACTFL
2613 * nodes, as what they fold to isn't known until runtime.) The fold
2614 * possibilities for the non-UTF8 patterns are quite simple, except for
2615 * the sharp s. All the ones that don't involve a UTF-8 target string
2616 * are members of a fold-pair, and arrays are set up for all of them
2617 * that quickly find the other member of the pair. It might actually
2618 * be faster to pre-fold these, but it isn't currently done, except for
2619 * the sharp s. Code elsewhere in this file makes sure that it gets
2620 * folded to 'ss', even if the pattern isn't UTF-8. This avoids the
2621 * issues described in the next item.
2622 * 4) A problem remains for the sharp s in EXACTF nodes. Whether it matches
2623 * 'ss' or not is not knowable at compile time. It will match iff the
2624 * target string is in UTF-8, unlike the EXACTFU nodes, where it always
2625 * matches; and the EXACTFL and EXACTFA nodes where it never does. Thus
2626 * it can't be folded to "ss" at compile time, unlike EXACTFU does as
2627 * described in item 3). An assumption that the optimizer part of
2628 * regexec.c (probably unwittingly) makes is that a character in the
2629 * pattern corresponds to at most a single character in the target string.
2630 * (And I do mean character, and not byte here, unlike other parts of the
2631 * documentation that have never been updated to account for multibyte
2632 * Unicode.) This assumption is wrong only in this case, as all other
2633 * cases are either 1-1 folds when no UTF-8 is involved; or is true by
2634 * virtue of having this file pre-fold UTF-8 patterns. I'm
2635 * reluctant to try to change this assumption, so instead the code punts.
2636 * This routine examines EXACTF nodes for the sharp s, and returns a
2637 * boolean indicating whether or not the node is an EXACTF node that
2638 * contains a sharp s. When it is true, the caller sets a flag that later
2639 * causes the optimizer in this file to not set values for the floating
2640 * and fixed string lengths, and thus avoids the optimizer code in
2641 * regexec.c that makes the invalid assumption. Thus, there is no
2642 * optimization based on string lengths for EXACTF nodes that contain the
2643 * sharp s. This only happens for /id rules (which means the pattern
2647 #define JOIN_EXACT(scan,min_subtract,has_exactf_sharp_s, flags) \
2648 if (PL_regkind[OP(scan)] == EXACT) \
2649 join_exact(pRExC_state,(scan),(min_subtract),has_exactf_sharp_s, (flags),NULL,depth+1)
2652 S_join_exact(pTHX_ RExC_state_t *pRExC_state, regnode *scan, UV *min_subtract, bool *has_exactf_sharp_s, U32 flags,regnode *val, U32 depth) {
2653 /* Merge several consecutive EXACTish nodes into one. */
2654 regnode *n = regnext(scan);
2656 regnode *next = scan + NODE_SZ_STR(scan);
2660 regnode *stop = scan;
2661 GET_RE_DEBUG_FLAGS_DECL;
2663 PERL_UNUSED_ARG(depth);
2666 PERL_ARGS_ASSERT_JOIN_EXACT;
2667 #ifndef EXPERIMENTAL_INPLACESCAN
2668 PERL_UNUSED_ARG(flags);
2669 PERL_UNUSED_ARG(val);
2671 DEBUG_PEEP("join",scan,depth);
2673 /* Look through the subsequent nodes in the chain. Skip NOTHING, merge
2674 * EXACT ones that are mergeable to the current one. */
2676 && (PL_regkind[OP(n)] == NOTHING
2677 || (stringok && OP(n) == OP(scan)))
2679 && NEXT_OFF(scan) + NEXT_OFF(n) < I16_MAX)
2682 if (OP(n) == TAIL || n > next)
2684 if (PL_regkind[OP(n)] == NOTHING) {
2685 DEBUG_PEEP("skip:",n,depth);
2686 NEXT_OFF(scan) += NEXT_OFF(n);
2687 next = n + NODE_STEP_REGNODE;
2694 else if (stringok) {
2695 const unsigned int oldl = STR_LEN(scan);
2696 regnode * const nnext = regnext(n);
2698 if (oldl + STR_LEN(n) > U8_MAX)
2701 DEBUG_PEEP("merg",n,depth);
2704 NEXT_OFF(scan) += NEXT_OFF(n);
2705 STR_LEN(scan) += STR_LEN(n);
2706 next = n + NODE_SZ_STR(n);
2707 /* Now we can overwrite *n : */
2708 Move(STRING(n), STRING(scan) + oldl, STR_LEN(n), char);
2716 #ifdef EXPERIMENTAL_INPLACESCAN
2717 if (flags && !NEXT_OFF(n)) {
2718 DEBUG_PEEP("atch", val, depth);
2719 if (reg_off_by_arg[OP(n)]) {
2720 ARG_SET(n, val - n);
2723 NEXT_OFF(n) = val - n;
2731 *has_exactf_sharp_s = FALSE;
2733 /* Here, all the adjacent mergeable EXACTish nodes have been merged. We
2734 * can now analyze for sequences of problematic code points. (Prior to
2735 * this final joining, sequences could have been split over boundaries, and
2736 * hence missed). The sequences only happen in folding, hence for any
2737 * non-EXACT EXACTish node */
2738 if (OP(scan) != EXACT) {
2740 U8 * s0 = (U8*) STRING(scan);
2741 U8 * const s_end = s0 + STR_LEN(scan);
2743 /* The below is perhaps overboard, but this allows us to save a test
2744 * each time through the loop at the expense of a mask. This is
2745 * because on both EBCDIC and ASCII machines, 'S' and 's' differ by a
2746 * single bit. On ASCII they are 32 apart; on EBCDIC, they are 64.
2747 * This uses an exclusive 'or' to find that bit and then inverts it to
2748 * form a mask, with just a single 0, in the bit position where 'S' and
2750 const U8 S_or_s_mask = (U8) ~ ('S' ^ 's');
2751 const U8 s_masked = 's' & S_or_s_mask;
2753 /* One pass is made over the node's string looking for all the
2754 * possibilities. to avoid some tests in the loop, there are two main
2755 * cases, for UTF-8 patterns (which can't have EXACTF nodes) and
2759 /* There are two problematic Greek code points in Unicode
2762 * U+0390 - GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
2763 * U+03B0 - GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
2769 * U+03B9 U+0308 U+0301 0xCE 0xB9 0xCC 0x88 0xCC 0x81
2770 * U+03C5 U+0308 U+0301 0xCF 0x85 0xCC 0x88 0xCC 0x81
2772 * This means that in case-insensitive matching (or "loose
2773 * matching", as Unicode calls it), an EXACTF of length six (the
2774 * UTF-8 encoded byte length of the above casefolded versions) can
2775 * match a target string of length two (the byte length of UTF-8
2776 * encoded U+0390 or U+03B0). This would rather mess up the
2777 * minimum length computation. (there are other code points that
2778 * also fold to these two sequences, but the delta is smaller)
2780 * If these sequences are found, the minimum length is decreased by
2781 * four (six minus two).
2783 * Similarly, 'ss' may match the single char and byte LATIN SMALL
2784 * LETTER SHARP S. We decrease the min length by 1 for each
2785 * occurrence of 'ss' found */
2787 #ifdef EBCDIC /* RD tunifold greek 0390 and 03B0 */
2788 # define U390_first_byte 0xb4
2789 const U8 U390_tail[] = "\x68\xaf\x49\xaf\x42";
2790 # define U3B0_first_byte 0xb5
2791 const U8 U3B0_tail[] = "\x46\xaf\x49\xaf\x42";
2793 # define U390_first_byte 0xce
2794 const U8 U390_tail[] = "\xb9\xcc\x88\xcc\x81";
2795 # define U3B0_first_byte 0xcf
2796 const U8 U3B0_tail[] = "\x85\xcc\x88\xcc\x81";
2798 const U8 len = sizeof(U390_tail); /* (-1 for NUL; +1 for 1st byte;
2799 yields a net of 0 */
2800 /* Examine the string for one of the problematic sequences */
2802 s < s_end - 1; /* Can stop 1 before the end, as minimum length
2803 * sequence we are looking for is 2 */
2807 /* Look for the first byte in each problematic sequence */
2809 /* We don't have to worry about other things that fold to
2810 * 's' (such as the long s, U+017F), as all above-latin1
2811 * code points have been pre-folded */
2815 /* Current character is an 's' or 'S'. If next one is
2816 * as well, we have the dreaded sequence */
2817 if (((*(s+1) & S_or_s_mask) == s_masked)
2818 /* These two node types don't have special handling
2820 && OP(scan) != EXACTFL && OP(scan) != EXACTFA)
2823 OP(scan) = EXACTFU_SS;
2824 s++; /* No need to look at this character again */
2828 case U390_first_byte:
2829 if (s_end - s >= len
2831 /* The 1's are because are skipping comparing the
2833 && memEQ(s + 1, U390_tail, len - 1))
2835 goto greek_sequence;
2839 case U3B0_first_byte:
2840 if (! (s_end - s >= len
2841 && memEQ(s + 1, U3B0_tail, len - 1)))
2848 /* This can't currently be handled by trie's, so change
2849 * the node type to indicate this. If EXACTFA and
2850 * EXACTFL were ever to be handled by trie's, this
2851 * would have to be changed. If this node has already
2852 * been changed to EXACTFU_SS in this loop, leave it as
2853 * is. (I (khw) think it doesn't matter in regexec.c
2854 * for UTF patterns, but no need to change it */
2855 if (OP(scan) == EXACTFU) {
2856 OP(scan) = EXACTFU_TRICKYFOLD;
2858 s += 6; /* We already know what this sequence is. Skip
2864 else if (OP(scan) != EXACTFL && OP(scan) != EXACTFA) {
2866 /* Here, the pattern is not UTF-8. We need to look only for the
2867 * 'ss' sequence, and in the EXACTF case, the sharp s, which can be
2868 * in the final position. Otherwise we can stop looking 1 byte
2869 * earlier because have to find both the first and second 's' */
2870 const U8* upper = (OP(scan) == EXACTF) ? s_end : s_end -1;
2872 for (s = s0; s < upper; s++) {
2877 && ((*(s+1) & S_or_s_mask) == s_masked))
2881 /* EXACTF nodes need to know that the minimum
2882 * length changed so that a sharp s in the string
2883 * can match this ss in the pattern, but they
2884 * remain EXACTF nodes, as they are not trie'able,
2885 * so don't have to invent a new node type to
2886 * exclude them from the trie code */
2887 if (OP(scan) != EXACTF) {
2888 OP(scan) = EXACTFU_SS;
2893 case LATIN_SMALL_LETTER_SHARP_S:
2894 if (OP(scan) == EXACTF) {
2895 *has_exactf_sharp_s = TRUE;
2904 /* Allow dumping but overwriting the collection of skipped
2905 * ops and/or strings with fake optimized ops */
2906 n = scan + NODE_SZ_STR(scan);
2914 DEBUG_OPTIMISE_r(if (merged){DEBUG_PEEP("finl",scan,depth)});
2918 /* REx optimizer. Converts nodes into quicker variants "in place".
2919 Finds fixed substrings. */
2921 /* Stops at toplevel WHILEM as well as at "last". At end *scanp is set
2922 to the position after last scanned or to NULL. */
2924 #define INIT_AND_WITHP \
2925 assert(!and_withp); \
2926 Newx(and_withp,1,struct regnode_charclass_class); \
2927 SAVEFREEPV(and_withp)
2929 /* this is a chain of data about sub patterns we are processing that
2930 need to be handled separately/specially in study_chunk. Its so
2931 we can simulate recursion without losing state. */
2933 typedef struct scan_frame {
2934 regnode *last; /* last node to process in this frame */
2935 regnode *next; /* next node to process when last is reached */
2936 struct scan_frame *prev; /*previous frame*/
2937 I32 stop; /* what stopparen do we use */
2941 #define SCAN_COMMIT(s, data, m) scan_commit(s, data, m, is_inf)
2943 #define CASE_SYNST_FNC(nAmE) \
2945 if (flags & SCF_DO_STCLASS_AND) { \
2946 for (value = 0; value < 256; value++) \
2947 if (!is_ ## nAmE ## _cp(value)) \
2948 ANYOF_BITMAP_CLEAR(data->start_class, value); \
2951 for (value = 0; value < 256; value++) \
2952 if (is_ ## nAmE ## _cp(value)) \
2953 ANYOF_BITMAP_SET(data->start_class, value); \
2957 if (flags & SCF_DO_STCLASS_AND) { \
2958 for (value = 0; value < 256; value++) \
2959 if (is_ ## nAmE ## _cp(value)) \
2960 ANYOF_BITMAP_CLEAR(data->start_class, value); \
2963 for (value = 0; value < 256; value++) \
2964 if (!is_ ## nAmE ## _cp(value)) \
2965 ANYOF_BITMAP_SET(data->start_class, value); \
2972 S_study_chunk(pTHX_ RExC_state_t *pRExC_state, regnode **scanp,
2973 I32 *minlenp, I32 *deltap,
2978 struct regnode_charclass_class *and_withp,
2979 U32 flags, U32 depth)
2980 /* scanp: Start here (read-write). */
2981 /* deltap: Write maxlen-minlen here. */
2982 /* last: Stop before this one. */
2983 /* data: string data about the pattern */
2984 /* stopparen: treat close N as END */
2985 /* recursed: which subroutines have we recursed into */
2986 /* and_withp: Valid if flags & SCF_DO_STCLASS_OR */
2989 I32 min = 0, pars = 0, code;
2990 regnode *scan = *scanp, *next;
2992 int is_inf = (flags & SCF_DO_SUBSTR) && (data->flags & SF_IS_INF);
2993 int is_inf_internal = 0; /* The studied chunk is infinite */
2994 I32 is_par = OP(scan) == OPEN ? ARG(scan) : 0;
2995 scan_data_t data_fake;
2996 SV *re_trie_maxbuff = NULL;
2997 regnode *first_non_open = scan;
2998 I32 stopmin = I32_MAX;
2999 scan_frame *frame = NULL;
3000 GET_RE_DEBUG_FLAGS_DECL;
3002 PERL_ARGS_ASSERT_STUDY_CHUNK;
3005 StructCopy(&zero_scan_data, &data_fake, scan_data_t);
3009 while (first_non_open && OP(first_non_open) == OPEN)
3010 first_non_open=regnext(first_non_open);
3015 while ( scan && OP(scan) != END && scan < last ){
3016 UV min_subtract = 0; /* How much to subtract from the minimum node
3017 length to get a real minimum (because the
3018 folded version may be shorter) */
3019 bool has_exactf_sharp_s = FALSE;
3020 /* Peephole optimizer: */
3021 DEBUG_STUDYDATA("Peep:", data,depth);
3022 DEBUG_PEEP("Peep",scan,depth);
3024 /* Its not clear to khw or hv why this is done here, and not in the
3025 * clauses that deal with EXACT nodes. khw's guess is that it's
3026 * because of a previous design */
3027 JOIN_EXACT(scan,&min_subtract, &has_exactf_sharp_s, 0);
3029 /* Follow the next-chain of the current node and optimize
3030 away all the NOTHINGs from it. */
3031 if (OP(scan) != CURLYX) {
3032 const int max = (reg_off_by_arg[OP(scan)]
3034 /* I32 may be smaller than U16 on CRAYs! */
3035 : (I32_MAX < U16_MAX ? I32_MAX : U16_MAX));
3036 int off = (reg_off_by_arg[OP(scan)] ? ARG(scan) : NEXT_OFF(scan));
3040 /* Skip NOTHING and LONGJMP. */
3041 while ((n = regnext(n))
3042 && ((PL_regkind[OP(n)] == NOTHING && (noff = NEXT_OFF(n)))
3043 || ((OP(n) == LONGJMP) && (noff = ARG(n))))
3044 && off + noff < max)
3046 if (reg_off_by_arg[OP(scan)])
3049 NEXT_OFF(scan) = off;
3054 /* The principal pseudo-switch. Cannot be a switch, since we
3055 look into several different things. */
3056 if (OP(scan) == BRANCH || OP(scan) == BRANCHJ
3057 || OP(scan) == IFTHEN) {
3058 next = regnext(scan);
3060 /* demq: the op(next)==code check is to see if we have "branch-branch" AFAICT */
3062 if (OP(next) == code || code == IFTHEN) {
3063 /* NOTE - There is similar code to this block below for handling
3064 TRIE nodes on a re-study. If you change stuff here check there
3066 I32 max1 = 0, min1 = I32_MAX, num = 0;
3067 struct regnode_charclass_class accum;
3068 regnode * const startbranch=scan;
3070 if (flags & SCF_DO_SUBSTR)
3071 SCAN_COMMIT(pRExC_state, data, minlenp); /* Cannot merge strings after this. */
3072 if (flags & SCF_DO_STCLASS)
3073 cl_init_zero(pRExC_state, &accum);
3075 while (OP(scan) == code) {
3076 I32 deltanext, minnext, f = 0, fake;
3077 struct regnode_charclass_class this_class;
3080 data_fake.flags = 0;
3082 data_fake.whilem_c = data->whilem_c;
3083 data_fake.last_closep = data->last_closep;
3086 data_fake.last_closep = &fake;
3088 data_fake.pos_delta = delta;
3089 next = regnext(scan);
3090 scan = NEXTOPER(scan);
3092 scan = NEXTOPER(scan);
3093 if (flags & SCF_DO_STCLASS) {
3094 cl_init(pRExC_state, &this_class);
3095 data_fake.start_class = &this_class;
3096 f = SCF_DO_STCLASS_AND;
3098 if (flags & SCF_WHILEM_VISITED_POS)
3099 f |= SCF_WHILEM_VISITED_POS;
3101 /* we suppose the run is continuous, last=next...*/
3102 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext,
3104 stopparen, recursed, NULL, f,depth+1);
3107 if (max1 < minnext + deltanext)
3108 max1 = minnext + deltanext;
3109 if (deltanext == I32_MAX)
3110 is_inf = is_inf_internal = 1;
3112 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
3114 if (data_fake.flags & SCF_SEEN_ACCEPT) {
3115 if ( stopmin > minnext)
3116 stopmin = min + min1;
3117 flags &= ~SCF_DO_SUBSTR;
3119 data->flags |= SCF_SEEN_ACCEPT;
3122 if (data_fake.flags & SF_HAS_EVAL)
3123 data->flags |= SF_HAS_EVAL;
3124 data->whilem_c = data_fake.whilem_c;
3126 if (flags & SCF_DO_STCLASS)
3127 cl_or(pRExC_state, &accum, &this_class);
3129 if (code == IFTHEN && num < 2) /* Empty ELSE branch */
3131 if (flags & SCF_DO_SUBSTR) {
3132 data->pos_min += min1;
3133 data->pos_delta += max1 - min1;
3134 if (max1 != min1 || is_inf)
3135 data->longest = &(data->longest_float);
3138 delta += max1 - min1;
3139 if (flags & SCF_DO_STCLASS_OR) {
3140 cl_or(pRExC_state, data->start_class, &accum);
3142 cl_and(data->start_class, and_withp);
3143 flags &= ~SCF_DO_STCLASS;
3146 else if (flags & SCF_DO_STCLASS_AND) {
3148 cl_and(data->start_class, &accum);
3149 flags &= ~SCF_DO_STCLASS;
3152 /* Switch to OR mode: cache the old value of
3153 * data->start_class */
3155 StructCopy(data->start_class, and_withp,
3156 struct regnode_charclass_class);
3157 flags &= ~SCF_DO_STCLASS_AND;
3158 StructCopy(&accum, data->start_class,
3159 struct regnode_charclass_class);
3160 flags |= SCF_DO_STCLASS_OR;
3161 data->start_class->flags |= ANYOF_EOS;
3165 if (PERL_ENABLE_TRIE_OPTIMISATION && OP( startbranch ) == BRANCH ) {
3168 Assuming this was/is a branch we are dealing with: 'scan' now
3169 points at the item that follows the branch sequence, whatever
3170 it is. We now start at the beginning of the sequence and look
3177 which would be constructed from a pattern like /A|LIST|OF|WORDS/
3179 If we can find such a subsequence we need to turn the first
3180 element into a trie and then add the subsequent branch exact
3181 strings to the trie.
3185 1. patterns where the whole set of branches can be converted.
3187 2. patterns where only a subset can be converted.
3189 In case 1 we can replace the whole set with a single regop
3190 for the trie. In case 2 we need to keep the start and end
3193 'BRANCH EXACT; BRANCH EXACT; BRANCH X'
3194 becomes BRANCH TRIE; BRANCH X;
3196 There is an additional case, that being where there is a
3197 common prefix, which gets split out into an EXACT like node
3198 preceding the TRIE node.
3200 If x(1..n)==tail then we can do a simple trie, if not we make
3201 a "jump" trie, such that when we match the appropriate word
3202 we "jump" to the appropriate tail node. Essentially we turn
3203 a nested if into a case structure of sorts.
3208 if (!re_trie_maxbuff) {
3209 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1);
3210 if (!SvIOK(re_trie_maxbuff))
3211 sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT);
3213 if ( SvIV(re_trie_maxbuff)>=0 ) {
3215 regnode *first = (regnode *)NULL;
3216 regnode *last = (regnode *)NULL;
3217 regnode *tail = scan;
3222 SV * const mysv = sv_newmortal(); /* for dumping */
3224 /* var tail is used because there may be a TAIL
3225 regop in the way. Ie, the exacts will point to the
3226 thing following the TAIL, but the last branch will
3227 point at the TAIL. So we advance tail. If we
3228 have nested (?:) we may have to move through several
3232 while ( OP( tail ) == TAIL ) {
3233 /* this is the TAIL generated by (?:) */
3234 tail = regnext( tail );
3239 regprop(RExC_rx, mysv, tail );
3240 PerlIO_printf( Perl_debug_log, "%*s%s%s\n",
3241 (int)depth * 2 + 2, "",
3242 "Looking for TRIE'able sequences. Tail node is: ",
3243 SvPV_nolen_const( mysv )
3249 Step through the branches
3250 cur represents each branch,
3251 noper is the first thing to be matched as part of that branch
3252 noper_next is the regnext() of that node.
3254 We normally handle a case like this /FOO[xyz]|BAR[pqr]/
3255 via a "jump trie" but we also support building with NOJUMPTRIE,
3256 which restricts the trie logic to structures like /FOO|BAR/.
3258 If noper is a trieable nodetype then the branch is a possible optimization
3259 target. If we are building under NOJUMPTRIE then we require that noper_next
3260 is the same as scan (our current position in the regex program).
3262 Once we have two or more consecutive such branches we can create a
3263 trie of the EXACT's contents and stitch it in place into the program.
3265 If the sequence represents all of the branches in the alternation we
3266 replace the entire thing with a single TRIE node.
3268 Otherwise when it is a subsequence we need to stitch it in place and
3269 replace only the relevant branches. This means the first branch has
3270 to remain as it is used by the alternation logic, and its next pointer,
3271 and needs to be repointed at the item on the branch chain following
3272 the last branch we have optimized away.
3274 This could be either a BRANCH, in which case the subsequence is internal,
3275 or it could be the item following the branch sequence in which case the
3276 subsequence is at the end (which does not necessarily mean the first node
3277 is the start of the alternation).
3279 TRIE_TYPE(X) is a define which maps the optype to a trietype.
3282 ----------------+-----------
3286 EXACTFU_SS | EXACTFU
3287 EXACTFU_TRICKYFOLD | EXACTFU
3292 #define TRIE_TYPE(X) ( ( NOTHING == (X) ) ? NOTHING : \
3293 ( EXACT == (X) ) ? EXACT : \
3294 ( EXACTFU == (X) || EXACTFU_SS == (X) || EXACTFU_TRICKYFOLD == (X) ) ? EXACTFU : \
3297 /* dont use tail as the end marker for this traverse */
3298 for ( cur = startbranch ; cur != scan ; cur = regnext( cur ) ) {
3299 regnode * const noper = NEXTOPER( cur );
3300 U8 noper_type = OP( noper );
3301 U8 noper_trietype = TRIE_TYPE( noper_type );
3302 #if defined(DEBUGGING) || defined(NOJUMPTRIE)
3303 regnode * const noper_next = regnext( noper );
3307 regprop(RExC_rx, mysv, cur);
3308 PerlIO_printf( Perl_debug_log, "%*s- %s (%d)",
3309 (int)depth * 2 + 2,"", SvPV_nolen_const( mysv ), REG_NODE_NUM(cur) );
3311 regprop(RExC_rx, mysv, noper);
3312 PerlIO_printf( Perl_debug_log, " -> %s",
3313 SvPV_nolen_const(mysv));
3316 regprop(RExC_rx, mysv, noper_next );
3317 PerlIO_printf( Perl_debug_log,"\t=> %s\t",
3318 SvPV_nolen_const(mysv));
3320 PerlIO_printf( Perl_debug_log, "(First==%d,Last==%d,Cur==%d)\n",
3321 REG_NODE_NUM(first), REG_NODE_NUM(last), REG_NODE_NUM(cur) );
3324 /* Is noper a trieable nodetype that can be merged with the
3325 * current trie (if there is one)? */
3329 /* XXX: Currently we cannot allow a NOTHING node to be the first element
3330 * of a TRIEABLE sequence, Otherwise we will overwrite the regop following
3331 * the NOTHING with the TRIE regop later on. This is because a NOTHING node
3332 * is only one regnode wide, and a TRIE is two regnodes. An example of a
3333 * problematic pattern is: "x" =~ /\A(?>(?:(?:)A|B|C?x))\z/
3334 * At a later point of time we can somewhat workaround this by handling
3335 * NOTHING -> EXACT sequences as generated by /(?:)A|(?:)B/ type patterns,
3336 * as we can effectively ignore the NOTHING regop in that case.
3337 * This clause, which allows NOTHING to start a sequence is left commented
3338 * out as a reference.
3341 ( noper_trietype == NOTHING)
3342 || ( trietype == NOTHING )
3344 ( noper_trietype == NOTHING && trietype )
3345 || ( trietype == noper_trietype )
3348 && noper_next == tail
3352 /* Handle mergable triable node
3353 * Either we are the first node in a new trieable sequence,
3354 * in which case we do some bookkeeping, otherwise we update
3355 * the end pointer. */
3359 trietype = noper_trietype;
3361 if ( trietype == NOTHING )
3362 trietype = noper_trietype;
3365 } /* end handle mergable triable node */
3367 /* handle unmergable node -
3368 * noper may either be a triable node which can not be tried
3369 * together with the current trie, or a non triable node */
3371 /* If last is set and trietype is not NOTHING then we have found
3372 * at least two triable branch sequences in a row of a similar
3373 * trietype so we can turn them into a trie. If/when we
3374 * allow NOTHING to start a trie sequence this condition will be
3375 * required, and it isn't expensive so we leave it in for now. */
3376 if ( trietype != NOTHING )
3377 make_trie( pRExC_state,
3378 startbranch, first, cur, tail, count,
3379 trietype, depth+1 );
3380 last = NULL; /* note: we clear/update first, trietype etc below, so we dont do it here */
3384 && noper_next == tail
3387 /* noper is triable, so we can start a new trie sequence */
3390 trietype = noper_trietype;
3392 /* if we already saw a first but the current node is not triable then we have
3393 * to reset the first information. */
3398 } /* end handle unmergable node */
3399 } /* loop over branches */
3401 regprop(RExC_rx, mysv, cur);
3402 PerlIO_printf( Perl_debug_log,
3403 "%*s- %s (%d) <SCAN FINISHED>\n", (int)depth * 2 + 2,
3404 "", SvPV_nolen_const( mysv ),REG_NODE_NUM(cur));
3407 if ( last && trietype != NOTHING ) {
3408 /* the last branch of the sequence was part of a trie,
3409 * so we have to construct it here outside of the loop
3411 made= make_trie( pRExC_state, startbranch, first, scan, tail, count, trietype, depth+1 );
3412 #ifdef TRIE_STUDY_OPT
3413 if ( ((made == MADE_EXACT_TRIE &&
3414 startbranch == first)
3415 || ( first_non_open == first )) &&
3417 flags |= SCF_TRIE_RESTUDY;
3418 if ( startbranch == first
3421 RExC_seen &=~REG_TOP_LEVEL_BRANCHES;
3425 } /* end if ( last) */
3426 } /* TRIE_MAXBUF is non zero */
3431 else if ( code == BRANCHJ ) { /* single branch is optimized. */
3432 scan = NEXTOPER(NEXTOPER(scan));
3433 } else /* single branch is optimized. */
3434 scan = NEXTOPER(scan);
3436 } else if (OP(scan) == SUSPEND || OP(scan) == GOSUB || OP(scan) == GOSTART) {
3437 scan_frame *newframe = NULL;
3442 if (OP(scan) != SUSPEND) {
3443 /* set the pointer */
3444 if (OP(scan) == GOSUB) {
3446 RExC_recurse[ARG2L(scan)] = scan;
3447 start = RExC_open_parens[paren-1];
3448 end = RExC_close_parens[paren-1];
3451 start = RExC_rxi->program + 1;
3455 Newxz(recursed, (((RExC_npar)>>3) +1), U8);
3456 SAVEFREEPV(recursed);
3458 if (!PAREN_TEST(recursed,paren+1)) {
3459 PAREN_SET(recursed,paren+1);
3460 Newx(newframe,1,scan_frame);
3462 if (flags & SCF_DO_SUBSTR) {
3463 SCAN_COMMIT(pRExC_state,data,minlenp);
3464 data->longest = &(data->longest_float);
3466 is_inf = is_inf_internal = 1;
3467 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */
3468 cl_anything(pRExC_state, data->start_class);
3469 flags &= ~SCF_DO_STCLASS;
3472 Newx(newframe,1,scan_frame);
3475 end = regnext(scan);
3480 SAVEFREEPV(newframe);
3481 newframe->next = regnext(scan);
3482 newframe->last = last;
3483 newframe->stop = stopparen;
3484 newframe->prev = frame;
3494 else if (OP(scan) == EXACT) {
3495 I32 l = STR_LEN(scan);
3498 const U8 * const s = (U8*)STRING(scan);
3499 uc = utf8_to_uvchr_buf(s, s + l, NULL);
3500 l = utf8_length(s, s + l);
3502 uc = *((U8*)STRING(scan));
3505 if (flags & SCF_DO_SUBSTR) { /* Update longest substr. */
3506 /* The code below prefers earlier match for fixed
3507 offset, later match for variable offset. */
3508 if (data->last_end == -1) { /* Update the start info. */
3509 data->last_start_min = data->pos_min;
3510 data->last_start_max = is_inf
3511 ? I32_MAX : data->pos_min + data->pos_delta;
3513 sv_catpvn(data->last_found, STRING(scan), STR_LEN(scan));
3515 SvUTF8_on(data->last_found);
3517 SV * const sv = data->last_found;
3518 MAGIC * const mg = SvUTF8(sv) && SvMAGICAL(sv) ?
3519 mg_find(sv, PERL_MAGIC_utf8) : NULL;
3520 if (mg && mg->mg_len >= 0)
3521 mg->mg_len += utf8_length((U8*)STRING(scan),
3522 (U8*)STRING(scan)+STR_LEN(scan));
3524 data->last_end = data->pos_min + l;
3525 data->pos_min += l; /* As in the first entry. */
3526 data->flags &= ~SF_BEFORE_EOL;
3528 if (flags & SCF_DO_STCLASS_AND) {
3529 /* Check whether it is compatible with what we know already! */
3533 /* If compatible, we or it in below. It is compatible if is
3534 * in the bitmp and either 1) its bit or its fold is set, or 2)
3535 * it's for a locale. Even if there isn't unicode semantics
3536 * here, at runtime there may be because of matching against a
3537 * utf8 string, so accept a possible false positive for
3538 * latin1-range folds */
3540 (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE))
3541 && !ANYOF_BITMAP_TEST(data->start_class, uc)
3542 && (!(data->start_class->flags & ANYOF_LOC_NONBITMAP_FOLD)
3543 || !ANYOF_BITMAP_TEST(data->start_class, PL_fold_latin1[uc])))
3548 ANYOF_CLASS_ZERO(data->start_class);
3549 ANYOF_BITMAP_ZERO(data->start_class);
3551 ANYOF_BITMAP_SET(data->start_class, uc);
3552 else if (uc >= 0x100) {
3555 /* Some Unicode code points fold to the Latin1 range; as
3556 * XXX temporary code, instead of figuring out if this is
3557 * one, just assume it is and set all the start class bits
3558 * that could be some such above 255 code point's fold
3559 * which will generate fals positives. As the code
3560 * elsewhere that does compute the fold settles down, it
3561 * can be extracted out and re-used here */
3562 for (i = 0; i < 256; i++){
3563 if (_HAS_NONLATIN1_FOLD_CLOSURE_ONLY_FOR_USE_BY_REGCOMP_DOT_C_AND_REGEXEC_DOT_C(i)) {
3564 ANYOF_BITMAP_SET(data->start_class, i);
3568 data->start_class->flags &= ~ANYOF_EOS;
3570 data->start_class->flags &= ~ANYOF_UNICODE_ALL;
3572 else if (flags & SCF_DO_STCLASS_OR) {
3573 /* false positive possible if the class is case-folded */
3575 ANYOF_BITMAP_SET(data->start_class, uc);
3577 data->start_class->flags |= ANYOF_UNICODE_ALL;
3578 data->start_class->flags &= ~ANYOF_EOS;
3579 cl_and(data->start_class, and_withp);
3581 flags &= ~SCF_DO_STCLASS;
3583 else if (PL_regkind[OP(scan)] == EXACT) { /* But OP != EXACT! */
3584 I32 l = STR_LEN(scan);
3585 UV uc = *((U8*)STRING(scan));
3587 /* Search for fixed substrings supports EXACT only. */
3588 if (flags & SCF_DO_SUBSTR) {
3590 SCAN_COMMIT(pRExC_state, data, minlenp);
3593 const U8 * const s = (U8 *)STRING(scan);
3594 uc = utf8_to_uvchr_buf(s, s + l, NULL);
3595 l = utf8_length(s, s + l);
3597 else if (has_exactf_sharp_s) {
3598 RExC_seen |= REG_SEEN_EXACTF_SHARP_S;
3600 min += l - min_subtract;
3604 delta += min_subtract;
3605 if (flags & SCF_DO_SUBSTR) {
3606 data->pos_min += l - min_subtract;
3607 if (data->pos_min < 0) {
3610 data->pos_delta += min_subtract;
3612 data->longest = &(data->longest_float);
3615 if (flags & SCF_DO_STCLASS_AND) {
3616 /* Check whether it is compatible with what we know already! */
3619 (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE))
3620 && !ANYOF_BITMAP_TEST(data->start_class, uc)
3621 && !ANYOF_BITMAP_TEST(data->start_class, PL_fold_latin1[uc])))
3625 ANYOF_CLASS_ZERO(data->start_class);
3626 ANYOF_BITMAP_ZERO(data->start_class);
3628 ANYOF_BITMAP_SET(data->start_class, uc);
3629 data->start_class->flags &= ~ANYOF_EOS;
3630 data->start_class->flags |= ANYOF_LOC_NONBITMAP_FOLD;
3631 if (OP(scan) == EXACTFL) {
3632 /* XXX This set is probably no longer necessary, and
3633 * probably wrong as LOCALE now is on in the initial
3635 data->start_class->flags |= ANYOF_LOCALE;
3639 /* Also set the other member of the fold pair. In case
3640 * that unicode semantics is called for at runtime, use
3641 * the full latin1 fold. (Can't do this for locale,
3642 * because not known until runtime) */
3643 ANYOF_BITMAP_SET(data->start_class, PL_fold_latin1[uc]);
3645 /* All other (EXACTFL handled above) folds except under
3646 * /iaa that include s, S, and sharp_s also may include
3648 if (OP(scan) != EXACTFA) {
3649 if (uc == 's' || uc == 'S') {
3650 ANYOF_BITMAP_SET(data->start_class,
3651 LATIN_SMALL_LETTER_SHARP_S);
3653 else if (uc == LATIN_SMALL_LETTER_SHARP_S) {
3654 ANYOF_BITMAP_SET(data->start_class, 's');
3655 ANYOF_BITMAP_SET(data->start_class, 'S');
3660 else if (uc >= 0x100) {
3662 for (i = 0; i < 256; i++){
3663 if (_HAS_NONLATIN1_FOLD_CLOSURE_ONLY_FOR_USE_BY_REGCOMP_DOT_C_AND_REGEXEC_DOT_C(i)) {
3664 ANYOF_BITMAP_SET(data->start_class, i);
3669 else if (flags & SCF_DO_STCLASS_OR) {
3670 if (data->start_class->flags & ANYOF_LOC_NONBITMAP_FOLD) {
3671 /* false positive possible if the class is case-folded.
3672 Assume that the locale settings are the same... */
3674 ANYOF_BITMAP_SET(data->start_class, uc);
3675 if (OP(scan) != EXACTFL) {
3677 /* And set the other member of the fold pair, but
3678 * can't do that in locale because not known until
3680 ANYOF_BITMAP_SET(data->start_class,
3681 PL_fold_latin1[uc]);
3683 /* All folds except under /iaa that include s, S,
3684 * and sharp_s also may include the others */
3685 if (OP(scan) != EXACTFA) {
3686 if (uc == 's' || uc == 'S') {
3687 ANYOF_BITMAP_SET(data->start_class,
3688 LATIN_SMALL_LETTER_SHARP_S);
3690 else if (uc == LATIN_SMALL_LETTER_SHARP_S) {
3691 ANYOF_BITMAP_SET(data->start_class, 's');
3692 ANYOF_BITMAP_SET(data->start_class, 'S');
3697 data->start_class->flags &= ~ANYOF_EOS;
3699 cl_and(data->start_class, and_withp);
3701 flags &= ~SCF_DO_STCLASS;
3703 else if (REGNODE_VARIES(OP(scan))) {
3704 I32 mincount, maxcount, minnext, deltanext, fl = 0;
3705 I32 f = flags, pos_before = 0;
3706 regnode * const oscan = scan;
3707 struct regnode_charclass_class this_class;
3708 struct regnode_charclass_class *oclass = NULL;
3709 I32 next_is_eval = 0;
3711 switch (PL_regkind[OP(scan)]) {
3712 case WHILEM: /* End of (?:...)* . */
3713 scan = NEXTOPER(scan);
3716 if (flags & (SCF_DO_SUBSTR | SCF_DO_STCLASS)) {
3717 next = NEXTOPER(scan);
3718 if (OP(next) == EXACT || (flags & SCF_DO_STCLASS)) {
3720 maxcount = REG_INFTY;
3721 next = regnext(scan);
3722 scan = NEXTOPER(scan);
3726 if (flags & SCF_DO_SUBSTR)
3731 if (flags & SCF_DO_STCLASS) {
3733 maxcount = REG_INFTY;
3734 next = regnext(scan);
3735 scan = NEXTOPER(scan);
3738 is_inf = is_inf_internal = 1;
3739 scan = regnext(scan);
3740 if (flags & SCF_DO_SUBSTR) {
3741 SCAN_COMMIT(pRExC_state, data, minlenp); /* Cannot extend fixed substrings */
3742 data->longest = &(data->longest_float);
3744 goto optimize_curly_tail;
3746 if (stopparen>0 && (OP(scan)==CURLYN || OP(scan)==CURLYM)
3747 && (scan->flags == stopparen))
3752 mincount = ARG1(scan);
3753 maxcount = ARG2(scan);
3755 next = regnext(scan);
3756 if (OP(scan) == CURLYX) {
3757 I32 lp = (data ? *(data->last_closep) : 0);
3758 scan->flags = ((lp <= (I32)U8_MAX) ? (U8)lp : U8_MAX);
3760 scan = NEXTOPER(scan) + EXTRA_STEP_2ARGS;
3761 next_is_eval = (OP(scan) == EVAL);
3763 if (flags & SCF_DO_SUBSTR) {
3764 if (mincount == 0) SCAN_COMMIT(pRExC_state,data,minlenp); /* Cannot extend fixed substrings */
3765 pos_before = data->pos_min;
3769 data->flags &= ~(SF_HAS_PAR|SF_IN_PAR|SF_HAS_EVAL);
3771 data->flags |= SF_IS_INF;
3773 if (flags & SCF_DO_STCLASS) {
3774 cl_init(pRExC_state, &this_class);
3775 oclass = data->start_class;
3776 data->start_class = &this_class;
3777 f |= SCF_DO_STCLASS_AND;
3778 f &= ~SCF_DO_STCLASS_OR;
3780 /* Exclude from super-linear cache processing any {n,m}
3781 regops for which the combination of input pos and regex
3782 pos is not enough information to determine if a match
3785 For example, in the regex /foo(bar\s*){4,8}baz/ with the
3786 regex pos at the \s*, the prospects for a match depend not
3787 only on the input position but also on how many (bar\s*)
3788 repeats into the {4,8} we are. */
3789 if ((mincount > 1) || (maxcount > 1 && maxcount != REG_INFTY))
3790 f &= ~SCF_WHILEM_VISITED_POS;
3792 /* This will finish on WHILEM, setting scan, or on NULL: */
3793 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext,
3794 last, data, stopparen, recursed, NULL,
3796 ? (f & ~SCF_DO_SUBSTR) : f),depth+1);
3798 if (flags & SCF_DO_STCLASS)
3799 data->start_class = oclass;
3800 if (mincount == 0 || minnext == 0) {
3801 if (flags & SCF_DO_STCLASS_OR) {
3802 cl_or(pRExC_state, data->start_class, &this_class);
3804 else if (flags & SCF_DO_STCLASS_AND) {
3805 /* Switch to OR mode: cache the old value of
3806 * data->start_class */
3808 StructCopy(data->start_class, and_withp,
3809 struct regnode_charclass_class);
3810 flags &= ~SCF_DO_STCLASS_AND;
3811 StructCopy(&this_class, data->start_class,
3812 struct regnode_charclass_class);
3813 flags |= SCF_DO_STCLASS_OR;
3814 data->start_class->flags |= ANYOF_EOS;
3816 } else { /* Non-zero len */
3817 if (flags & SCF_DO_STCLASS_OR) {
3818 cl_or(pRExC_state, data->start_class, &this_class);
3819 cl_and(data->start_class, and_withp);
3821 else if (flags & SCF_DO_STCLASS_AND)
3822 cl_and(data->start_class, &this_class);
3823 flags &= ~SCF_DO_STCLASS;
3825 if (!scan) /* It was not CURLYX, but CURLY. */
3827 if ( /* ? quantifier ok, except for (?{ ... }) */
3828 (next_is_eval || !(mincount == 0 && maxcount == 1))
3829 && (minnext == 0) && (deltanext == 0)
3830 && data && !(data->flags & (SF_HAS_PAR|SF_IN_PAR))
3831 && maxcount <= REG_INFTY/3) /* Complement check for big count */
3833 ckWARNreg(RExC_parse,
3834 "Quantifier unexpected on zero-length expression");
3837 min += minnext * mincount;
3838 is_inf_internal |= ((maxcount == REG_INFTY
3839 && (minnext + deltanext) > 0)
3840 || deltanext == I32_MAX);
3841 is_inf |= is_inf_internal;
3842 delta += (minnext + deltanext) * maxcount - minnext * mincount;
3844 /* Try powerful optimization CURLYX => CURLYN. */
3845 if ( OP(oscan) == CURLYX && data
3846 && data->flags & SF_IN_PAR
3847 && !(data->flags & SF_HAS_EVAL)
3848 && !deltanext && minnext == 1 ) {
3849 /* Try to optimize to CURLYN. */
3850 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS;
3851 regnode * const nxt1 = nxt;
3858 if (!REGNODE_SIMPLE(OP(nxt))
3859 && !(PL_regkind[OP(nxt)] == EXACT
3860 && STR_LEN(nxt) == 1))
3866 if (OP(nxt) != CLOSE)
3868 if (RExC_open_parens) {
3869 RExC_open_parens[ARG(nxt1)-1]=oscan; /*open->CURLYM*/
3870 RExC_close_parens[ARG(nxt1)-1]=nxt+2; /*close->while*/
3872 /* Now we know that nxt2 is the only contents: */
3873 oscan->flags = (U8)ARG(nxt);
3875 OP(nxt1) = NOTHING; /* was OPEN. */
3878 OP(nxt1 + 1) = OPTIMIZED; /* was count. */
3879 NEXT_OFF(nxt1+ 1) = 0; /* just for consistency. */
3880 NEXT_OFF(nxt2) = 0; /* just for consistency with CURLY. */
3881 OP(nxt) = OPTIMIZED; /* was CLOSE. */
3882 OP(nxt + 1) = OPTIMIZED; /* was count. */
3883 NEXT_OFF(nxt+ 1) = 0; /* just for consistency. */
3888 /* Try optimization CURLYX => CURLYM. */
3889 if ( OP(oscan) == CURLYX && data
3890 && !(data->flags & SF_HAS_PAR)
3891 && !(data->flags & SF_HAS_EVAL)
3892 && !deltanext /* atom is fixed width */
3893 && minnext != 0 /* CURLYM can't handle zero width */
3895 /* XXXX How to optimize if data == 0? */
3896 /* Optimize to a simpler form. */
3897 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN */
3901 while ( (nxt2 = regnext(nxt)) /* skip over embedded stuff*/
3902 && (OP(nxt2) != WHILEM))
3904 OP(nxt2) = SUCCEED; /* Whas WHILEM */
3905 /* Need to optimize away parenths. */
3906 if ((data->flags & SF_IN_PAR) && OP(nxt) == CLOSE) {
3907 /* Set the parenth number. */
3908 regnode *nxt1 = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN*/
3910 oscan->flags = (U8)ARG(nxt);
3911 if (RExC_open_parens) {
3912 RExC_open_parens[ARG(nxt1)-1]=oscan; /*open->CURLYM*/
3913 RExC_close_parens[ARG(nxt1)-1]=nxt2+1; /*close->NOTHING*/
3915 OP(nxt1) = OPTIMIZED; /* was OPEN. */
3916 OP(nxt) = OPTIMIZED; /* was CLOSE. */
3919 OP(nxt1 + 1) = OPTIMIZED; /* was count. */
3920 OP(nxt + 1) = OPTIMIZED; /* was count. */
3921 NEXT_OFF(nxt1 + 1) = 0; /* just for consistency. */
3922 NEXT_OFF(nxt + 1) = 0; /* just for consistency. */
3925 while ( nxt1 && (OP(nxt1) != WHILEM)) {
3926 regnode *nnxt = regnext(nxt1);
3928 if (reg_off_by_arg[OP(nxt1)])
3929 ARG_SET(nxt1, nxt2 - nxt1);
3930 else if (nxt2 - nxt1 < U16_MAX)
3931 NEXT_OFF(nxt1) = nxt2 - nxt1;
3933 OP(nxt) = NOTHING; /* Cannot beautify */
3938 /* Optimize again: */
3939 study_chunk(pRExC_state, &nxt1, minlenp, &deltanext, nxt,
3940 NULL, stopparen, recursed, NULL, 0,depth+1);
3945 else if ((OP(oscan) == CURLYX)
3946 && (flags & SCF_WHILEM_VISITED_POS)
3947 /* See the comment on a similar expression above.
3948 However, this time it's not a subexpression
3949 we care about, but the expression itself. */
3950 && (maxcount == REG_INFTY)
3951 && data && ++data->whilem_c < 16) {
3952 /* This stays as CURLYX, we can put the count/of pair. */
3953 /* Find WHILEM (as in regexec.c) */
3954 regnode *nxt = oscan + NEXT_OFF(oscan);
3956 if (OP(PREVOPER(nxt)) == NOTHING) /* LONGJMP */
3958 PREVOPER(nxt)->flags = (U8)(data->whilem_c
3959 | (RExC_whilem_seen << 4)); /* On WHILEM */
3961 if (data && fl & (SF_HAS_PAR|SF_IN_PAR))
3963 if (flags & SCF_DO_SUBSTR) {
3964 SV *last_str = NULL;
3965 int counted = mincount != 0;
3967 if (data->last_end > 0 && mincount != 0) { /* Ends with a string. */
3968 #if defined(SPARC64_GCC_WORKAROUND)
3971 const char *s = NULL;
3974 if (pos_before >= data->last_start_min)
3977 b = data->last_start_min;
3980 s = SvPV_const(data->last_found, l);
3981 old = b - data->last_start_min;
3984 I32 b = pos_before >= data->last_start_min
3985 ? pos_before : data->last_start_min;
3987 const char * const s = SvPV_const(data->last_found, l);
3988 I32 old = b - data->last_start_min;
3992 old = utf8_hop((U8*)s, old) - (U8*)s;
3994 /* Get the added string: */
3995 last_str = newSVpvn_utf8(s + old, l, UTF);
3996 if (deltanext == 0 && pos_before == b) {
3997 /* What was added is a constant string */
3999 SvGROW(last_str, (mincount * l) + 1);
4000 repeatcpy(SvPVX(last_str) + l,
4001 SvPVX_const(last_str), l, mincount - 1);
4002 SvCUR_set(last_str, SvCUR(last_str) * mincount);
4003 /* Add additional parts. */
4004 SvCUR_set(data->last_found,
4005 SvCUR(data->last_found) - l);
4006 sv_catsv(data->last_found, last_str);
4008 SV * sv = data->last_found;
4010 SvUTF8(sv) && SvMAGICAL(sv) ?
4011 mg_find(sv, PERL_MAGIC_utf8) : NULL;
4012 if (mg && mg->mg_len >= 0)
4013 mg->mg_len += CHR_SVLEN(last_str) - l;
4015 data->last_end += l * (mincount - 1);
4018 /* start offset must point into the last copy */
4019 data->last_start_min += minnext * (mincount - 1);
4020 data->last_start_max += is_inf ? I32_MAX
4021 : (maxcount - 1) * (minnext + data->pos_delta);
4024 /* It is counted once already... */
4025 data->pos_min += minnext * (mincount - counted);
4026 data->pos_delta += - counted * deltanext +
4027 (minnext + deltanext) * maxcount - minnext * mincount;
4028 if (mincount != maxcount) {
4029 /* Cannot extend fixed substrings found inside
4031 SCAN_COMMIT(pRExC_state,data,minlenp);
4032 if (mincount && last_str) {
4033 SV * const sv = data->last_found;
4034 MAGIC * const mg = SvUTF8(sv) && SvMAGICAL(sv) ?
4035 mg_find(sv, PERL_MAGIC_utf8) : NULL;
4039 sv_setsv(sv, last_str);
4040 data->last_end = data->pos_min;
4041 data->last_start_min =
4042 data->pos_min - CHR_SVLEN(last_str);
4043 data->last_start_max = is_inf
4045 : data->pos_min + data->pos_delta
4046 - CHR_SVLEN(last_str);
4048 data->longest = &(data->longest_float);
4050 SvREFCNT_dec(last_str);
4052 if (data && (fl & SF_HAS_EVAL))
4053 data->flags |= SF_HAS_EVAL;
4054 optimize_curly_tail:
4055 if (OP(oscan) != CURLYX) {
4056 while (PL_regkind[OP(next = regnext(oscan))] == NOTHING
4058 NEXT_OFF(oscan) += NEXT_OFF(next);
4061 default: /* REF, ANYOFV, and CLUMP only? */
4062 if (flags & SCF_DO_SUBSTR) {
4063 SCAN_COMMIT(pRExC_state,data,minlenp); /* Cannot expect anything... */
4064 data->longest = &(data->longest_float);
4066 is_inf = is_inf_internal = 1;
4067 if (flags & SCF_DO_STCLASS_OR)
4068 cl_anything(pRExC_state, data->start_class);
4069 flags &= ~SCF_DO_STCLASS;
4073 else if (OP(scan) == LNBREAK) {
4074 if (flags & SCF_DO_STCLASS) {
4076 data->start_class->flags &= ~ANYOF_EOS; /* No match on empty */
4077 if (flags & SCF_DO_STCLASS_AND) {
4078 for (value = 0; value < 256; value++)
4079 if (!is_VERTWS_cp(value))
4080 ANYOF_BITMAP_CLEAR(data->start_class, value);
4083 for (value = 0; value < 256; value++)
4084 if (is_VERTWS_cp(value))
4085 ANYOF_BITMAP_SET(data->start_class, value);
4087 if (flags & SCF_DO_STCLASS_OR)
4088 cl_and(data->start_class, and_withp);
4089 flags &= ~SCF_DO_STCLASS;
4093 if (flags & SCF_DO_SUBSTR) {
4094 SCAN_COMMIT(pRExC_state,data,minlenp); /* Cannot expect anything... */
4096 data->pos_delta += 1;
4097 data->longest = &(data->longest_float);
4100 else if (REGNODE_SIMPLE(OP(scan))) {
4103 if (flags & SCF_DO_SUBSTR) {
4104 SCAN_COMMIT(pRExC_state,data,minlenp);
4108 if (flags & SCF_DO_STCLASS) {
4109 data->start_class->flags &= ~ANYOF_EOS; /* No match on empty */
4111 /* Some of the logic below assumes that switching
4112 locale on will only add false positives. */
4113 switch (PL_regkind[OP(scan)]) {
4117 /* Perl_croak(aTHX_ "panic: unexpected simple REx opcode %d", OP(scan)); */
4118 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */
4119 cl_anything(pRExC_state, data->start_class);
4122 if (OP(scan) == SANY)
4124 if (flags & SCF_DO_STCLASS_OR) { /* Everything but \n */
4125 value = (ANYOF_BITMAP_TEST(data->start_class,'\n')
4126 || ANYOF_CLASS_TEST_ANY_SET(data->start_class));
4127 cl_anything(pRExC_state, data->start_class);
4129 if (flags & SCF_DO_STCLASS_AND || !value)
4130 ANYOF_BITMAP_CLEAR(data->start_class,'\n');
4133 if (flags & SCF_DO_STCLASS_AND)
4134 cl_and(data->start_class,
4135 (struct regnode_charclass_class*)scan);
4137 cl_or(pRExC_state, data->start_class,
4138 (struct regnode_charclass_class*)scan);
4141 if (flags & SCF_DO_STCLASS_AND) {
4142 if (!(data->start_class->flags & ANYOF_LOCALE)) {
4143 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NALNUM);
4144 if (OP(scan) == ALNUMU) {
4145 for (value = 0; value < 256; value++) {
4146 if (!isWORDCHAR_L1(value)) {
4147 ANYOF_BITMAP_CLEAR(data->start_class, value);
4151 for (value = 0; value < 256; value++) {
4152 if (!isALNUM(value)) {
4153 ANYOF_BITMAP_CLEAR(data->start_class, value);
4160 if (data->start_class->flags & ANYOF_LOCALE)
4161 ANYOF_CLASS_SET(data->start_class,ANYOF_ALNUM);
4163 /* Even if under locale, set the bits for non-locale
4164 * in case it isn't a true locale-node. This will
4165 * create false positives if it truly is locale */
4166 if (OP(scan) == ALNUMU) {
4167 for (value = 0; value < 256; value++) {
4168 if (isWORDCHAR_L1(value)) {
4169 ANYOF_BITMAP_SET(data->start_class, value);
4173 for (value = 0; value < 256; value++) {
4174 if (isALNUM(value)) {
4175 ANYOF_BITMAP_SET(data->start_class, value);
4182 if (flags & SCF_DO_STCLASS_AND) {
4183 if (!(data->start_class->flags & ANYOF_LOCALE)) {
4184 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_ALNUM);
4185 if (OP(scan) == NALNUMU) {
4186 for (value = 0; value < 256; value++) {
4187 if (isWORDCHAR_L1(value)) {
4188 ANYOF_BITMAP_CLEAR(data->start_class, value);
4192 for (value = 0; value < 256; value++) {
4193 if (isALNUM(value)) {
4194 ANYOF_BITMAP_CLEAR(data->start_class, value);
4201 if (data->start_class->flags & ANYOF_LOCALE)
4202 ANYOF_CLASS_SET(data->start_class,ANYOF_NALNUM);
4204 /* Even if under locale, set the bits for non-locale in
4205 * case it isn't a true locale-node. This will create
4206 * false positives if it truly is locale */
4207 if (OP(scan) == NALNUMU) {
4208 for (value = 0; value < 256; value++) {
4209 if (! isWORDCHAR_L1(value)) {
4210 ANYOF_BITMAP_SET(data->start_class, value);
4214 for (value = 0; value < 256; value++) {
4215 if (! isALNUM(value)) {
4216 ANYOF_BITMAP_SET(data->start_class, value);
4223 if (flags & SCF_DO_STCLASS_AND) {
4224 if (!(data->start_class->flags & ANYOF_LOCALE)) {
4225 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NSPACE);
4226 if (OP(scan) == SPACEU) {
4227 for (value = 0; value < 256; value++) {
4228 if (!isSPACE_L1(value)) {
4229 ANYOF_BITMAP_CLEAR(data->start_class, value);
4233 for (value = 0; value < 256; value++) {
4234 if (!isSPACE(value)) {
4235 ANYOF_BITMAP_CLEAR(data->start_class, value);
4242 if (data->start_class->flags & ANYOF_LOCALE) {
4243 ANYOF_CLASS_SET(data->start_class,ANYOF_SPACE);
4245 if (OP(scan) == SPACEU) {
4246 for (value = 0; value < 256; value++) {
4247 if (isSPACE_L1(value)) {
4248 ANYOF_BITMAP_SET(data->start_class, value);
4252 for (value = 0; value < 256; value++) {
4253 if (isSPACE(value)) {
4254 ANYOF_BITMAP_SET(data->start_class, value);
4261 if (flags & SCF_DO_STCLASS_AND) {
4262 if (!(data->start_class->flags & ANYOF_LOCALE)) {
4263 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_SPACE);
4264 if (OP(scan) == NSPACEU) {
4265 for (value = 0; value < 256; value++) {
4266 if (isSPACE_L1(value)) {
4267 ANYOF_BITMAP_CLEAR(data->start_class, value);
4271 for (value = 0; value < 256; value++) {
4272 if (isSPACE(value)) {
4273 ANYOF_BITMAP_CLEAR(data->start_class, value);
4280 if (data->start_class->flags & ANYOF_LOCALE)
4281 ANYOF_CLASS_SET(data->start_class,ANYOF_NSPACE);
4282 if (OP(scan) == NSPACEU) {
4283 for (value = 0; value < 256; value++) {
4284 if (!isSPACE_L1(value)) {
4285 ANYOF_BITMAP_SET(data->start_class, value);
4290 for (value = 0; value < 256; value++) {
4291 if (!isSPACE(value)) {
4292 ANYOF_BITMAP_SET(data->start_class, value);
4299 if (flags & SCF_DO_STCLASS_AND) {
4300 if (!(data->start_class->flags & ANYOF_LOCALE)) {
4301 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NDIGIT);
4302 for (value = 0; value < 256; value++)
4303 if (!isDIGIT(value))
4304 ANYOF_BITMAP_CLEAR(data->start_class, value);
4308 if (data->start_class->flags & ANYOF_LOCALE)
4309 ANYOF_CLASS_SET(data->start_class,ANYOF_DIGIT);
4310 for (value = 0; value < 256; value++)
4312 ANYOF_BITMAP_SET(data->start_class, value);
4316 if (flags & SCF_DO_STCLASS_AND) {
4317 if (!(data->start_class->flags & ANYOF_LOCALE))
4318 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_DIGIT);
4319 for (value = 0; value < 256; value++)
4321 ANYOF_BITMAP_CLEAR(data->start_class, value);
4324 if (data->start_class->flags & ANYOF_LOCALE)
4325 ANYOF_CLASS_SET(data->start_class,ANYOF_NDIGIT);
4326 for (value = 0; value < 256; value++)
4327 if (!isDIGIT(value))
4328 ANYOF_BITMAP_SET(data->start_class, value);
4331 CASE_SYNST_FNC(VERTWS);
4332 CASE_SYNST_FNC(HORIZWS);
4335 if (flags & SCF_DO_STCLASS_OR)
4336 cl_and(data->start_class, and_withp);
4337 flags &= ~SCF_DO_STCLASS;
4340 else if (PL_regkind[OP(scan)] == EOL && flags & SCF_DO_SUBSTR) {
4341 data->flags |= (OP(scan) == MEOL
4345 else if ( PL_regkind[OP(scan)] == BRANCHJ
4346 /* Lookbehind, or need to calculate parens/evals/stclass: */
4347 && (scan->flags || data || (flags & SCF_DO_STCLASS))
4348 && (OP(scan) == IFMATCH || OP(scan) == UNLESSM)) {
4349 if ( !PERL_ENABLE_POSITIVE_ASSERTION_STUDY
4350 || OP(scan) == UNLESSM )
4352 /* Negative Lookahead/lookbehind
4353 In this case we can't do fixed string optimisation.
4356 I32 deltanext, minnext, fake = 0;
4358 struct regnode_charclass_class intrnl;
4361 data_fake.flags = 0;
4363 data_fake.whilem_c = data->whilem_c;
4364 data_fake.last_closep = data->last_closep;
4367 data_fake.last_closep = &fake;
4368 data_fake.pos_delta = delta;
4369 if ( flags & SCF_DO_STCLASS && !scan->flags
4370 && OP(scan) == IFMATCH ) { /* Lookahead */
4371 cl_init(pRExC_state, &intrnl);
4372 data_fake.start_class = &intrnl;
4373 f |= SCF_DO_STCLASS_AND;
4375 if (flags & SCF_WHILEM_VISITED_POS)
4376 f |= SCF_WHILEM_VISITED_POS;
4377 next = regnext(scan);
4378 nscan = NEXTOPER(NEXTOPER(scan));
4379 minnext = study_chunk(pRExC_state, &nscan, minlenp, &deltanext,
4380 last, &data_fake, stopparen, recursed, NULL, f, depth+1);
4383 FAIL("Variable length lookbehind not implemented");
4385 else if (minnext > (I32)U8_MAX) {
4386 FAIL2("Lookbehind longer than %"UVuf" not implemented", (UV)U8_MAX);
4388 scan->flags = (U8)minnext;
4391 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
4393 if (data_fake.flags & SF_HAS_EVAL)
4394 data->flags |= SF_HAS_EVAL;
4395 data->whilem_c = data_fake.whilem_c;
4397 if (f & SCF_DO_STCLASS_AND) {
4398 if (flags & SCF_DO_STCLASS_OR) {
4399 /* OR before, AND after: ideally we would recurse with
4400 * data_fake to get the AND applied by study of the
4401 * remainder of the pattern, and then derecurse;
4402 * *** HACK *** for now just treat as "no information".
4403 * See [perl #56690].
4405 cl_init(pRExC_state, data->start_class);
4407 /* AND before and after: combine and continue */
4408 const int was = (data->start_class->flags & ANYOF_EOS);
4410 cl_and(data->start_class, &intrnl);
4412 data->start_class->flags |= ANYOF_EOS;
4416 #if PERL_ENABLE_POSITIVE_ASSERTION_STUDY
4418 /* Positive Lookahead/lookbehind
4419 In this case we can do fixed string optimisation,
4420 but we must be careful about it. Note in the case of
4421 lookbehind the positions will be offset by the minimum
4422 length of the pattern, something we won't know about
4423 until after the recurse.
4425 I32 deltanext, fake = 0;
4427 struct regnode_charclass_class intrnl;
4429 /* We use SAVEFREEPV so that when the full compile
4430 is finished perl will clean up the allocated
4431 minlens when it's all done. This way we don't
4432 have to worry about freeing them when we know
4433 they wont be used, which would be a pain.
4436 Newx( minnextp, 1, I32 );
4437 SAVEFREEPV(minnextp);
4440 StructCopy(data, &data_fake, scan_data_t);
4441 if ((flags & SCF_DO_SUBSTR) && data->last_found) {
4444 SCAN_COMMIT(pRExC_state, &data_fake,minlenp);
4445 data_fake.last_found=newSVsv(data->last_found);
4449 data_fake.last_closep = &fake;
4450 data_fake.flags = 0;
4451 data_fake.pos_delta = delta;
4453 data_fake.flags |= SF_IS_INF;
4454 if ( flags & SCF_DO_STCLASS && !scan->flags
4455 && OP(scan) == IFMATCH ) { /* Lookahead */
4456 cl_init(pRExC_state, &intrnl);
4457 data_fake.start_class = &intrnl;
4458 f |= SCF_DO_STCLASS_AND;
4460 if (flags & SCF_WHILEM_VISITED_POS)
4461 f |= SCF_WHILEM_VISITED_POS;
4462 next = regnext(scan);
4463 nscan = NEXTOPER(NEXTOPER(scan));
4465 *minnextp = study_chunk(pRExC_state, &nscan, minnextp, &deltanext,
4466 last, &data_fake, stopparen, recursed, NULL, f,depth+1);
4469 FAIL("Variable length lookbehind not implemented");
4471 else if (*minnextp > (I32)U8_MAX) {
4472 FAIL2("Lookbehind longer than %"UVuf" not implemented", (UV)U8_MAX);
4474 scan->flags = (U8)*minnextp;
4479 if (f & SCF_DO_STCLASS_AND) {
4480 const int was = (data->start_class->flags & ANYOF_EOS);
4482 cl_and(data->start_class, &intrnl);
4484 data->start_class->flags |= ANYOF_EOS;
4487 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
4489 if (data_fake.flags & SF_HAS_EVAL)
4490 data->flags |= SF_HAS_EVAL;
4491 data->whilem_c = data_fake.whilem_c;
4492 if ((flags & SCF_DO_SUBSTR) && data_fake.last_found) {
4493 if (RExC_rx->minlen<*minnextp)
4494 RExC_rx->minlen=*minnextp;
4495 SCAN_COMMIT(pRExC_state, &data_fake, minnextp);
4496 SvREFCNT_dec(data_fake.last_found);
4498 if ( data_fake.minlen_fixed != minlenp )
4500 data->offset_fixed= data_fake.offset_fixed;
4501 data->minlen_fixed= data_fake.minlen_fixed;
4502 data->lookbehind_fixed+= scan->flags;
4504 if ( data_fake.minlen_float != minlenp )
4506 data->minlen_float= data_fake.minlen_float;
4507 data->offset_float_min=data_fake.offset_float_min;
4508 data->offset_float_max=data_fake.offset_float_max;
4509 data->lookbehind_float+= scan->flags;
4518 else if (OP(scan) == OPEN) {
4519 if (stopparen != (I32)ARG(scan))
4522 else if (OP(scan) == CLOSE) {
4523 if (stopparen == (I32)ARG(scan)) {
4526 if ((I32)ARG(scan) == is_par) {
4527 next = regnext(scan);
4529 if ( next && (OP(next) != WHILEM) && next < last)
4530 is_par = 0; /* Disable optimization */
4533 *(data->last_closep) = ARG(scan);
4535 else if (OP(scan) == EVAL) {
4537 data->flags |= SF_HAS_EVAL;
4539 else if ( PL_regkind[OP(scan)] == ENDLIKE ) {
4540 if (flags & SCF_DO_SUBSTR) {
4541 SCAN_COMMIT(pRExC_state,data,minlenp);
4542 flags &= ~SCF_DO_SUBSTR;
4544 if (data && OP(scan)==ACCEPT) {
4545 data->flags |= SCF_SEEN_ACCEPT;
4550 else if (OP(scan) == LOGICAL && scan->flags == 2) /* Embedded follows */
4552 if (flags & SCF_DO_SUBSTR) {
4553 SCAN_COMMIT(pRExC_state,data,minlenp);
4554 data->longest = &(data->longest_float);
4556 is_inf = is_inf_internal = 1;
4557 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */
4558 cl_anything(pRExC_state, data->start_class);
4559 flags &= ~SCF_DO_STCLASS;
4561 else if (OP(scan) == GPOS) {
4562 if (!(RExC_rx->extflags & RXf_GPOS_FLOAT) &&
4563 !(delta || is_inf || (data && data->pos_delta)))
4565 if (!(RExC_rx->extflags & RXf_ANCH) && (flags & SCF_DO_SUBSTR))
4566 RExC_rx->extflags |= RXf_ANCH_GPOS;
4567 if (RExC_rx->gofs < (U32)min)
4568 RExC_rx->gofs = min;
4570 RExC_rx->extflags |= RXf_GPOS_FLOAT;
4574 #ifdef TRIE_STUDY_OPT
4575 #ifdef FULL_TRIE_STUDY
4576 else if (PL_regkind[OP(scan)] == TRIE) {
4577 /* NOTE - There is similar code to this block above for handling
4578 BRANCH nodes on the initial study. If you change stuff here
4580 regnode *trie_node= scan;
4581 regnode *tail= regnext(scan);
4582 reg_trie_data *trie = (reg_trie_data*)RExC_rxi->data->data[ ARG(scan) ];
4583 I32 max1 = 0, min1 = I32_MAX;
4584 struct regnode_charclass_class accum;
4586 if (flags & SCF_DO_SUBSTR) /* XXXX Add !SUSPEND? */
4587 SCAN_COMMIT(pRExC_state, data,minlenp); /* Cannot merge strings after this. */
4588 if (flags & SCF_DO_STCLASS)
4589 cl_init_zero(pRExC_state, &accum);
4595 const regnode *nextbranch= NULL;
4598 for ( word=1 ; word <= trie->wordcount ; word++)
4600 I32 deltanext=0, minnext=0, f = 0, fake;
4601 struct regnode_charclass_class this_class;
4603 data_fake.flags = 0;
4605 data_fake.whilem_c = data->whilem_c;
4606 data_fake.last_closep = data->last_closep;
4609 data_fake.last_closep = &fake;
4610 data_fake.pos_delta = delta;
4611 if (flags & SCF_DO_STCLASS) {
4612 cl_init(pRExC_state, &this_class);
4613 data_fake.start_class = &this_class;
4614 f = SCF_DO_STCLASS_AND;
4616 if (flags & SCF_WHILEM_VISITED_POS)
4617 f |= SCF_WHILEM_VISITED_POS;
4619 if (trie->jump[word]) {
4621 nextbranch = trie_node + trie->jump[0];
4622 scan= trie_node + trie->jump[word];
4623 /* We go from the jump point to the branch that follows
4624 it. Note this means we need the vestigal unused branches
4625 even though they arent otherwise used.
4627 minnext = study_chunk(pRExC_state, &scan, minlenp,
4628 &deltanext, (regnode *)nextbranch, &data_fake,
4629 stopparen, recursed, NULL, f,depth+1);
4631 if (nextbranch && PL_regkind[OP(nextbranch)]==BRANCH)
4632 nextbranch= regnext((regnode*)nextbranch);
4634 if (min1 > (I32)(minnext + trie->minlen))
4635 min1 = minnext + trie->minlen;
4636 if (max1 < (I32)(minnext + deltanext + trie->maxlen))
4637 max1 = minnext + deltanext + trie->maxlen;
4638 if (deltanext == I32_MAX)
4639 is_inf = is_inf_internal = 1;
4641 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
4643 if (data_fake.flags & SCF_SEEN_ACCEPT) {
4644 if ( stopmin > min + min1)
4645 stopmin = min + min1;
4646 flags &= ~SCF_DO_SUBSTR;
4648 data->flags |= SCF_SEEN_ACCEPT;
4651 if (data_fake.flags & SF_HAS_EVAL)
4652 data->flags |= SF_HAS_EVAL;
4653 data->whilem_c = data_fake.whilem_c;
4655 if (flags & SCF_DO_STCLASS)
4656 cl_or(pRExC_state, &accum, &this_class);
4659 if (flags & SCF_DO_SUBSTR) {
4660 data->pos_min += min1;
4661 data->pos_delta += max1 - min1;
4662 if (max1 != min1 || is_inf)
4663 data->longest = &(data->longest_float);
4666 delta += max1 - min1;
4667 if (flags & SCF_DO_STCLASS_OR) {
4668 cl_or(pRExC_state, data->start_class, &accum);
4670 cl_and(data->start_class, and_withp);
4671 flags &= ~SCF_DO_STCLASS;
4674 else if (flags & SCF_DO_STCLASS_AND) {
4676 cl_and(data->start_class, &accum);
4677 flags &= ~SCF_DO_STCLASS;
4680 /* Switch to OR mode: cache the old value of
4681 * data->start_class */
4683 StructCopy(data->start_class, and_withp,
4684 struct regnode_charclass_class);
4685 flags &= ~SCF_DO_STCLASS_AND;
4686 StructCopy(&accum, data->start_class,
4687 struct regnode_charclass_class);
4688 flags |= SCF_DO_STCLASS_OR;
4689 data->start_class->flags |= ANYOF_EOS;
4696 else if (PL_regkind[OP(scan)] == TRIE) {
4697 reg_trie_data *trie = (reg_trie_data*)RExC_rxi->data->data[ ARG(scan) ];
4700 min += trie->minlen;
4701 delta += (trie->maxlen - trie->minlen);
4702 flags &= ~SCF_DO_STCLASS; /* xxx */
4703 if (flags & SCF_DO_SUBSTR) {
4704 SCAN_COMMIT(pRExC_state,data,minlenp); /* Cannot expect anything... */
4705 data->pos_min += trie->minlen;
4706 data->pos_delta += (trie->maxlen - trie->minlen);
4707 if (trie->maxlen != trie->minlen)
4708 data->longest = &(data->longest_float);
4710 if (trie->jump) /* no more substrings -- for now /grr*/
4711 flags &= ~SCF_DO_SUBSTR;
4713 #endif /* old or new */
4714 #endif /* TRIE_STUDY_OPT */
4716 /* Else: zero-length, ignore. */
4717 scan = regnext(scan);
4722 stopparen = frame->stop;
4723 frame = frame->prev;
4724 goto fake_study_recurse;
4729 DEBUG_STUDYDATA("pre-fin:",data,depth);
4732 *deltap = is_inf_internal ? I32_MAX : delta;
4733 if (flags & SCF_DO_SUBSTR && is_inf)
4734 data->pos_delta = I32_MAX - data->pos_min;
4735 if (is_par > (I32)U8_MAX)
4737 if (is_par && pars==1 && data) {
4738 data->flags |= SF_IN_PAR;
4739 data->flags &= ~SF_HAS_PAR;
4741 else if (pars && data) {
4742 data->flags |= SF_HAS_PAR;
4743 data->flags &= ~SF_IN_PAR;
4745 if (flags & SCF_DO_STCLASS_OR)
4746 cl_and(data->start_class, and_withp);
4747 if (flags & SCF_TRIE_RESTUDY)
4748 data->flags |= SCF_TRIE_RESTUDY;
4750 DEBUG_STUDYDATA("post-fin:",data,depth);
4752 return min < stopmin ? min : stopmin;
4756 S_add_data(RExC_state_t *pRExC_state, U32 n, const char *s)
4758 U32 count = RExC_rxi->data ? RExC_rxi->data->count : 0;
4760 PERL_ARGS_ASSERT_ADD_DATA;
4762 Renewc(RExC_rxi->data,
4763 sizeof(*RExC_rxi->data) + sizeof(void*) * (count + n - 1),
4764 char, struct reg_data);
4766 Renew(RExC_rxi->data->what, count + n, U8);
4768 Newx(RExC_rxi->data->what, n, U8);
4769 RExC_rxi->data->count = count + n;
4770 Copy(s, RExC_rxi->data->what + count, n, U8);
4774 /*XXX: todo make this not included in a non debugging perl */
4775 #ifndef PERL_IN_XSUB_RE
4777 Perl_reginitcolors(pTHX)
4780 const char * const s = PerlEnv_getenv("PERL_RE_COLORS");
4782 char *t = savepv(s);
4786 t = strchr(t, '\t');
4792 PL_colors[i] = t = (char *)"";
4797 PL_colors[i++] = (char *)"";
4804 #ifdef TRIE_STUDY_OPT
4805 #define CHECK_RESTUDY_GOTO \
4807 (data.flags & SCF_TRIE_RESTUDY) \
4811 #define CHECK_RESTUDY_GOTO
4815 - pregcomp - compile a regular expression into internal code
4817 * We can't allocate space until we know how big the compiled form will be,
4818 * but we can't compile it (and thus know how big it is) until we've got a
4819 * place to put the code. So we cheat: we compile it twice, once with code
4820 * generation turned off and size counting turned on, and once "for real".
4821 * This also means that we don't allocate space until we are sure that the
4822 * thing really will compile successfully, and we never have to move the
4823 * code and thus invalidate pointers into it. (Note that it has to be in
4824 * one piece because free() must be able to free it all.) [NB: not true in perl]
4826 * Beware that the optimization-preparation code in here knows about some
4827 * of the structure of the compiled regexp. [I'll say.]
4832 #ifndef PERL_IN_XSUB_RE
4833 #define RE_ENGINE_PTR &reh_regexp_engine
4835 extern const struct regexp_engine my_reg_engine;
4836 #define RE_ENGINE_PTR &my_reg_engine
4839 #ifndef PERL_IN_XSUB_RE
4841 Perl_pregcomp(pTHX_ SV * const pattern, const U32 flags)
4844 HV * const table = GvHV(PL_hintgv);
4846 PERL_ARGS_ASSERT_PREGCOMP;
4848 /* Dispatch a request to compile a regexp to correct
4851 SV **ptr= hv_fetchs(table, "regcomp", FALSE);
4852 GET_RE_DEBUG_FLAGS_DECL;
4853 if (ptr && SvIOK(*ptr) && SvIV(*ptr)) {
4854 const regexp_engine *eng=INT2PTR(regexp_engine*,SvIV(*ptr));
4856 PerlIO_printf(Perl_debug_log, "Using engine %"UVxf"\n",
4859 return CALLREGCOMP_ENG(eng, pattern, flags);
4862 return Perl_re_compile(aTHX_ pattern, flags);
4867 Perl_re_compile(pTHX_ SV * const pattern, U32 orig_pm_flags)
4872 register regexp_internal *ri;
4881 /* these are all flags - maybe they should be turned
4882 * into a single int with different bit masks */
4883 I32 sawlookahead = 0;
4886 bool used_setjump = FALSE;
4887 regex_charset initial_charset = get_regex_charset(orig_pm_flags);
4892 RExC_state_t RExC_state;
4893 RExC_state_t * const pRExC_state = &RExC_state;
4894 #ifdef TRIE_STUDY_OPT
4896 RExC_state_t copyRExC_state;
4898 GET_RE_DEBUG_FLAGS_DECL;
4900 PERL_ARGS_ASSERT_RE_COMPILE;
4902 DEBUG_r(if (!PL_colorset) reginitcolors());
4904 #ifndef PERL_IN_XSUB_RE
4905 /* Initialize these here instead of as-needed, as is quick and avoids
4906 * having to test them each time otherwise */
4907 if (! PL_AboveLatin1) {
4908 PL_AboveLatin1 = _new_invlist_C_array(AboveLatin1_invlist);
4909 PL_ASCII = _new_invlist_C_array(ASCII_invlist);
4910 PL_Latin1 = _new_invlist_C_array(Latin1_invlist);
4912 PL_L1PosixAlnum = _new_invlist_C_array(L1PosixAlnum_invlist);
4913 PL_PosixAlnum = _new_invlist_C_array(PosixAlnum_invlist);
4915 PL_L1PosixAlpha = _new_invlist_C_array(L1PosixAlpha_invlist);
4916 PL_PosixAlpha = _new_invlist_C_array(PosixAlpha_invlist);
4918 PL_PosixBlank = _new_invlist_C_array(PosixBlank_invlist);
4919 PL_XPosixBlank = _new_invlist_C_array(XPosixBlank_invlist);
4921 PL_L1Cased = _new_invlist_C_array(L1Cased_invlist);
4923 PL_PosixCntrl = _new_invlist_C_array(PosixCntrl_invlist);
4924 PL_XPosixCntrl = _new_invlist_C_array(XPosixCntrl_invlist);
4926 PL_PosixDigit = _new_invlist_C_array(PosixDigit_invlist);
4928 PL_L1PosixGraph = _new_invlist_C_array(L1PosixGraph_invlist);
4929 PL_PosixGraph = _new_invlist_C_array(PosixGraph_invlist);
4931 PL_L1PosixAlnum = _new_invlist_C_array(L1PosixAlnum_invlist);
4932 PL_PosixAlnum = _new_invlist_C_array(PosixAlnum_invlist);
4934 PL_L1PosixLower = _new_invlist_C_array(L1PosixLower_invlist);
4935 PL_PosixLower = _new_invlist_C_array(PosixLower_invlist);
4937 PL_L1PosixPrint = _new_invlist_C_array(L1PosixPrint_invlist);
4938 PL_PosixPrint = _new_invlist_C_array(PosixPrint_invlist);
4940 PL_L1PosixPunct = _new_invlist_C_array(L1PosixPunct_invlist);
4941 PL_PosixPunct = _new_invlist_C_array(PosixPunct_invlist);
4943 PL_PerlSpace = _new_invlist_C_array(PerlSpace_invlist);
4944 PL_XPerlSpace = _new_invlist_C_array(XPerlSpace_invlist);
4946 PL_PosixSpace = _new_invlist_C_array(PosixSpace_invlist);
4947 PL_XPosixSpace = _new_invlist_C_array(XPosixSpace_invlist);
4949 PL_L1PosixUpper = _new_invlist_C_array(L1PosixUpper_invlist);
4950 PL_PosixUpper = _new_invlist_C_array(PosixUpper_invlist);
4952 PL_VertSpace = _new_invlist_C_array(VertSpace_invlist);
4954 PL_PosixWord = _new_invlist_C_array(PosixWord_invlist);
4955 PL_L1PosixWord = _new_invlist_C_array(L1PosixWord_invlist);
4957 PL_PosixXDigit = _new_invlist_C_array(PosixXDigit_invlist);
4958 PL_XPosixXDigit = _new_invlist_C_array(XPosixXDigit_invlist);
4962 exp = SvPV(pattern, plen);
4964 if (plen == 0) { /* ignore the utf8ness if the pattern is 0 length */
4965 RExC_utf8 = RExC_orig_utf8 = 0;
4968 RExC_utf8 = RExC_orig_utf8 = SvUTF8(pattern);
4970 RExC_uni_semantics = 0;
4971 RExC_contains_locale = 0;
4973 /****************** LONG JUMP TARGET HERE***********************/
4974 /* Longjmp back to here if have to switch in midstream to utf8 */
4975 if (! RExC_orig_utf8) {
4976 JMPENV_PUSH(jump_ret);
4977 used_setjump = TRUE;
4980 if (jump_ret == 0) { /* First time through */
4984 SV *dsv= sv_newmortal();
4985 RE_PV_QUOTED_DECL(s, RExC_utf8,
4986 dsv, exp, plen, 60);
4987 PerlIO_printf(Perl_debug_log, "%sCompiling REx%s %s\n",
4988 PL_colors[4],PL_colors[5],s);
4991 else { /* longjumped back */
4994 /* If the cause for the longjmp was other than changing to utf8, pop
4995 * our own setjmp, and longjmp to the correct handler */
4996 if (jump_ret != UTF8_LONGJMP) {
4998 JMPENV_JUMP(jump_ret);
5003 /* It's possible to write a regexp in ascii that represents Unicode
5004 codepoints outside of the byte range, such as via \x{100}. If we
5005 detect such a sequence we have to convert the entire pattern to utf8
5006 and then recompile, as our sizing calculation will have been based
5007 on 1 byte == 1 character, but we will need to use utf8 to encode
5008 at least some part of the pattern, and therefore must convert the whole
5011 DEBUG_PARSE_r(PerlIO_printf(Perl_debug_log,
5012 "UTF8 mismatch! Converting to utf8 for resizing and compile\n"));
5013 exp = (char*)Perl_bytes_to_utf8(aTHX_
5014 (U8*)SvPV_nomg(pattern, plen),
5017 RExC_orig_utf8 = RExC_utf8 = 1;
5021 #ifdef TRIE_STUDY_OPT
5025 pm_flags = orig_pm_flags;
5027 if (initial_charset == REGEX_LOCALE_CHARSET) {
5028 RExC_contains_locale = 1;
5030 else if (RExC_utf8 && initial_charset == REGEX_DEPENDS_CHARSET) {
5032 /* Set to use unicode semantics if the pattern is in utf8 and has the
5033 * 'depends' charset specified, as it means unicode when utf8 */
5034 set_regex_charset(&pm_flags, REGEX_UNICODE_CHARSET);
5038 RExC_flags = pm_flags;
5042 RExC_in_lookbehind = 0;
5043 RExC_seen_zerolen = *exp == '^' ? -1 : 0;
5044 RExC_seen_evals = 0;
5046 RExC_override_recoding = 0;
5048 /* First pass: determine size, legality. */
5056 RExC_emit = &PL_regdummy;
5057 RExC_whilem_seen = 0;
5058 RExC_open_parens = NULL;
5059 RExC_close_parens = NULL;
5061 RExC_paren_names = NULL;
5063 RExC_paren_name_list = NULL;
5065 RExC_recurse = NULL;
5066 RExC_recurse_count = 0;
5068 #if 0 /* REGC() is (currently) a NOP at the first pass.
5069 * Clever compilers notice this and complain. --jhi */
5070 REGC((U8)REG_MAGIC, (char*)RExC_emit);
5073 PerlIO_printf(Perl_debug_log, "Starting first pass (sizing)\n");
5075 RExC_lastparse=NULL;
5077 if (reg(pRExC_state, 0, &flags,1) == NULL) {
5078 RExC_precomp = NULL;
5082 /* Here, finished first pass. Get rid of any added setjmp */
5088 PerlIO_printf(Perl_debug_log,
5089 "Required size %"IVdf" nodes\n"
5090 "Starting second pass (creation)\n",
5093 RExC_lastparse=NULL;
5096 /* The first pass could have found things that force Unicode semantics */
5097 if ((RExC_utf8 || RExC_uni_semantics)
5098 && get_regex_charset(pm_flags) == REGEX_DEPENDS_CHARSET)
5100 set_regex_charset(&pm_flags, REGEX_UNICODE_CHARSET);
5103 /* Small enough for pointer-storage convention?
5104 If extralen==0, this means that we will not need long jumps. */
5105 if (RExC_size >= 0x10000L && RExC_extralen)
5106 RExC_size += RExC_extralen;
5109 if (RExC_whilem_seen > 15)
5110 RExC_whilem_seen = 15;
5112 /* Allocate space and zero-initialize. Note, the two step process
5113 of zeroing when in debug mode, thus anything assigned has to
5114 happen after that */
5115 rx = (REGEXP*) newSV_type(SVt_REGEXP);
5116 r = (struct regexp*)SvANY(rx);
5117 Newxc(ri, sizeof(regexp_internal) + (unsigned)RExC_size * sizeof(regnode),
5118 char, regexp_internal);
5119 if ( r == NULL || ri == NULL )
5120 FAIL("Regexp out of space");
5122 /* avoid reading uninitialized memory in DEBUGGING code in study_chunk() */
5123 Zero(ri, sizeof(regexp_internal) + (unsigned)RExC_size * sizeof(regnode), char);
5125 /* bulk initialize base fields with 0. */
5126 Zero(ri, sizeof(regexp_internal), char);
5129 /* non-zero initialization begins here */
5131 r->engine= RE_ENGINE_PTR;
5132 r->extflags = pm_flags;
5134 bool has_p = ((r->extflags & RXf_PMf_KEEPCOPY) == RXf_PMf_KEEPCOPY);
5135 bool has_charset = (get_regex_charset(r->extflags) != REGEX_DEPENDS_CHARSET);
5137 /* The caret is output if there are any defaults: if not all the STD
5138 * flags are set, or if no character set specifier is needed */
5140 (((r->extflags & RXf_PMf_STD_PMMOD) != RXf_PMf_STD_PMMOD)
5142 bool has_runon = ((RExC_seen & REG_SEEN_RUN_ON_COMMENT)==REG_SEEN_RUN_ON_COMMENT);
5143 U16 reganch = (U16)((r->extflags & RXf_PMf_STD_PMMOD)
5144 >> RXf_PMf_STD_PMMOD_SHIFT);
5145 const char *fptr = STD_PAT_MODS; /*"msix"*/
5147 /* Allocate for the worst case, which is all the std flags are turned
5148 * on. If more precision is desired, we could do a population count of
5149 * the flags set. This could be done with a small lookup table, or by
5150 * shifting, masking and adding, or even, when available, assembly
5151 * language for a machine-language population count.
5152 * We never output a minus, as all those are defaults, so are
5153 * covered by the caret */
5154 const STRLEN wraplen = plen + has_p + has_runon
5155 + has_default /* If needs a caret */
5157 /* If needs a character set specifier */
5158 + ((has_charset) ? MAX_CHARSET_NAME_LENGTH : 0)
5159 + (sizeof(STD_PAT_MODS) - 1)
5160 + (sizeof("(?:)") - 1);
5162 p = sv_grow(MUTABLE_SV(rx), wraplen + 1); /* +1 for the ending NUL */
5164 SvFLAGS(rx) |= SvUTF8(pattern);
5167 /* If a default, cover it using the caret */
5169 *p++= DEFAULT_PAT_MOD;
5173 const char* const name = get_regex_charset_name(r->extflags, &len);
5174 Copy(name, p, len, char);
5178 *p++ = KEEPCOPY_PAT_MOD; /*'p'*/
5181 while((ch = *fptr++)) {
5189 Copy(RExC_precomp, p, plen, char);
5190 assert ((RX_WRAPPED(rx) - p) < 16);
5191 r->pre_prefix = p - RX_WRAPPED(rx);
5197 SvCUR_set(rx, p - SvPVX_const(rx));
5201 r->nparens = RExC_npar - 1; /* set early to validate backrefs */
5203 if (RExC_seen & REG_SEEN_RECURSE) {
5204 Newxz(RExC_open_parens, RExC_npar,regnode *);
5205 SAVEFREEPV(RExC_open_parens);
5206 Newxz(RExC_close_parens,RExC_npar,regnode *);
5207 SAVEFREEPV(RExC_close_parens);
5210 /* Useful during FAIL. */
5211 #ifdef RE_TRACK_PATTERN_OFFSETS
5212 Newxz(ri->u.offsets, 2*RExC_size+1, U32); /* MJD 20001228 */
5213 DEBUG_OFFSETS_r(PerlIO_printf(Perl_debug_log,
5214 "%s %"UVuf" bytes for offset annotations.\n",
5215 ri->u.offsets ? "Got" : "Couldn't get",
5216 (UV)((2*RExC_size+1) * sizeof(U32))));
5218 SetProgLen(ri,RExC_size);
5222 REH_CALL_COMP_BEGIN_HOOK(pRExC_state->rx);
5224 /* Second pass: emit code. */
5225 RExC_flags = pm_flags; /* don't let top level (?i) bleed */
5230 RExC_emit_start = ri->program;
5231 RExC_emit = ri->program;
5232 RExC_emit_bound = ri->program + RExC_size + 1;
5234 /* Store the count of eval-groups for security checks: */
5235 RExC_rx->seen_evals = RExC_seen_evals;
5236 REGC((U8)REG_MAGIC, (char*) RExC_emit++);
5237 if (reg(pRExC_state, 0, &flags,1) == NULL) {
5241 /* XXXX To minimize changes to RE engine we always allocate
5242 3-units-long substrs field. */
5243 Newx(r->substrs, 1, struct reg_substr_data);
5244 if (RExC_recurse_count) {
5245 Newxz(RExC_recurse,RExC_recurse_count,regnode *);
5246 SAVEFREEPV(RExC_recurse);
5250 r->minlen = minlen = sawlookahead = sawplus = sawopen = 0;
5251 Zero(r->substrs, 1, struct reg_substr_data);
5253 #ifdef TRIE_STUDY_OPT
5255 StructCopy(&zero_scan_data, &data, scan_data_t);
5256 copyRExC_state = RExC_state;
5259 DEBUG_OPTIMISE_r(PerlIO_printf(Perl_debug_log,"Restudying\n"));
5261 RExC_state = copyRExC_state;
5262 if (seen & REG_TOP_LEVEL_BRANCHES)
5263 RExC_seen |= REG_TOP_LEVEL_BRANCHES;
5265 RExC_seen &= ~REG_TOP_LEVEL_BRANCHES;
5266 if (data.last_found) {
5267 SvREFCNT_dec(data.longest_fixed);
5268 SvREFCNT_dec(data.longest_float);
5269 SvREFCNT_dec(data.last_found);
5271 StructCopy(&zero_scan_data, &data, scan_data_t);
5274 StructCopy(&zero_scan_data, &data, scan_data_t);
5277 /* Dig out information for optimizations. */
5278 r->extflags = RExC_flags; /* was pm_op */
5279 /*dmq: removed as part of de-PMOP: pm->op_pmflags = RExC_flags; */
5282 SvUTF8_on(rx); /* Unicode in it? */
5283 ri->regstclass = NULL;
5284 if (RExC_naughty >= 10) /* Probably an expensive pattern. */
5285 r->intflags |= PREGf_NAUGHTY;
5286 scan = ri->program + 1; /* First BRANCH. */
5288 /* testing for BRANCH here tells us whether there is "must appear"
5289 data in the pattern. If there is then we can use it for optimisations */
5290 if (!(RExC_seen & REG_TOP_LEVEL_BRANCHES)) { /* Only one top-level choice. */
5292 STRLEN longest_float_length, longest_fixed_length;
5293 struct regnode_charclass_class ch_class; /* pointed to by data */
5295 I32 last_close = 0; /* pointed to by data */
5296 regnode *first= scan;
5297 regnode *first_next= regnext(first);
5299 * Skip introductions and multiplicators >= 1
5300 * so that we can extract the 'meat' of the pattern that must
5301 * match in the large if() sequence following.
5302 * NOTE that EXACT is NOT covered here, as it is normally
5303 * picked up by the optimiser separately.
5305 * This is unfortunate as the optimiser isnt handling lookahead
5306 * properly currently.
5309 while ((OP(first) == OPEN && (sawopen = 1)) ||
5310 /* An OR of *one* alternative - should not happen now. */
5311 (OP(first) == BRANCH && OP(first_next) != BRANCH) ||
5312 /* for now we can't handle lookbehind IFMATCH*/
5313 (OP(first) == IFMATCH && !first->flags && (sawlookahead = 1)) ||
5314 (OP(first) == PLUS) ||
5315 (OP(first) == MINMOD) ||
5316 /* An {n,m} with n>0 */
5317 (PL_regkind[OP(first)] == CURLY && ARG1(first) > 0) ||
5318 (OP(first) == NOTHING && PL_regkind[OP(first_next)] != END ))
5321 * the only op that could be a regnode is PLUS, all the rest
5322 * will be regnode_1 or regnode_2.
5325 if (OP(first) == PLUS)
5328 first += regarglen[OP(first)];
5330 first = NEXTOPER(first);
5331 first_next= regnext(first);
5334 /* Starting-point info. */
5336 DEBUG_PEEP("first:",first,0);
5337 /* Ignore EXACT as we deal with it later. */
5338 if (PL_regkind[OP(first)] == EXACT) {
5339 if (OP(first) == EXACT)
5340 NOOP; /* Empty, get anchored substr later. */
5342 ri->regstclass = first;
5345 else if (PL_regkind[OP(first)] == TRIE &&
5346 ((reg_trie_data *)ri->data->data[ ARG(first) ])->minlen>0)
5349 /* this can happen only on restudy */
5350 if ( OP(first) == TRIE ) {
5351 struct regnode_1 *trieop = (struct regnode_1 *)
5352 PerlMemShared_calloc(1, sizeof(struct regnode_1));
5353 StructCopy(first,trieop,struct regnode_1);
5354 trie_op=(regnode *)trieop;
5356 struct regnode_charclass *trieop = (struct regnode_charclass *)
5357 PerlMemShared_calloc(1, sizeof(struct regnode_charclass));
5358 StructCopy(first,trieop,struct regnode_charclass);
5359 trie_op=(regnode *)trieop;
5362 make_trie_failtable(pRExC_state, (regnode *)first, trie_op, 0);
5363 ri->regstclass = trie_op;
5366 else if (REGNODE_SIMPLE(OP(first)))
5367 ri->regstclass = first;
5368 else if (PL_regkind[OP(first)] == BOUND ||
5369 PL_regkind[OP(first)] == NBOUND)
5370 ri->regstclass = first;
5371 else if (PL_regkind[OP(first)] == BOL) {
5372 r->extflags |= (OP(first) == MBOL
5374 : (OP(first) == SBOL
5377 first = NEXTOPER(first);
5380 else if (OP(first) == GPOS) {
5381 r->extflags |= RXf_ANCH_GPOS;
5382 first = NEXTOPER(first);
5385 else if ((!sawopen || !RExC_sawback) &&
5386 (OP(first) == STAR &&
5387 PL_regkind[OP(NEXTOPER(first))] == REG_ANY) &&
5388 !(r->extflags & RXf_ANCH) && !(RExC_seen & REG_SEEN_EVAL))
5390 /* turn .* into ^.* with an implied $*=1 */
5392 (OP(NEXTOPER(first)) == REG_ANY)
5395 r->extflags |= type;
5396 r->intflags |= PREGf_IMPLICIT;
5397 first = NEXTOPER(first);
5400 if (sawplus && !sawlookahead && (!sawopen || !RExC_sawback)
5401 && !(RExC_seen & REG_SEEN_EVAL)) /* May examine pos and $& */
5402 /* x+ must match at the 1st pos of run of x's */
5403 r->intflags |= PREGf_SKIP;
5405 /* Scan is after the zeroth branch, first is atomic matcher. */
5406 #ifdef TRIE_STUDY_OPT
5409 PerlIO_printf(Perl_debug_log, "first at %"IVdf"\n",
5410 (IV)(first - scan + 1))
5414 PerlIO_printf(Perl_debug_log, "first at %"IVdf"\n",
5415 (IV)(first - scan + 1))
5421 * If there's something expensive in the r.e., find the
5422 * longest literal string that must appear and make it the
5423 * regmust. Resolve ties in favor of later strings, since
5424 * the regstart check works with the beginning of the r.e.
5425 * and avoiding duplication strengthens checking. Not a
5426 * strong reason, but sufficient in the absence of others.
5427 * [Now we resolve ties in favor of the earlier string if
5428 * it happens that c_offset_min has been invalidated, since the
5429 * earlier string may buy us something the later one won't.]
5432 data.longest_fixed = newSVpvs("");
5433 data.longest_float = newSVpvs("");
5434 data.last_found = newSVpvs("");
5435 data.longest = &(data.longest_fixed);
5437 if (!ri->regstclass) {
5438 cl_init(pRExC_state, &ch_class);
5439 data.start_class = &ch_class;
5440 stclass_flag = SCF_DO_STCLASS_AND;
5441 } else /* XXXX Check for BOUND? */
5443 data.last_closep = &last_close;
5445 minlen = study_chunk(pRExC_state, &first, &minlen, &fake, scan + RExC_size, /* Up to end */
5446 &data, -1, NULL, NULL,
5447 SCF_DO_SUBSTR | SCF_WHILEM_VISITED_POS | stclass_flag,0);
5453 if ( RExC_npar == 1 && data.longest == &(data.longest_fixed)
5454 && data.last_start_min == 0 && data.last_end > 0
5455 && !RExC_seen_zerolen
5456 && !(RExC_seen & REG_SEEN_VERBARG)
5457 && (!(RExC_seen & REG_SEEN_GPOS) || (r->extflags & RXf_ANCH_GPOS)))
5458 r->extflags |= RXf_CHECK_ALL;
5459 scan_commit(pRExC_state, &data,&minlen,0);
5460 SvREFCNT_dec(data.last_found);
5462 /* Note that code very similar to this but for anchored string
5463 follows immediately below, changes may need to be made to both.
5466 longest_float_length = CHR_SVLEN(data.longest_float);
5467 if (longest_float_length
5468 || (data.flags & SF_FL_BEFORE_EOL
5469 && (!(data.flags & SF_FL_BEFORE_MEOL)
5470 || (RExC_flags & RXf_PMf_MULTILINE))))
5474 /* See comments for join_exact for why REG_SEEN_EXACTF_SHARP_S */
5475 if ((RExC_seen & REG_SEEN_EXACTF_SHARP_S)
5476 || (SvCUR(data.longest_fixed) /* ok to leave SvCUR */
5477 && data.offset_fixed == data.offset_float_min
5478 && SvCUR(data.longest_fixed) == SvCUR(data.longest_float)))
5479 goto remove_float; /* As in (a)+. */
5481 /* copy the information about the longest float from the reg_scan_data
5482 over to the program. */
5483 if (SvUTF8(data.longest_float)) {
5484 r->float_utf8 = data.longest_float;
5485 r->float_substr = NULL;
5487 r->float_substr = data.longest_float;
5488 r->float_utf8 = NULL;
5490 /* float_end_shift is how many chars that must be matched that
5491 follow this item. We calculate it ahead of time as once the
5492 lookbehind offset is added in we lose the ability to correctly
5494 ml = data.minlen_float ? *(data.minlen_float)
5495 : (I32)longest_float_length;
5496 r->float_end_shift = ml - data.offset_float_min
5497 - longest_float_length + (SvTAIL(data.longest_float) != 0)
5498 + data.lookbehind_float;
5499 r->float_min_offset = data.offset_float_min - data.lookbehind_float;
5500 r->float_max_offset = data.offset_float_max;
5501 if (data.offset_float_max < I32_MAX) /* Don't offset infinity */
5502 r->float_max_offset -= data.lookbehind_float;
5504 t = (data.flags & SF_FL_BEFORE_EOL /* Can't have SEOL and MULTI */
5505 && (!(data.flags & SF_FL_BEFORE_MEOL)
5506 || (RExC_flags & RXf_PMf_MULTILINE)));
5507 fbm_compile(data.longest_float, t ? FBMcf_TAIL : 0);
5511 r->float_substr = r->float_utf8 = NULL;
5512 SvREFCNT_dec(data.longest_float);
5513 longest_float_length = 0;
5516 /* Note that code very similar to this but for floating string
5517 is immediately above, changes may need to be made to both.
5520 longest_fixed_length = CHR_SVLEN(data.longest_fixed);
5522 /* See comments for join_exact for why REG_SEEN_EXACTF_SHARP_S */
5523 if (! (RExC_seen & REG_SEEN_EXACTF_SHARP_S)
5524 && (longest_fixed_length
5525 || (data.flags & SF_FIX_BEFORE_EOL /* Cannot have SEOL and MULTI */
5526 && (!(data.flags & SF_FIX_BEFORE_MEOL)
5527 || (RExC_flags & RXf_PMf_MULTILINE)))) )
5531 /* copy the information about the longest fixed
5532 from the reg_scan_data over to the program. */
5533 if (SvUTF8(data.longest_fixed)) {
5534 r->anchored_utf8 = data.longest_fixed;
5535 r->anchored_substr = NULL;
5537 r->anchored_substr = data.longest_fixed;
5538 r->anchored_utf8 = NULL;
5540 /* fixed_end_shift is how many chars that must be matched that
5541 follow this item. We calculate it ahead of time as once the
5542 lookbehind offset is added in we lose the ability to correctly
5544 ml = data.minlen_fixed ? *(data.minlen_fixed)
5545 : (I32)longest_fixed_length;
5546 r->anchored_end_shift = ml - data.offset_fixed
5547 - longest_fixed_length + (SvTAIL(data.longest_fixed) != 0)
5548 + data.lookbehind_fixed;
5549 r->anchored_offset = data.offset_fixed - data.lookbehind_fixed;
5551 t = (data.flags & SF_FIX_BEFORE_EOL /* Can't have SEOL and MULTI */
5552 && (!(data.flags & SF_FIX_BEFORE_MEOL)
5553 || (RExC_flags & RXf_PMf_MULTILINE)));
5554 fbm_compile(data.longest_fixed, t ? FBMcf_TAIL : 0);
5557 r->anchored_substr = r->anchored_utf8 = NULL;
5558 SvREFCNT_dec(data.longest_fixed);
5559 longest_fixed_length = 0;
5562 && (OP(ri->regstclass) == REG_ANY || OP(ri->regstclass) == SANY))
5563 ri->regstclass = NULL;
5565 if ((!(r->anchored_substr || r->anchored_utf8) || r->anchored_offset)
5567 && !(data.start_class->flags & ANYOF_EOS)
5568 && !cl_is_anything(data.start_class))
5570 const U32 n = add_data(pRExC_state, 1, "f");
5571 data.start_class->flags |= ANYOF_IS_SYNTHETIC;
5573 Newx(RExC_rxi->data->data[n], 1,
5574 struct regnode_charclass_class);
5575 StructCopy(data.start_class,
5576 (struct regnode_charclass_class*)RExC_rxi->data->data[n],
5577 struct regnode_charclass_class);
5578 ri->regstclass = (regnode*)RExC_rxi->data->data[n];
5579 r->intflags &= ~PREGf_SKIP; /* Used in find_byclass(). */
5580 DEBUG_COMPILE_r({ SV *sv = sv_newmortal();
5581 regprop(r, sv, (regnode*)data.start_class);
5582 PerlIO_printf(Perl_debug_log,
5583 "synthetic stclass \"%s\".\n",
5584 SvPVX_const(sv));});
5587 /* A temporary algorithm prefers floated substr to fixed one to dig more info. */
5588 if (longest_fixed_length > longest_float_length) {
5589 r->check_end_shift = r->anchored_end_shift;
5590 r->check_substr = r->anchored_substr;
5591 r->check_utf8 = r->anchored_utf8;
5592 r->check_offset_min = r->check_offset_max = r->anchored_offset;
5593 if (r->extflags & RXf_ANCH_SINGLE)
5594 r->extflags |= RXf_NOSCAN;
5597 r->check_end_shift = r->float_end_shift;
5598 r->check_substr = r->float_substr;
5599 r->check_utf8 = r->float_utf8;
5600 r->check_offset_min = r->float_min_offset;
5601 r->check_offset_max = r->float_max_offset;
5603 /* XXXX Currently intuiting is not compatible with ANCH_GPOS.
5604 This should be changed ASAP! */
5605 if ((r->check_substr || r->check_utf8) && !(r->extflags & RXf_ANCH_GPOS)) {
5606 r->extflags |= RXf_USE_INTUIT;
5607 if (SvTAIL(r->check_substr ? r->check_substr : r->check_utf8))
5608 r->extflags |= RXf_INTUIT_TAIL;
5610 /* XXX Unneeded? dmq (shouldn't as this is handled elsewhere)
5611 if ( (STRLEN)minlen < longest_float_length )
5612 minlen= longest_float_length;
5613 if ( (STRLEN)minlen < longest_fixed_length )
5614 minlen= longest_fixed_length;
5618 /* Several toplevels. Best we can is to set minlen. */
5620 struct regnode_charclass_class ch_class;
5623 DEBUG_PARSE_r(PerlIO_printf(Perl_debug_log, "\nMulti Top Level\n"));
5625 scan = ri->program + 1;
5626 cl_init(pRExC_state, &ch_class);
5627 data.start_class = &ch_class;
5628 data.last_closep = &last_close;
5631 minlen = study_chunk(pRExC_state, &scan, &minlen, &fake, scan + RExC_size,
5632 &data, -1, NULL, NULL, SCF_DO_STCLASS_AND|SCF_WHILEM_VISITED_POS,0);
5636 r->check_substr = r->check_utf8 = r->anchored_substr = r->anchored_utf8
5637 = r->float_substr = r->float_utf8 = NULL;
5639 if (!(data.start_class->flags & ANYOF_EOS)
5640 && !cl_is_anything(data.start_class))
5642 const U32 n = add_data(pRExC_state, 1, "f");
5643 data.start_class->flags |= ANYOF_IS_SYNTHETIC;
5645 Newx(RExC_rxi->data->data[n], 1,
5646 struct regnode_charclass_class);
5647 StructCopy(data.start_class,
5648 (struct regnode_charclass_class*)RExC_rxi->data->data[n],
5649 struct regnode_charclass_class);
5650 ri->regstclass = (regnode*)RExC_rxi->data->data[n];
5651 r->intflags &= ~PREGf_SKIP; /* Used in find_byclass(). */
5652 DEBUG_COMPILE_r({ SV* sv = sv_newmortal();
5653 regprop(r, sv, (regnode*)data.start_class);
5654 PerlIO_printf(Perl_debug_log,
5655 "synthetic stclass \"%s\".\n",
5656 SvPVX_const(sv));});
5660 /* Guard against an embedded (?=) or (?<=) with a longer minlen than
5661 the "real" pattern. */
5663 PerlIO_printf(Perl_debug_log,"minlen: %"IVdf" r->minlen:%"IVdf"\n",
5664 (IV)minlen, (IV)r->minlen);
5666 r->minlenret = minlen;
5667 if (r->minlen < minlen)
5670 if (RExC_seen & REG_SEEN_GPOS)
5671 r->extflags |= RXf_GPOS_SEEN;
5672 if (RExC_seen & REG_SEEN_LOOKBEHIND)
5673 r->extflags |= RXf_LOOKBEHIND_SEEN;
5674 if (RExC_seen & REG_SEEN_EVAL)
5675 r->extflags |= RXf_EVAL_SEEN;
5676 if (RExC_seen & REG_SEEN_CANY)
5677 r->extflags |= RXf_CANY_SEEN;
5678 if (RExC_seen & REG_SEEN_VERBARG)
5679 r->intflags |= PREGf_VERBARG_SEEN;
5680 if (RExC_seen & REG_SEEN_CUTGROUP)
5681 r->intflags |= PREGf_CUTGROUP_SEEN;
5682 if (RExC_paren_names)
5683 RXp_PAREN_NAMES(r) = MUTABLE_HV(SvREFCNT_inc(RExC_paren_names));
5685 RXp_PAREN_NAMES(r) = NULL;
5687 #ifdef STUPID_PATTERN_CHECKS
5688 if (RX_PRELEN(rx) == 0)
5689 r->extflags |= RXf_NULL;
5690 if (r->extflags & RXf_SPLIT && RX_PRELEN(rx) == 1 && RX_PRECOMP(rx)[0] == ' ')
5691 /* XXX: this should happen BEFORE we compile */
5692 r->extflags |= (RXf_SKIPWHITE|RXf_WHITE);
5693 else if (RX_PRELEN(rx) == 3 && memEQ("\\s+", RX_PRECOMP(rx), 3))
5694 r->extflags |= RXf_WHITE;
5695 else if (RX_PRELEN(rx) == 1 && RXp_PRECOMP(rx)[0] == '^')
5696 r->extflags |= RXf_START_ONLY;
5698 if (r->extflags & RXf_SPLIT && RX_PRELEN(rx) == 1 && RX_PRECOMP(rx)[0] == ' ')
5699 /* XXX: this should happen BEFORE we compile */
5700 r->extflags |= (RXf_SKIPWHITE|RXf_WHITE);
5702 regnode *first = ri->program + 1;
5705 if (PL_regkind[fop] == NOTHING && OP(NEXTOPER(first)) == END)
5706 r->extflags |= RXf_NULL;
5707 else if (PL_regkind[fop] == BOL && OP(NEXTOPER(first)) == END)
5708 r->extflags |= RXf_START_ONLY;
5709 else if (fop == PLUS && OP(NEXTOPER(first)) == SPACE
5710 && OP(regnext(first)) == END)
5711 r->extflags |= RXf_WHITE;
5715 if (RExC_paren_names) {
5716 ri->name_list_idx = add_data( pRExC_state, 1, "a" );
5717 ri->data->data[ri->name_list_idx] = (void*)SvREFCNT_inc(RExC_paren_name_list);
5720 ri->name_list_idx = 0;
5722 if (RExC_recurse_count) {
5723 for ( ; RExC_recurse_count ; RExC_recurse_count-- ) {
5724 const regnode *scan = RExC_recurse[RExC_recurse_count-1];
5725 ARG2L_SET( scan, RExC_open_parens[ARG(scan)-1] - scan );
5728 Newxz(r->offs, RExC_npar, regexp_paren_pair);
5729 /* assume we don't need to swap parens around before we match */
5732 PerlIO_printf(Perl_debug_log,"Final program:\n");
5735 #ifdef RE_TRACK_PATTERN_OFFSETS
5736 DEBUG_OFFSETS_r(if (ri->u.offsets) {
5737 const U32 len = ri->u.offsets[0];
5739 GET_RE_DEBUG_FLAGS_DECL;
5740 PerlIO_printf(Perl_debug_log, "Offsets: [%"UVuf"]\n\t", (UV)ri->u.offsets[0]);
5741 for (i = 1; i <= len; i++) {
5742 if (ri->u.offsets[i*2-1] || ri->u.offsets[i*2])
5743 PerlIO_printf(Perl_debug_log, "%"UVuf":%"UVuf"[%"UVuf"] ",
5744 (UV)i, (UV)ri->u.offsets[i*2-1], (UV)ri->u.offsets[i*2]);
5746 PerlIO_printf(Perl_debug_log, "\n");
5752 #undef RE_ENGINE_PTR
5756 Perl_reg_named_buff(pTHX_ REGEXP * const rx, SV * const key, SV * const value,
5759 PERL_ARGS_ASSERT_REG_NAMED_BUFF;
5761 PERL_UNUSED_ARG(value);
5763 if (flags & RXapif_FETCH) {
5764 return reg_named_buff_fetch(rx, key, flags);
5765 } else if (flags & (RXapif_STORE | RXapif_DELETE | RXapif_CLEAR)) {
5766 Perl_croak_no_modify(aTHX);
5768 } else if (flags & RXapif_EXISTS) {
5769 return reg_named_buff_exists(rx, key, flags)
5772 } else if (flags & RXapif_REGNAMES) {
5773 return reg_named_buff_all(rx, flags);
5774 } else if (flags & (RXapif_SCALAR | RXapif_REGNAMES_COUNT)) {
5775 return reg_named_buff_scalar(rx, flags);
5777 Perl_croak(aTHX_ "panic: Unknown flags %d in named_buff", (int)flags);
5783 Perl_reg_named_buff_iter(pTHX_ REGEXP * const rx, const SV * const lastkey,
5786 PERL_ARGS_ASSERT_REG_NAMED_BUFF_ITER;
5787 PERL_UNUSED_ARG(lastkey);
5789 if (flags & RXapif_FIRSTKEY)
5790 return reg_named_buff_firstkey(rx, flags);
5791 else if (flags & RXapif_NEXTKEY)
5792 return reg_named_buff_nextkey(rx, flags);
5794 Perl_croak(aTHX_ "panic: Unknown flags %d in named_buff_iter", (int)flags);
5800 Perl_reg_named_buff_fetch(pTHX_ REGEXP * const r, SV * const namesv,
5803 AV *retarray = NULL;
5805 struct regexp *const rx = (struct regexp *)SvANY(r);
5807 PERL_ARGS_ASSERT_REG_NAMED_BUFF_FETCH;
5809 if (flags & RXapif_ALL)
5812 if (rx && RXp_PAREN_NAMES(rx)) {
5813 HE *he_str = hv_fetch_ent( RXp_PAREN_NAMES(rx), namesv, 0, 0 );
5816 SV* sv_dat=HeVAL(he_str);
5817 I32 *nums=(I32*)SvPVX(sv_dat);
5818 for ( i=0; i<SvIVX(sv_dat); i++ ) {
5819 if ((I32)(rx->nparens) >= nums[i]
5820 && rx->offs[nums[i]].start != -1
5821 && rx->offs[nums[i]].end != -1)
5824 CALLREG_NUMBUF_FETCH(r,nums[i],ret);
5829 ret = newSVsv(&PL_sv_undef);
5832 av_push(retarray, ret);
5835 return newRV_noinc(MUTABLE_SV(retarray));
5842 Perl_reg_named_buff_exists(pTHX_ REGEXP * const r, SV * const key,
5845 struct regexp *const rx = (struct regexp *)SvANY(r);
5847 PERL_ARGS_ASSERT_REG_NAMED_BUFF_EXISTS;
5849 if (rx && RXp_PAREN_NAMES(rx)) {
5850 if (flags & RXapif_ALL) {
5851 return hv_exists_ent(RXp_PAREN_NAMES(rx), key, 0);
5853 SV *sv = CALLREG_NAMED_BUFF_FETCH(r, key, flags);
5867 Perl_reg_named_buff_firstkey(pTHX_ REGEXP * const r, const U32 flags)
5869 struct regexp *const rx = (struct regexp *)SvANY(r);
5871 PERL_ARGS_ASSERT_REG_NAMED_BUFF_FIRSTKEY;
5873 if ( rx && RXp_PAREN_NAMES(rx) ) {
5874 (void)hv_iterinit(RXp_PAREN_NAMES(rx));
5876 return CALLREG_NAMED_BUFF_NEXTKEY(r, NULL, flags & ~RXapif_FIRSTKEY);
5883 Perl_reg_named_buff_nextkey(pTHX_ REGEXP * const r, const U32 flags)
5885 struct regexp *const rx = (struct regexp *)SvANY(r);
5886 GET_RE_DEBUG_FLAGS_DECL;
5888 PERL_ARGS_ASSERT_REG_NAMED_BUFF_NEXTKEY;
5890 if (rx && RXp_PAREN_NAMES(rx)) {
5891 HV *hv = RXp_PAREN_NAMES(rx);
5893 while ( (temphe = hv_iternext_flags(hv,0)) ) {
5896 SV* sv_dat = HeVAL(temphe);
5897 I32 *nums = (I32*)SvPVX(sv_dat);
5898 for ( i = 0; i < SvIVX(sv_dat); i++ ) {
5899 if ((I32)(rx->lastparen) >= nums[i] &&
5900 rx->offs[nums[i]].start != -1 &&
5901 rx->offs[nums[i]].end != -1)
5907 if (parno || flags & RXapif_ALL) {
5908 return newSVhek(HeKEY_hek(temphe));
5916 Perl_reg_named_buff_scalar(pTHX_ REGEXP * const r, const U32 flags)
5921 struct regexp *const rx = (struct regexp *)SvANY(r);
5923 PERL_ARGS_ASSERT_REG_NAMED_BUFF_SCALAR;
5925 if (rx && RXp_PAREN_NAMES(rx)) {
5926 if (flags & (RXapif_ALL | RXapif_REGNAMES_COUNT)) {
5927 return newSViv(HvTOTALKEYS(RXp_PAREN_NAMES(rx)));
5928 } else if (flags & RXapif_ONE) {
5929 ret = CALLREG_NAMED_BUFF_ALL(r, (flags | RXapif_REGNAMES));
5930 av = MUTABLE_AV(SvRV(ret));
5931 length = av_len(av);
5933 return newSViv(length + 1);
5935 Perl_croak(aTHX_ "panic: Unknown flags %d in named_buff_scalar", (int)flags);
5939 return &PL_sv_undef;
5943 Perl_reg_named_buff_all(pTHX_ REGEXP * const r, const U32 flags)
5945 struct regexp *const rx = (struct regexp *)SvANY(r);
5948 PERL_ARGS_ASSERT_REG_NAMED_BUFF_ALL;
5950 if (rx && RXp_PAREN_NAMES(rx)) {
5951 HV *hv= RXp_PAREN_NAMES(rx);
5953 (void)hv_iterinit(hv);
5954 while ( (temphe = hv_iternext_flags(hv,0)) ) {
5957 SV* sv_dat = HeVAL(temphe);
5958 I32 *nums = (I32*)SvPVX(sv_dat);
5959 for ( i = 0; i < SvIVX(sv_dat); i++ ) {
5960 if ((I32)(rx->lastparen) >= nums[i] &&
5961 rx->offs[nums[i]].start != -1 &&
5962 rx->offs[nums[i]].end != -1)
5968 if (parno || flags & RXapif_ALL) {
5969 av_push(av, newSVhek(HeKEY_hek(temphe)));
5974 return newRV_noinc(MUTABLE_SV(av));
5978 Perl_reg_numbered_buff_fetch(pTHX_ REGEXP * const r, const I32 paren,
5981 struct regexp *const rx = (struct regexp *)SvANY(r);
5986 PERL_ARGS_ASSERT_REG_NUMBERED_BUFF_FETCH;
5989 sv_setsv(sv,&PL_sv_undef);
5993 if (paren == RX_BUFF_IDX_PREMATCH && rx->offs[0].start != -1) {
5995 i = rx->offs[0].start;
5999 if (paren == RX_BUFF_IDX_POSTMATCH && rx->offs[0].end != -1) {
6001 s = rx->subbeg + rx->offs[0].end;
6002 i = rx->sublen - rx->offs[0].end;
6005 if ( 0 <= paren && paren <= (I32)rx->nparens &&
6006 (s1 = rx->offs[paren].start) != -1 &&
6007 (t1 = rx->offs[paren].end) != -1)
6011 s = rx->subbeg + s1;
6013 sv_setsv(sv,&PL_sv_undef);
6016 assert(rx->sublen >= (s - rx->subbeg) + i );
6018 const int oldtainted = PL_tainted;
6020 sv_setpvn(sv, s, i);
6021 PL_tainted = oldtainted;
6022 if ( (rx->extflags & RXf_CANY_SEEN)
6023 ? (RXp_MATCH_UTF8(rx)
6024 && (!i || is_utf8_string((U8*)s, i)))
6025 : (RXp_MATCH_UTF8(rx)) )
6032 if (RXp_MATCH_TAINTED(rx)) {
6033 if (SvTYPE(sv) >= SVt_PVMG) {
6034 MAGIC* const mg = SvMAGIC(sv);
6037 SvMAGIC_set(sv, mg->mg_moremagic);
6039 if ((mgt = SvMAGIC(sv))) {
6040 mg->mg_moremagic = mgt;
6041 SvMAGIC_set(sv, mg);
6051 sv_setsv(sv,&PL_sv_undef);
6057 Perl_reg_numbered_buff_store(pTHX_ REGEXP * const rx, const I32 paren,
6058 SV const * const value)
6060 PERL_ARGS_ASSERT_REG_NUMBERED_BUFF_STORE;
6062 PERL_UNUSED_ARG(rx);
6063 PERL_UNUSED_ARG(paren);
6064 PERL_UNUSED_ARG(value);
6067 Perl_croak_no_modify(aTHX);
6071 Perl_reg_numbered_buff_length(pTHX_ REGEXP * const r, const SV * const sv,
6074 struct regexp *const rx = (struct regexp *)SvANY(r);
6078 PERL_ARGS_ASSERT_REG_NUMBERED_BUFF_LENGTH;
6080 /* Some of this code was originally in C<Perl_magic_len> in F<mg.c> */
6082 /* $` / ${^PREMATCH} */
6083 case RX_BUFF_IDX_PREMATCH:
6084 if (rx->offs[0].start != -1) {
6085 i = rx->offs[0].start;
6093 /* $' / ${^POSTMATCH} */
6094 case RX_BUFF_IDX_POSTMATCH:
6095 if (rx->offs[0].end != -1) {
6096 i = rx->sublen - rx->offs[0].end;
6098 s1 = rx->offs[0].end;
6104 /* $& / ${^MATCH}, $1, $2, ... */
6106 if (paren <= (I32)rx->nparens &&
6107 (s1 = rx->offs[paren].start) != -1 &&
6108 (t1 = rx->offs[paren].end) != -1)
6113 if (ckWARN(WARN_UNINITIALIZED))
6114 report_uninit((const SV *)sv);
6119 if (i > 0 && RXp_MATCH_UTF8(rx)) {
6120 const char * const s = rx->subbeg + s1;
6125 if (is_utf8_string_loclen((U8*)s, i, &ep, &el))
6132 Perl_reg_qr_package(pTHX_ REGEXP * const rx)
6134 PERL_ARGS_ASSERT_REG_QR_PACKAGE;
6135 PERL_UNUSED_ARG(rx);
6139 return newSVpvs("Regexp");
6142 /* Scans the name of a named buffer from the pattern.
6143 * If flags is REG_RSN_RETURN_NULL returns null.
6144 * If flags is REG_RSN_RETURN_NAME returns an SV* containing the name
6145 * If flags is REG_RSN_RETURN_DATA returns the data SV* corresponding
6146 * to the parsed name as looked up in the RExC_paren_names hash.
6147 * If there is an error throws a vFAIL().. type exception.
6150 #define REG_RSN_RETURN_NULL 0
6151 #define REG_RSN_RETURN_NAME 1
6152 #define REG_RSN_RETURN_DATA 2
6155 S_reg_scan_name(pTHX_ RExC_state_t *pRExC_state, U32 flags)
6157 char *name_start = RExC_parse;
6159 PERL_ARGS_ASSERT_REG_SCAN_NAME;
6161 if (isIDFIRST_lazy_if(RExC_parse, UTF)) {
6162 /* skip IDFIRST by using do...while */
6165 RExC_parse += UTF8SKIP(RExC_parse);
6166 } while (isALNUM_utf8((U8*)RExC_parse));
6170 } while (isALNUM(*RExC_parse));
6175 = newSVpvn_flags(name_start, (int)(RExC_parse - name_start),
6176 SVs_TEMP | (UTF ? SVf_UTF8 : 0));
6177 if ( flags == REG_RSN_RETURN_NAME)
6179 else if (flags==REG_RSN_RETURN_DATA) {
6182 if ( ! sv_name ) /* should not happen*/
6183 Perl_croak(aTHX_ "panic: no svname in reg_scan_name");
6184 if (RExC_paren_names)
6185 he_str = hv_fetch_ent( RExC_paren_names, sv_name, 0, 0 );
6187 sv_dat = HeVAL(he_str);
6189 vFAIL("Reference to nonexistent named group");
6193 Perl_croak(aTHX_ "panic: bad flag %lx in reg_scan_name",
6194 (unsigned long) flags);
6201 #define DEBUG_PARSE_MSG(funcname) DEBUG_PARSE_r({ \
6202 int rem=(int)(RExC_end - RExC_parse); \
6211 if (RExC_lastparse!=RExC_parse) \
6212 PerlIO_printf(Perl_debug_log," >%.*s%-*s", \
6215 iscut ? "..." : "<" \
6218 PerlIO_printf(Perl_debug_log,"%16s",""); \
6221 num = RExC_size + 1; \
6223 num=REG_NODE_NUM(RExC_emit); \
6224 if (RExC_lastnum!=num) \
6225 PerlIO_printf(Perl_debug_log,"|%4d",num); \
6227 PerlIO_printf(Perl_debug_log,"|%4s",""); \
6228 PerlIO_printf(Perl_debug_log,"|%*s%-4s", \
6229 (int)((depth*2)), "", \
6233 RExC_lastparse=RExC_parse; \
6238 #define DEBUG_PARSE(funcname) DEBUG_PARSE_r({ \
6239 DEBUG_PARSE_MSG((funcname)); \
6240 PerlIO_printf(Perl_debug_log,"%4s","\n"); \
6242 #define DEBUG_PARSE_FMT(funcname,fmt,args) DEBUG_PARSE_r({ \
6243 DEBUG_PARSE_MSG((funcname)); \
6244 PerlIO_printf(Perl_debug_log,fmt "\n",args); \
6247 /* This section of code defines the inversion list object and its methods. The
6248 * interfaces are highly subject to change, so as much as possible is static to
6249 * this file. An inversion list is here implemented as a malloc'd C UV array
6250 * with some added info that is placed as UVs at the beginning in a header
6251 * portion. An inversion list for Unicode is an array of code points, sorted
6252 * by ordinal number. The zeroth element is the first code point in the list.
6253 * The 1th element is the first element beyond that not in the list. In other
6254 * words, the first range is
6255 * invlist[0]..(invlist[1]-1)
6256 * The other ranges follow. Thus every element whose index is divisible by two
6257 * marks the beginning of a range that is in the list, and every element not
6258 * divisible by two marks the beginning of a range not in the list. A single
6259 * element inversion list that contains the single code point N generally
6260 * consists of two elements
6263 * (The exception is when N is the highest representable value on the
6264 * machine, in which case the list containing just it would be a single
6265 * element, itself. By extension, if the last range in the list extends to
6266 * infinity, then the first element of that range will be in the inversion list
6267 * at a position that is divisible by two, and is the final element in the
6269 * Taking the complement (inverting) an inversion list is quite simple, if the
6270 * first element is 0, remove it; otherwise add a 0 element at the beginning.
6271 * This implementation reserves an element at the beginning of each inversion list
6272 * to contain 0 when the list contains 0, and contains 1 otherwise. The actual
6273 * beginning of the list is either that element if 0, or the next one if 1.
6275 * More about inversion lists can be found in "Unicode Demystified"
6276 * Chapter 13 by Richard Gillam, published by Addison-Wesley.
6277 * More will be coming when functionality is added later.
6279 * The inversion list data structure is currently implemented as an SV pointing
6280 * to an array of UVs that the SV thinks are bytes. This allows us to have an
6281 * array of UV whose memory management is automatically handled by the existing
6282 * facilities for SV's.
6284 * Some of the methods should always be private to the implementation, and some
6285 * should eventually be made public */
6287 #define INVLIST_LEN_OFFSET 0 /* Number of elements in the inversion list */
6288 #define INVLIST_ITER_OFFSET 1 /* Current iteration position */
6290 /* This is a combination of a version and data structure type, so that one
6291 * being passed in can be validated to be an inversion list of the correct
6292 * vintage. When the structure of the header is changed, a new random number
6293 * in the range 2**31-1 should be generated and the new() method changed to
6294 * insert that at this location. Then, if an auxiliary program doesn't change
6295 * correspondingly, it will be discovered immediately */
6296 #define INVLIST_VERSION_ID_OFFSET 2
6297 #define INVLIST_VERSION_ID 1064334010
6299 /* For safety, when adding new elements, remember to #undef them at the end of
6300 * the inversion list code section */
6302 #define INVLIST_ZERO_OFFSET 3 /* 0 or 1; must be last element in header */
6303 /* The UV at position ZERO contains either 0 or 1. If 0, the inversion list
6304 * contains the code point U+00000, and begins here. If 1, the inversion list
6305 * doesn't contain U+0000, and it begins at the next UV in the array.
6306 * Inverting an inversion list consists of adding or removing the 0 at the
6307 * beginning of it. By reserving a space for that 0, inversion can be made
6310 #define HEADER_LENGTH (INVLIST_ZERO_OFFSET + 1)
6312 /* Internally things are UVs */
6313 #define TO_INTERNAL_SIZE(x) ((x + HEADER_LENGTH) * sizeof(UV))
6314 #define FROM_INTERNAL_SIZE(x) ((x / sizeof(UV)) - HEADER_LENGTH)
6316 #define INVLIST_INITIAL_LEN 10
6318 PERL_STATIC_INLINE UV*
6319 S__invlist_array_init(pTHX_ SV* const invlist, const bool will_have_0)
6321 /* Returns a pointer to the first element in the inversion list's array.
6322 * This is called upon initialization of an inversion list. Where the
6323 * array begins depends on whether the list has the code point U+0000
6324 * in it or not. The other parameter tells it whether the code that
6325 * follows this call is about to put a 0 in the inversion list or not.
6326 * The first element is either the element with 0, if 0, or the next one,
6329 UV* zero = get_invlist_zero_addr(invlist);
6331 PERL_ARGS_ASSERT__INVLIST_ARRAY_INIT;
6334 assert(! *get_invlist_len_addr(invlist));
6336 /* 1^1 = 0; 1^0 = 1 */
6337 *zero = 1 ^ will_have_0;
6338 return zero + *zero;
6341 PERL_STATIC_INLINE UV*
6342 S_invlist_array(pTHX_ SV* const invlist)
6344 /* Returns the pointer to the inversion list's array. Every time the
6345 * length changes, this needs to be called in case malloc or realloc moved
6348 PERL_ARGS_ASSERT_INVLIST_ARRAY;
6350 /* Must not be empty. If these fail, you probably didn't check for <len>
6351 * being non-zero before trying to get the array */
6352 assert(*get_invlist_len_addr(invlist));
6353 assert(*get_invlist_zero_addr(invlist) == 0
6354 || *get_invlist_zero_addr(invlist) == 1);
6356 /* The array begins either at the element reserved for zero if the
6357 * list contains 0 (that element will be set to 0), or otherwise the next
6358 * element (in which case the reserved element will be set to 1). */
6359 return (UV *) (get_invlist_zero_addr(invlist)
6360 + *get_invlist_zero_addr(invlist));
6363 PERL_STATIC_INLINE UV*
6364 S_get_invlist_len_addr(pTHX_ SV* invlist)
6366 /* Return the address of the UV that contains the current number
6367 * of used elements in the inversion list */
6369 PERL_ARGS_ASSERT_GET_INVLIST_LEN_ADDR;
6371 return (UV *) (SvPVX(invlist) + (INVLIST_LEN_OFFSET * sizeof (UV)));
6374 PERL_STATIC_INLINE UV
6375 S_invlist_len(pTHX_ SV* const invlist)
6377 /* Returns the current number of elements stored in the inversion list's
6380 PERL_ARGS_ASSERT_INVLIST_LEN;
6382 return *get_invlist_len_addr(invlist);
6385 PERL_STATIC_INLINE void
6386 S_invlist_set_len(pTHX_ SV* const invlist, const UV len)
6388 /* Sets the current number of elements stored in the inversion list */
6390 PERL_ARGS_ASSERT_INVLIST_SET_LEN;
6392 *get_invlist_len_addr(invlist) = len;
6394 assert(len <= SvLEN(invlist));
6396 SvCUR_set(invlist, TO_INTERNAL_SIZE(len));
6397 /* If the list contains U+0000, that element is part of the header,
6398 * and should not be counted as part of the array. It will contain
6399 * 0 in that case, and 1 otherwise. So we could flop 0=>1, 1=>0 and
6401 * SvCUR_set(invlist,
6402 * TO_INTERNAL_SIZE(len
6403 * - (*get_invlist_zero_addr(inv_list) ^ 1)));
6404 * But, this is only valid if len is not 0. The consequences of not doing
6405 * this is that the memory allocation code may think that 1 more UV is
6406 * being used than actually is, and so might do an unnecessary grow. That
6407 * seems worth not bothering to make this the precise amount.
6409 * Note that when inverting, SvCUR shouldn't change */
6412 PERL_STATIC_INLINE UV
6413 S_invlist_max(pTHX_ SV* const invlist)
6415 /* Returns the maximum number of elements storable in the inversion list's
6416 * array, without having to realloc() */
6418 PERL_ARGS_ASSERT_INVLIST_MAX;
6420 return FROM_INTERNAL_SIZE(SvLEN(invlist));
6423 PERL_STATIC_INLINE UV*
6424 S_get_invlist_zero_addr(pTHX_ SV* invlist)
6426 /* Return the address of the UV that is reserved to hold 0 if the inversion
6427 * list contains 0. This has to be the last element of the heading, as the
6428 * list proper starts with either it if 0, or the next element if not.
6429 * (But we force it to contain either 0 or 1) */
6431 PERL_ARGS_ASSERT_GET_INVLIST_ZERO_ADDR;
6433 return (UV *) (SvPVX(invlist) + (INVLIST_ZERO_OFFSET * sizeof (UV)));
6436 #ifndef PERL_IN_XSUB_RE
6438 Perl__new_invlist(pTHX_ IV initial_size)
6441 /* Return a pointer to a newly constructed inversion list, with enough
6442 * space to store 'initial_size' elements. If that number is negative, a
6443 * system default is used instead */
6447 if (initial_size < 0) {
6448 initial_size = INVLIST_INITIAL_LEN;
6451 /* Allocate the initial space */
6452 new_list = newSV(TO_INTERNAL_SIZE(initial_size));
6453 invlist_set_len(new_list, 0);
6455 /* Force iterinit() to be used to get iteration to work */
6456 *get_invlist_iter_addr(new_list) = UV_MAX;
6458 /* This should force a segfault if a method doesn't initialize this
6460 *get_invlist_zero_addr(new_list) = UV_MAX;
6462 *get_invlist_version_id_addr(new_list) = INVLIST_VERSION_ID;
6463 #if HEADER_LENGTH != 4
6464 # error Need to regenerate VERSION_ID by running perl -E 'say int(rand 2**31-1)', and then changing the #if to the new length
6472 S__new_invlist_C_array(pTHX_ UV* list)
6474 /* Return a pointer to a newly constructed inversion list, initialized to
6475 * point to <list>, which has to be in the exact correct inversion list
6476 * form, including internal fields. Thus this is a dangerous routine that
6477 * should not be used in the wrong hands */
6479 SV* invlist = newSV_type(SVt_PV);
6481 PERL_ARGS_ASSERT__NEW_INVLIST_C_ARRAY;
6483 SvPV_set(invlist, (char *) list);
6484 SvLEN_set(invlist, 0); /* Means we own the contents, and the system
6485 shouldn't touch it */
6486 SvCUR_set(invlist, TO_INTERNAL_SIZE(invlist_len(invlist)));
6488 if (*get_invlist_version_id_addr(invlist) != INVLIST_VERSION_ID) {
6489 Perl_croak(aTHX_ "panic: Incorrect version for previously generated inversion list");
6496 S_invlist_extend(pTHX_ SV* const invlist, const UV new_max)
6498 /* Grow the maximum size of an inversion list */
6500 PERL_ARGS_ASSERT_INVLIST_EXTEND;
6502 SvGROW((SV *)invlist, TO_INTERNAL_SIZE(new_max));
6505 PERL_STATIC_INLINE void
6506 S_invlist_trim(pTHX_ SV* const invlist)
6508 PERL_ARGS_ASSERT_INVLIST_TRIM;
6510 /* Change the length of the inversion list to how many entries it currently
6513 SvPV_shrink_to_cur((SV *) invlist);
6516 /* An element is in an inversion list iff its index is even numbered: 0, 2, 4,
6518 #define ELEMENT_RANGE_MATCHES_INVLIST(i) (! ((i) & 1))
6519 #define PREV_RANGE_MATCHES_INVLIST(i) (! ELEMENT_RANGE_MATCHES_INVLIST(i))
6521 #define _invlist_union_complement_2nd(a, b, output) _invlist_union_maybe_complement_2nd(a, b, TRUE, output)
6523 #ifndef PERL_IN_XSUB_RE
6525 Perl__append_range_to_invlist(pTHX_ SV* const invlist, const UV start, const UV end)
6527 /* Subject to change or removal. Append the range from 'start' to 'end' at
6528 * the end of the inversion list. The range must be above any existing
6532 UV max = invlist_max(invlist);
6533 UV len = invlist_len(invlist);
6535 PERL_ARGS_ASSERT__APPEND_RANGE_TO_INVLIST;
6537 if (len == 0) { /* Empty lists must be initialized */
6538 array = _invlist_array_init(invlist, start == 0);
6541 /* Here, the existing list is non-empty. The current max entry in the
6542 * list is generally the first value not in the set, except when the
6543 * set extends to the end of permissible values, in which case it is
6544 * the first entry in that final set, and so this call is an attempt to
6545 * append out-of-order */
6547 UV final_element = len - 1;
6548 array = invlist_array(invlist);
6549 if (array[final_element] > start
6550 || ELEMENT_RANGE_MATCHES_INVLIST(final_element))
6552 Perl_croak(aTHX_ "panic: attempting to append to an inversion list, but wasn't at the end of the list, final=%"UVuf", start=%"UVuf", match=%c",
6553 array[final_element], start,
6554 ELEMENT_RANGE_MATCHES_INVLIST(final_element) ? 't' : 'f');
6557 /* Here, it is a legal append. If the new range begins with the first
6558 * value not in the set, it is extending the set, so the new first
6559 * value not in the set is one greater than the newly extended range.
6561 if (array[final_element] == start) {
6562 if (end != UV_MAX) {
6563 array[final_element] = end + 1;
6566 /* But if the end is the maximum representable on the machine,
6567 * just let the range that this would extend to have no end */
6568 invlist_set_len(invlist, len - 1);
6574 /* Here the new range doesn't extend any existing set. Add it */
6576 len += 2; /* Includes an element each for the start and end of range */
6578 /* If overflows the existing space, extend, which may cause the array to be
6581 invlist_extend(invlist, len);
6582 invlist_set_len(invlist, len); /* Have to set len here to avoid assert
6583 failure in invlist_array() */
6584 array = invlist_array(invlist);
6587 invlist_set_len(invlist, len);
6590 /* The next item on the list starts the range, the one after that is
6591 * one past the new range. */
6592 array[len - 2] = start;
6593 if (end != UV_MAX) {
6594 array[len - 1] = end + 1;
6597 /* But if the end is the maximum representable on the machine, just let
6598 * the range have no end */
6599 invlist_set_len(invlist, len - 1);
6604 S_invlist_search(pTHX_ SV* const invlist, const UV cp)
6606 /* Searches the inversion list for the entry that contains the input code
6607 * point <cp>. If <cp> is not in the list, -1 is returned. Otherwise, the
6608 * return value is the index into the list's array of the range that
6612 IV high = invlist_len(invlist);
6613 const UV * const array = invlist_array(invlist);
6615 PERL_ARGS_ASSERT_INVLIST_SEARCH;
6617 /* If list is empty or the code point is before the first element, return
6619 if (high == 0 || cp < array[0]) {
6623 /* Binary search. What we are looking for is <i> such that
6624 * array[i] <= cp < array[i+1]
6625 * The loop below converges on the i+1. */
6626 while (low < high) {
6627 IV mid = (low + high) / 2;
6628 if (array[mid] <= cp) {
6631 /* We could do this extra test to exit the loop early.
6632 if (cp < array[low]) {
6637 else { /* cp < array[mid] */
6646 Perl__invlist_populate_swatch(pTHX_ SV* const invlist, const UV start, const UV end, U8* swatch)
6648 /* populates a swatch of a swash the same way swatch_get() does in utf8.c,
6649 * but is used when the swash has an inversion list. This makes this much
6650 * faster, as it uses a binary search instead of a linear one. This is
6651 * intimately tied to that function, and perhaps should be in utf8.c,
6652 * except it is intimately tied to inversion lists as well. It assumes
6653 * that <swatch> is all 0's on input */
6656 const IV len = invlist_len(invlist);
6660 PERL_ARGS_ASSERT__INVLIST_POPULATE_SWATCH;
6662 if (len == 0) { /* Empty inversion list */
6666 array = invlist_array(invlist);
6668 /* Find which element it is */
6669 i = invlist_search(invlist, start);
6671 /* We populate from <start> to <end> */
6672 while (current < end) {
6675 /* The inversion list gives the results for every possible code point
6676 * after the first one in the list. Only those ranges whose index is
6677 * even are ones that the inversion list matches. For the odd ones,
6678 * and if the initial code point is not in the list, we have to skip
6679 * forward to the next element */
6680 if (i == -1 || ! ELEMENT_RANGE_MATCHES_INVLIST(i)) {
6682 if (i >= len) { /* Finished if beyond the end of the array */
6686 if (current >= end) { /* Finished if beyond the end of what we
6691 assert(current >= start);
6693 /* The current range ends one below the next one, except don't go past
6696 upper = (i < len && array[i] < end) ? array[i] : end;
6698 /* Here we are in a range that matches. Populate a bit in the 3-bit U8
6699 * for each code point in it */
6700 for (; current < upper; current++) {
6701 const STRLEN offset = (STRLEN)(current - start);
6702 swatch[offset >> 3] |= 1 << (offset & 7);
6705 /* Quit if at the end of the list */
6708 /* But first, have to deal with the highest possible code point on
6709 * the platform. The previous code assumes that <end> is one
6710 * beyond where we want to populate, but that is impossible at the
6711 * platform's infinity, so have to handle it specially */
6712 if (UNLIKELY(end == UV_MAX && ELEMENT_RANGE_MATCHES_INVLIST(len-1)))
6714 const STRLEN offset = (STRLEN)(end - start);
6715 swatch[offset >> 3] |= 1 << (offset & 7);
6720 /* Advance to the next range, which will be for code points not in the
6730 Perl__invlist_union_maybe_complement_2nd(pTHX_ SV* const a, SV* const b, bool complement_b, SV** output)
6732 /* Take the union of two inversion lists and point <output> to it. *output
6733 * should be defined upon input, and if it points to one of the two lists,
6734 * the reference count to that list will be decremented. The first list,
6735 * <a>, may be NULL, in which case a copy of the second list is returned.
6736 * If <complement_b> is TRUE, the union is taken of the complement
6737 * (inversion) of <b> instead of b itself.
6739 * The basis for this comes from "Unicode Demystified" Chapter 13 by
6740 * Richard Gillam, published by Addison-Wesley, and explained at some
6741 * length there. The preface says to incorporate its examples into your
6742 * code at your own risk.
6744 * The algorithm is like a merge sort.
6746 * XXX A potential performance improvement is to keep track as we go along
6747 * if only one of the inputs contributes to the result, meaning the other
6748 * is a subset of that one. In that case, we can skip the final copy and
6749 * return the larger of the input lists, but then outside code might need
6750 * to keep track of whether to free the input list or not */
6752 UV* array_a; /* a's array */
6754 UV len_a; /* length of a's array */
6757 SV* u; /* the resulting union */
6761 UV i_a = 0; /* current index into a's array */
6765 /* running count, as explained in the algorithm source book; items are
6766 * stopped accumulating and are output when the count changes to/from 0.
6767 * The count is incremented when we start a range that's in the set, and
6768 * decremented when we start a range that's not in the set. So its range
6769 * is 0 to 2. Only when the count is zero is something not in the set.
6773 PERL_ARGS_ASSERT__INVLIST_UNION_MAYBE_COMPLEMENT_2ND;
6776 /* If either one is empty, the union is the other one */
6777 if (a == NULL || ((len_a = invlist_len(a)) == 0)) {
6784 *output = invlist_clone(b);
6786 _invlist_invert(*output);
6788 } /* else *output already = b; */
6791 else if ((len_b = invlist_len(b)) == 0) {
6796 /* The complement of an empty list is a list that has everything in it,
6797 * so the union with <a> includes everything too */
6802 *output = _new_invlist(1);
6803 _append_range_to_invlist(*output, 0, UV_MAX);
6805 else if (*output != a) {
6806 *output = invlist_clone(a);
6808 /* else *output already = a; */
6812 /* Here both lists exist and are non-empty */
6813 array_a = invlist_array(a);
6814 array_b = invlist_array(b);
6816 /* If are to take the union of 'a' with the complement of b, set it
6817 * up so are looking at b's complement. */
6820 /* To complement, we invert: if the first element is 0, remove it. To
6821 * do this, we just pretend the array starts one later, and clear the
6822 * flag as we don't have to do anything else later */
6823 if (array_b[0] == 0) {
6826 complement_b = FALSE;
6830 /* But if the first element is not zero, we unshift a 0 before the
6831 * array. The data structure reserves a space for that 0 (which
6832 * should be a '1' right now), so physical shifting is unneeded,
6833 * but temporarily change that element to 0. Before exiting the
6834 * routine, we must restore the element to '1' */
6841 /* Size the union for the worst case: that the sets are completely
6843 u = _new_invlist(len_a + len_b);
6845 /* Will contain U+0000 if either component does */
6846 array_u = _invlist_array_init(u, (len_a > 0 && array_a[0] == 0)
6847 || (len_b > 0 && array_b[0] == 0));
6849 /* Go through each list item by item, stopping when exhausted one of
6851 while (i_a < len_a && i_b < len_b) {
6852 UV cp; /* The element to potentially add to the union's array */
6853 bool cp_in_set; /* is it in the the input list's set or not */
6855 /* We need to take one or the other of the two inputs for the union.
6856 * Since we are merging two sorted lists, we take the smaller of the
6857 * next items. In case of a tie, we take the one that is in its set
6858 * first. If we took one not in the set first, it would decrement the
6859 * count, possibly to 0 which would cause it to be output as ending the
6860 * range, and the next time through we would take the same number, and
6861 * output it again as beginning the next range. By doing it the
6862 * opposite way, there is no possibility that the count will be
6863 * momentarily decremented to 0, and thus the two adjoining ranges will
6864 * be seamlessly merged. (In a tie and both are in the set or both not
6865 * in the set, it doesn't matter which we take first.) */
6866 if (array_a[i_a] < array_b[i_b]
6867 || (array_a[i_a] == array_b[i_b]
6868 && ELEMENT_RANGE_MATCHES_INVLIST(i_a)))
6870 cp_in_set = ELEMENT_RANGE_MATCHES_INVLIST(i_a);
6874 cp_in_set = ELEMENT_RANGE_MATCHES_INVLIST(i_b);
6878 /* Here, have chosen which of the two inputs to look at. Only output
6879 * if the running count changes to/from 0, which marks the
6880 * beginning/end of a range in that's in the set */
6883 array_u[i_u++] = cp;
6890 array_u[i_u++] = cp;
6895 /* Here, we are finished going through at least one of the lists, which
6896 * means there is something remaining in at most one. We check if the list
6897 * that hasn't been exhausted is positioned such that we are in the middle
6898 * of a range in its set or not. (i_a and i_b point to the element beyond
6899 * the one we care about.) If in the set, we decrement 'count'; if 0, there
6900 * is potentially more to output.
6901 * There are four cases:
6902 * 1) Both weren't in their sets, count is 0, and remains 0. What's left
6903 * in the union is entirely from the non-exhausted set.
6904 * 2) Both were in their sets, count is 2. Nothing further should
6905 * be output, as everything that remains will be in the exhausted
6906 * list's set, hence in the union; decrementing to 1 but not 0 insures
6908 * 3) the exhausted was in its set, non-exhausted isn't, count is 1.
6909 * Nothing further should be output because the union includes
6910 * everything from the exhausted set. Not decrementing ensures that.
6911 * 4) the exhausted wasn't in its set, non-exhausted is, count is 1;
6912 * decrementing to 0 insures that we look at the remainder of the
6913 * non-exhausted set */
6914 if ((i_a != len_a && PREV_RANGE_MATCHES_INVLIST(i_a))
6915 || (i_b != len_b && PREV_RANGE_MATCHES_INVLIST(i_b)))
6920 /* The final length is what we've output so far, plus what else is about to
6921 * be output. (If 'count' is non-zero, then the input list we exhausted
6922 * has everything remaining up to the machine's limit in its set, and hence
6923 * in the union, so there will be no further output. */
6926 /* At most one of the subexpressions will be non-zero */
6927 len_u += (len_a - i_a) + (len_b - i_b);
6930 /* Set result to final length, which can change the pointer to array_u, so
6932 if (len_u != invlist_len(u)) {
6933 invlist_set_len(u, len_u);
6935 array_u = invlist_array(u);
6938 /* When 'count' is 0, the list that was exhausted (if one was shorter than
6939 * the other) ended with everything above it not in its set. That means
6940 * that the remaining part of the union is precisely the same as the
6941 * non-exhausted list, so can just copy it unchanged. (If both list were
6942 * exhausted at the same time, then the operations below will be both 0.)
6945 IV copy_count; /* At most one will have a non-zero copy count */
6946 if ((copy_count = len_a - i_a) > 0) {
6947 Copy(array_a + i_a, array_u + i_u, copy_count, UV);
6949 else if ((copy_count = len_b - i_b) > 0) {
6950 Copy(array_b + i_b, array_u + i_u, copy_count, UV);
6954 /* We may be removing a reference to one of the inputs */
6955 if (a == *output || b == *output) {
6956 SvREFCNT_dec(*output);
6959 /* If we've changed b, restore it */
6969 Perl__invlist_intersection_maybe_complement_2nd(pTHX_ SV* const a, SV* const b, bool complement_b, SV** i)
6971 /* Take the intersection of two inversion lists and point <i> to it. *i
6972 * should be defined upon input, and if it points to one of the two lists,
6973 * the reference count to that list will be decremented.
6974 * If <complement_b> is TRUE, the result will be the intersection of <a>
6975 * and the complement (or inversion) of <b> instead of <b> directly.
6977 * The basis for this comes from "Unicode Demystified" Chapter 13 by
6978 * Richard Gillam, published by Addison-Wesley, and explained at some
6979 * length there. The preface says to incorporate its examples into your
6980 * code at your own risk. In fact, it had bugs
6982 * The algorithm is like a merge sort, and is essentially the same as the
6986 UV* array_a; /* a's array */
6988 UV len_a; /* length of a's array */
6991 SV* r; /* the resulting intersection */
6995 UV i_a = 0; /* current index into a's array */
6999 /* running count, as explained in the algorithm source book; items are
7000 * stopped accumulating and are output when the count changes to/from 2.
7001 * The count is incremented when we start a range that's in the set, and
7002 * decremented when we start a range that's not in the set. So its range
7003 * is 0 to 2. Only when the count is 2 is something in the intersection.
7007 PERL_ARGS_ASSERT__INVLIST_INTERSECTION_MAYBE_COMPLEMENT_2ND;
7010 /* Special case if either one is empty */
7011 len_a = invlist_len(a);
7012 if ((len_a == 0) || ((len_b = invlist_len(b)) == 0)) {
7014 if (len_a != 0 && complement_b) {
7016 /* Here, 'a' is not empty, therefore from the above 'if', 'b' must
7017 * be empty. Here, also we are using 'b's complement, which hence
7018 * must be every possible code point. Thus the intersection is
7021 *i = invlist_clone(a);
7027 /* else *i is already 'a' */
7031 /* Here, 'a' or 'b' is empty and not using the complement of 'b'. The
7032 * intersection must be empty */
7039 *i = _new_invlist(0);
7043 /* Here both lists exist and are non-empty */
7044 array_a = invlist_array(a);
7045 array_b = invlist_array(b);
7047 /* If are to take the intersection of 'a' with the complement of b, set it
7048 * up so are looking at b's complement. */
7051 /* To complement, we invert: if the first element is 0, remove it. To
7052 * do this, we just pretend the array starts one later, and clear the
7053 * flag as we don't have to do anything else later */
7054 if (array_b[0] == 0) {
7057 complement_b = FALSE;
7061 /* But if the first element is not zero, we unshift a 0 before the
7062 * array. The data structure reserves a space for that 0 (which
7063 * should be a '1' right now), so physical shifting is unneeded,
7064 * but temporarily change that element to 0. Before exiting the
7065 * routine, we must restore the element to '1' */
7072 /* Size the intersection for the worst case: that the intersection ends up
7073 * fragmenting everything to be completely disjoint */
7074 r= _new_invlist(len_a + len_b);
7076 /* Will contain U+0000 iff both components do */
7077 array_r = _invlist_array_init(r, len_a > 0 && array_a[0] == 0
7078 && len_b > 0 && array_b[0] == 0);
7080 /* Go through each list item by item, stopping when exhausted one of
7082 while (i_a < len_a && i_b < len_b) {
7083 UV cp; /* The element to potentially add to the intersection's
7085 bool cp_in_set; /* Is it in the input list's set or not */
7087 /* We need to take one or the other of the two inputs for the
7088 * intersection. Since we are merging two sorted lists, we take the
7089 * smaller of the next items. In case of a tie, we take the one that
7090 * is not in its set first (a difference from the union algorithm). If
7091 * we took one in the set first, it would increment the count, possibly
7092 * to 2 which would cause it to be output as starting a range in the
7093 * intersection, and the next time through we would take that same
7094 * number, and output it again as ending the set. By doing it the
7095 * opposite of this, there is no possibility that the count will be
7096 * momentarily incremented to 2. (In a tie and both are in the set or
7097 * both not in the set, it doesn't matter which we take first.) */
7098 if (array_a[i_a] < array_b[i_b]
7099 || (array_a[i_a] == array_b[i_b]
7100 && ! ELEMENT_RANGE_MATCHES_INVLIST(i_a)))
7102 cp_in_set = ELEMENT_RANGE_MATCHES_INVLIST(i_a);
7106 cp_in_set = ELEMENT_RANGE_MATCHES_INVLIST(i_b);
7110 /* Here, have chosen which of the two inputs to look at. Only output
7111 * if the running count changes to/from 2, which marks the
7112 * beginning/end of a range that's in the intersection */
7116 array_r[i_r++] = cp;
7121 array_r[i_r++] = cp;
7127 /* Here, we are finished going through at least one of the lists, which
7128 * means there is something remaining in at most one. We check if the list
7129 * that has been exhausted is positioned such that we are in the middle
7130 * of a range in its set or not. (i_a and i_b point to elements 1 beyond
7131 * the ones we care about.) There are four cases:
7132 * 1) Both weren't in their sets, count is 0, and remains 0. There's
7133 * nothing left in the intersection.
7134 * 2) Both were in their sets, count is 2 and perhaps is incremented to
7135 * above 2. What should be output is exactly that which is in the
7136 * non-exhausted set, as everything it has is also in the intersection
7137 * set, and everything it doesn't have can't be in the intersection
7138 * 3) The exhausted was in its set, non-exhausted isn't, count is 1, and
7139 * gets incremented to 2. Like the previous case, the intersection is
7140 * everything that remains in the non-exhausted set.
7141 * 4) the exhausted wasn't in its set, non-exhausted is, count is 1, and
7142 * remains 1. And the intersection has nothing more. */
7143 if ((i_a == len_a && PREV_RANGE_MATCHES_INVLIST(i_a))
7144 || (i_b == len_b && PREV_RANGE_MATCHES_INVLIST(i_b)))
7149 /* The final length is what we've output so far plus what else is in the
7150 * intersection. At most one of the subexpressions below will be non-zero */
7153 len_r += (len_a - i_a) + (len_b - i_b);
7156 /* Set result to final length, which can change the pointer to array_r, so
7158 if (len_r != invlist_len(r)) {
7159 invlist_set_len(r, len_r);
7161 array_r = invlist_array(r);
7164 /* Finish outputting any remaining */
7165 if (count >= 2) { /* At most one will have a non-zero copy count */
7167 if ((copy_count = len_a - i_a) > 0) {
7168 Copy(array_a + i_a, array_r + i_r, copy_count, UV);
7170 else if ((copy_count = len_b - i_b) > 0) {
7171 Copy(array_b + i_b, array_r + i_r, copy_count, UV);
7175 /* We may be removing a reference to one of the inputs */
7176 if (a == *i || b == *i) {
7180 /* If we've changed b, restore it */
7192 S_add_range_to_invlist(pTHX_ SV* invlist, const UV start, const UV end)
7194 /* Add the range from 'start' to 'end' inclusive to the inversion list's
7195 * set. A pointer to the inversion list is returned. This may actually be
7196 * a new list, in which case the passed in one has been destroyed. The
7197 * passed in inversion list can be NULL, in which case a new one is created
7198 * with just the one range in it */
7203 if (invlist == NULL) {
7204 invlist = _new_invlist(2);
7208 len = invlist_len(invlist);
7211 /* If comes after the final entry, can just append it to the end */
7213 || start >= invlist_array(invlist)
7214 [invlist_len(invlist) - 1])
7216 _append_range_to_invlist(invlist, start, end);
7220 /* Here, can't just append things, create and return a new inversion list
7221 * which is the union of this range and the existing inversion list */
7222 range_invlist = _new_invlist(2);
7223 _append_range_to_invlist(range_invlist, start, end);
7225 _invlist_union(invlist, range_invlist, &invlist);
7227 /* The temporary can be freed */
7228 SvREFCNT_dec(range_invlist);
7233 PERL_STATIC_INLINE SV*
7234 S_add_cp_to_invlist(pTHX_ SV* invlist, const UV cp) {
7235 return add_range_to_invlist(invlist, cp, cp);
7238 #ifndef PERL_IN_XSUB_RE
7240 Perl__invlist_invert(pTHX_ SV* const invlist)
7242 /* Complement the input inversion list. This adds a 0 if the list didn't
7243 * have a zero; removes it otherwise. As described above, the data
7244 * structure is set up so that this is very efficient */
7246 UV* len_pos = get_invlist_len_addr(invlist);
7248 PERL_ARGS_ASSERT__INVLIST_INVERT;
7250 /* The inverse of matching nothing is matching everything */
7251 if (*len_pos == 0) {
7252 _append_range_to_invlist(invlist, 0, UV_MAX);
7256 /* The exclusive or complents 0 to 1; and 1 to 0. If the result is 1, the
7257 * zero element was a 0, so it is being removed, so the length decrements
7258 * by 1; and vice-versa. SvCUR is unaffected */
7259 if (*get_invlist_zero_addr(invlist) ^= 1) {
7268 Perl__invlist_invert_prop(pTHX_ SV* const invlist)
7270 /* Complement the input inversion list (which must be a Unicode property,
7271 * all of which don't match above the Unicode maximum code point.) And
7272 * Perl has chosen to not have the inversion match above that either. This
7273 * adds a 0x110000 if the list didn't end with it, and removes it if it did
7279 PERL_ARGS_ASSERT__INVLIST_INVERT_PROP;
7281 _invlist_invert(invlist);
7283 len = invlist_len(invlist);
7285 if (len != 0) { /* If empty do nothing */
7286 array = invlist_array(invlist);
7287 if (array[len - 1] != PERL_UNICODE_MAX + 1) {
7288 /* Add 0x110000. First, grow if necessary */
7290 if (invlist_max(invlist) < len) {
7291 invlist_extend(invlist, len);
7292 array = invlist_array(invlist);
7294 invlist_set_len(invlist, len);
7295 array[len - 1] = PERL_UNICODE_MAX + 1;
7297 else { /* Remove the 0x110000 */
7298 invlist_set_len(invlist, len - 1);
7306 PERL_STATIC_INLINE SV*
7307 S_invlist_clone(pTHX_ SV* const invlist)
7310 /* Return a new inversion list that is a copy of the input one, which is
7313 /* Need to allocate extra space to accommodate Perl's addition of a
7314 * trailing NUL to SvPV's, since it thinks they are always strings */
7315 SV* new_invlist = _new_invlist(invlist_len(invlist) + 1);
7316 STRLEN length = SvCUR(invlist);
7318 PERL_ARGS_ASSERT_INVLIST_CLONE;
7320 SvCUR_set(new_invlist, length); /* This isn't done automatically */
7321 Copy(SvPVX(invlist), SvPVX(new_invlist), length, char);
7326 PERL_STATIC_INLINE UV*
7327 S_get_invlist_iter_addr(pTHX_ SV* invlist)
7329 /* Return the address of the UV that contains the current iteration
7332 PERL_ARGS_ASSERT_GET_INVLIST_ITER_ADDR;
7334 return (UV *) (SvPVX(invlist) + (INVLIST_ITER_OFFSET * sizeof (UV)));
7337 PERL_STATIC_INLINE UV*
7338 S_get_invlist_version_id_addr(pTHX_ SV* invlist)
7340 /* Return the address of the UV that contains the version id. */
7342 PERL_ARGS_ASSERT_GET_INVLIST_VERSION_ID_ADDR;
7344 return (UV *) (SvPVX(invlist) + (INVLIST_VERSION_ID_OFFSET * sizeof (UV)));
7347 PERL_STATIC_INLINE void
7348 S_invlist_iterinit(pTHX_ SV* invlist) /* Initialize iterator for invlist */
7350 PERL_ARGS_ASSERT_INVLIST_ITERINIT;
7352 *get_invlist_iter_addr(invlist) = 0;
7356 S_invlist_iternext(pTHX_ SV* invlist, UV* start, UV* end)
7358 /* An C<invlist_iterinit> call on <invlist> must be used to set this up.
7359 * This call sets in <*start> and <*end>, the next range in <invlist>.
7360 * Returns <TRUE> if successful and the next call will return the next
7361 * range; <FALSE> if was already at the end of the list. If the latter,
7362 * <*start> and <*end> are unchanged, and the next call to this function
7363 * will start over at the beginning of the list */
7365 UV* pos = get_invlist_iter_addr(invlist);
7366 UV len = invlist_len(invlist);
7369 PERL_ARGS_ASSERT_INVLIST_ITERNEXT;
7372 *pos = UV_MAX; /* Force iternit() to be required next time */
7376 array = invlist_array(invlist);
7378 *start = array[(*pos)++];
7384 *end = array[(*pos)++] - 1;
7390 #ifndef PERL_IN_XSUB_RE
7392 Perl__invlist_contents(pTHX_ SV* const invlist)
7394 /* Get the contents of an inversion list into a string SV so that they can
7395 * be printed out. It uses the format traditionally done for debug tracing
7399 SV* output = newSVpvs("\n");
7401 PERL_ARGS_ASSERT__INVLIST_CONTENTS;
7403 invlist_iterinit(invlist);
7404 while (invlist_iternext(invlist, &start, &end)) {
7405 if (end == UV_MAX) {
7406 Perl_sv_catpvf(aTHX_ output, "%04"UVXf"\tINFINITY\n", start);
7408 else if (end != start) {
7409 Perl_sv_catpvf(aTHX_ output, "%04"UVXf"\t%04"UVXf"\n",
7413 Perl_sv_catpvf(aTHX_ output, "%04"UVXf"\n", start);
7423 S_invlist_dump(pTHX_ SV* const invlist, const char * const header)
7425 /* Dumps out the ranges in an inversion list. The string 'header'
7426 * if present is output on a line before the first range */
7430 if (header && strlen(header)) {
7431 PerlIO_printf(Perl_debug_log, "%s\n", header);
7433 invlist_iterinit(invlist);
7434 while (invlist_iternext(invlist, &start, &end)) {
7435 if (end == UV_MAX) {
7436 PerlIO_printf(Perl_debug_log, "0x%04"UVXf" .. INFINITY\n", start);
7439 PerlIO_printf(Perl_debug_log, "0x%04"UVXf" .. 0x%04"UVXf"\n", start, end);
7445 #undef HEADER_LENGTH
7446 #undef INVLIST_INITIAL_LENGTH
7447 #undef TO_INTERNAL_SIZE
7448 #undef FROM_INTERNAL_SIZE
7449 #undef INVLIST_LEN_OFFSET
7450 #undef INVLIST_ZERO_OFFSET
7451 #undef INVLIST_ITER_OFFSET
7452 #undef INVLIST_VERSION_ID
7454 /* End of inversion list object */
7457 - reg - regular expression, i.e. main body or parenthesized thing
7459 * Caller must absorb opening parenthesis.
7461 * Combining parenthesis handling with the base level of regular expression
7462 * is a trifle forced, but the need to tie the tails of the branches to what
7463 * follows makes it hard to avoid.
7465 #define REGTAIL(x,y,z) regtail((x),(y),(z),depth+1)
7467 #define REGTAIL_STUDY(x,y,z) regtail_study((x),(y),(z),depth+1)
7469 #define REGTAIL_STUDY(x,y,z) regtail((x),(y),(z),depth+1)
7473 S_reg(pTHX_ RExC_state_t *pRExC_state, I32 paren, I32 *flagp,U32 depth)
7474 /* paren: Parenthesized? 0=top, 1=(, inside: changed to letter. */
7477 register regnode *ret; /* Will be the head of the group. */
7478 register regnode *br;
7479 register regnode *lastbr;
7480 register regnode *ender = NULL;
7481 register I32 parno = 0;
7483 U32 oregflags = RExC_flags;
7484 bool have_branch = 0;
7486 I32 freeze_paren = 0;
7487 I32 after_freeze = 0;
7489 /* for (?g), (?gc), and (?o) warnings; warning
7490 about (?c) will warn about (?g) -- japhy */
7492 #define WASTED_O 0x01
7493 #define WASTED_G 0x02
7494 #define WASTED_C 0x04
7495 #define WASTED_GC (0x02|0x04)
7496 I32 wastedflags = 0x00;
7498 char * parse_start = RExC_parse; /* MJD */
7499 char * const oregcomp_parse = RExC_parse;
7501 GET_RE_DEBUG_FLAGS_DECL;
7503 PERL_ARGS_ASSERT_REG;
7504 DEBUG_PARSE("reg ");
7506 *flagp = 0; /* Tentatively. */
7509 /* Make an OPEN node, if parenthesized. */
7511 if ( *RExC_parse == '*') { /* (*VERB:ARG) */
7512 char *start_verb = RExC_parse;
7513 STRLEN verb_len = 0;
7514 char *start_arg = NULL;
7515 unsigned char op = 0;
7517 int internal_argval = 0; /* internal_argval is only useful if !argok */
7518 while ( *RExC_parse && *RExC_parse != ')' ) {
7519 if ( *RExC_parse == ':' ) {
7520 start_arg = RExC_parse + 1;
7526 verb_len = RExC_parse - start_verb;
7529 while ( *RExC_parse && *RExC_parse != ')' )
7531 if ( *RExC_parse != ')' )
7532 vFAIL("Unterminated verb pattern argument");
7533 if ( RExC_parse == start_arg )
7536 if ( *RExC_parse != ')' )
7537 vFAIL("Unterminated verb pattern");
7540 switch ( *start_verb ) {
7541 case 'A': /* (*ACCEPT) */
7542 if ( memEQs(start_verb,verb_len,"ACCEPT") ) {
7544 internal_argval = RExC_nestroot;
7547 case 'C': /* (*COMMIT) */
7548 if ( memEQs(start_verb,verb_len,"COMMIT") )
7551 case 'F': /* (*FAIL) */
7552 if ( verb_len==1 || memEQs(start_verb,verb_len,"FAIL") ) {
7557 case ':': /* (*:NAME) */
7558 case 'M': /* (*MARK:NAME) */
7559 if ( verb_len==0 || memEQs(start_verb,verb_len,"MARK") ) {
7564 case 'P': /* (*PRUNE) */
7565 if ( memEQs(start_verb,verb_len,"PRUNE") )
7568 case 'S': /* (*SKIP) */
7569 if ( memEQs(start_verb,verb_len,"SKIP") )
7572 case 'T': /* (*THEN) */
7573 /* [19:06] <TimToady> :: is then */
7574 if ( memEQs(start_verb,verb_len,"THEN") ) {
7576 RExC_seen |= REG_SEEN_CUTGROUP;
7582 vFAIL3("Unknown verb pattern '%.*s'",
7583 verb_len, start_verb);
7586 if ( start_arg && internal_argval ) {
7587 vFAIL3("Verb pattern '%.*s' may not have an argument",
7588 verb_len, start_verb);
7589 } else if ( argok < 0 && !start_arg ) {
7590 vFAIL3("Verb pattern '%.*s' has a mandatory argument",
7591 verb_len, start_verb);
7593 ret = reganode(pRExC_state, op, internal_argval);
7594 if ( ! internal_argval && ! SIZE_ONLY ) {
7596 SV *sv = newSVpvn( start_arg, RExC_parse - start_arg);
7597 ARG(ret) = add_data( pRExC_state, 1, "S" );
7598 RExC_rxi->data->data[ARG(ret)]=(void*)sv;
7605 if (!internal_argval)
7606 RExC_seen |= REG_SEEN_VERBARG;
7607 } else if ( start_arg ) {
7608 vFAIL3("Verb pattern '%.*s' may not have an argument",
7609 verb_len, start_verb);
7611 ret = reg_node(pRExC_state, op);
7613 nextchar(pRExC_state);
7616 if (*RExC_parse == '?') { /* (?...) */
7617 bool is_logical = 0;
7618 const char * const seqstart = RExC_parse;
7619 bool has_use_defaults = FALSE;
7622 paren = *RExC_parse++;
7623 ret = NULL; /* For look-ahead/behind. */
7626 case 'P': /* (?P...) variants for those used to PCRE/Python */
7627 paren = *RExC_parse++;
7628 if ( paren == '<') /* (?P<...>) named capture */
7630 else if (paren == '>') { /* (?P>name) named recursion */
7631 goto named_recursion;
7633 else if (paren == '=') { /* (?P=...) named backref */
7634 /* this pretty much dupes the code for \k<NAME> in regatom(), if
7635 you change this make sure you change that */
7636 char* name_start = RExC_parse;
7638 SV *sv_dat = reg_scan_name(pRExC_state,
7639 SIZE_ONLY ? REG_RSN_RETURN_NULL : REG_RSN_RETURN_DATA);
7640 if (RExC_parse == name_start || *RExC_parse != ')')
7641 vFAIL2("Sequence %.3s... not terminated",parse_start);
7644 num = add_data( pRExC_state, 1, "S" );
7645 RExC_rxi->data->data[num]=(void*)sv_dat;
7646 SvREFCNT_inc_simple_void(sv_dat);
7649 ret = reganode(pRExC_state,
7652 : (MORE_ASCII_RESTRICTED)
7654 : (AT_LEAST_UNI_SEMANTICS)
7662 Set_Node_Offset(ret, parse_start+1);
7663 Set_Node_Cur_Length(ret); /* MJD */
7665 nextchar(pRExC_state);
7669 vFAIL3("Sequence (%.*s...) not recognized", RExC_parse-seqstart, seqstart);
7671 case '<': /* (?<...) */
7672 if (*RExC_parse == '!')
7674 else if (*RExC_parse != '=')
7680 case '\'': /* (?'...') */
7681 name_start= RExC_parse;
7682 svname = reg_scan_name(pRExC_state,
7683 SIZE_ONLY ? /* reverse test from the others */
7684 REG_RSN_RETURN_NAME :
7685 REG_RSN_RETURN_NULL);
7686 if (RExC_parse == name_start) {
7688 vFAIL3("Sequence (%.*s...) not recognized", RExC_parse-seqstart, seqstart);
7691 if (*RExC_parse != paren)
7692 vFAIL2("Sequence (?%c... not terminated",
7693 paren=='>' ? '<' : paren);
7697 if (!svname) /* shouldn't happen */
7699 "panic: reg_scan_name returned NULL");
7700 if (!RExC_paren_names) {
7701 RExC_paren_names= newHV();
7702 sv_2mortal(MUTABLE_SV(RExC_paren_names));
7704 RExC_paren_name_list= newAV();
7705 sv_2mortal(MUTABLE_SV(RExC_paren_name_list));
7708 he_str = hv_fetch_ent( RExC_paren_names, svname, 1, 0 );
7710 sv_dat = HeVAL(he_str);
7712 /* croak baby croak */
7714 "panic: paren_name hash element allocation failed");
7715 } else if ( SvPOK(sv_dat) ) {
7716 /* (?|...) can mean we have dupes so scan to check
7717 its already been stored. Maybe a flag indicating
7718 we are inside such a construct would be useful,
7719 but the arrays are likely to be quite small, so
7720 for now we punt -- dmq */
7721 IV count = SvIV(sv_dat);
7722 I32 *pv = (I32*)SvPVX(sv_dat);
7724 for ( i = 0 ; i < count ; i++ ) {
7725 if ( pv[i] == RExC_npar ) {
7731 pv = (I32*)SvGROW(sv_dat, SvCUR(sv_dat) + sizeof(I32)+1);
7732 SvCUR_set(sv_dat, SvCUR(sv_dat) + sizeof(I32));
7733 pv[count] = RExC_npar;
7734 SvIV_set(sv_dat, SvIVX(sv_dat) + 1);
7737 (void)SvUPGRADE(sv_dat,SVt_PVNV);
7738 sv_setpvn(sv_dat, (char *)&(RExC_npar), sizeof(I32));
7740 SvIV_set(sv_dat, 1);
7743 /* Yes this does cause a memory leak in debugging Perls */
7744 if (!av_store(RExC_paren_name_list, RExC_npar, SvREFCNT_inc(svname)))
7745 SvREFCNT_dec(svname);
7748 /*sv_dump(sv_dat);*/
7750 nextchar(pRExC_state);
7752 goto capturing_parens;
7754 RExC_seen |= REG_SEEN_LOOKBEHIND;
7755 RExC_in_lookbehind++;
7757 case '=': /* (?=...) */
7758 RExC_seen_zerolen++;
7760 case '!': /* (?!...) */
7761 RExC_seen_zerolen++;
7762 if (*RExC_parse == ')') {
7763 ret=reg_node(pRExC_state, OPFAIL);
7764 nextchar(pRExC_state);
7768 case '|': /* (?|...) */
7769 /* branch reset, behave like a (?:...) except that
7770 buffers in alternations share the same numbers */
7772 after_freeze = freeze_paren = RExC_npar;
7774 case ':': /* (?:...) */
7775 case '>': /* (?>...) */
7777 case '$': /* (?$...) */
7778 case '@': /* (?@...) */
7779 vFAIL2("Sequence (?%c...) not implemented", (int)paren);
7781 case '#': /* (?#...) */
7782 while (*RExC_parse && *RExC_parse != ')')
7784 if (*RExC_parse != ')')
7785 FAIL("Sequence (?#... not terminated");
7786 nextchar(pRExC_state);
7789 case '0' : /* (?0) */
7790 case 'R' : /* (?R) */
7791 if (*RExC_parse != ')')
7792 FAIL("Sequence (?R) not terminated");
7793 ret = reg_node(pRExC_state, GOSTART);
7794 *flagp |= POSTPONED;
7795 nextchar(pRExC_state);
7798 { /* named and numeric backreferences */
7800 case '&': /* (?&NAME) */
7801 parse_start = RExC_parse - 1;
7804 SV *sv_dat = reg_scan_name(pRExC_state,
7805 SIZE_ONLY ? REG_RSN_RETURN_NULL : REG_RSN_RETURN_DATA);
7806 num = sv_dat ? *((I32 *)SvPVX(sv_dat)) : 0;
7808 goto gen_recurse_regop;
7811 if (!(RExC_parse[0] >= '1' && RExC_parse[0] <= '9')) {
7813 vFAIL("Illegal pattern");
7815 goto parse_recursion;
7817 case '-': /* (?-1) */
7818 if (!(RExC_parse[0] >= '1' && RExC_parse[0] <= '9')) {
7819 RExC_parse--; /* rewind to let it be handled later */
7823 case '1': case '2': case '3': case '4': /* (?1) */
7824 case '5': case '6': case '7': case '8': case '9':
7827 num = atoi(RExC_parse);
7828 parse_start = RExC_parse - 1; /* MJD */
7829 if (*RExC_parse == '-')
7831 while (isDIGIT(*RExC_parse))
7833 if (*RExC_parse!=')')
7834 vFAIL("Expecting close bracket");
7837 if ( paren == '-' ) {
7839 Diagram of capture buffer numbering.
7840 Top line is the normal capture buffer numbers
7841 Bottom line is the negative indexing as from
7845 /(a(x)y)(a(b(c(?-2)d)e)f)(g(h))/
7849 num = RExC_npar + num;
7852 vFAIL("Reference to nonexistent group");
7854 } else if ( paren == '+' ) {
7855 num = RExC_npar + num - 1;
7858 ret = reganode(pRExC_state, GOSUB, num);
7860 if (num > (I32)RExC_rx->nparens) {
7862 vFAIL("Reference to nonexistent group");
7864 ARG2L_SET( ret, RExC_recurse_count++);
7866 DEBUG_OPTIMISE_MORE_r(PerlIO_printf(Perl_debug_log,
7867 "Recurse #%"UVuf" to %"IVdf"\n", (UV)ARG(ret), (IV)ARG2L(ret)));
7871 RExC_seen |= REG_SEEN_RECURSE;
7872 Set_Node_Length(ret, 1 + regarglen[OP(ret)]); /* MJD */
7873 Set_Node_Offset(ret, parse_start); /* MJD */
7875 *flagp |= POSTPONED;
7876 nextchar(pRExC_state);
7878 } /* named and numeric backreferences */
7881 case '?': /* (??...) */
7883 if (*RExC_parse != '{') {
7885 vFAIL3("Sequence (%.*s...) not recognized", RExC_parse-seqstart, seqstart);
7888 *flagp |= POSTPONED;
7889 paren = *RExC_parse++;
7891 case '{': /* (?{...}) */
7896 char *s = RExC_parse;
7898 RExC_seen_zerolen++;
7899 RExC_seen |= REG_SEEN_EVAL;
7900 while (count && (c = *RExC_parse)) {
7911 if (*RExC_parse != ')') {
7913 vFAIL("Sequence (?{...}) not terminated or not {}-balanced");
7917 OP_4tree *sop, *rop;
7918 SV * const sv = newSVpvn(s, RExC_parse - 1 - s);
7921 Perl_save_re_context(aTHX);
7922 rop = Perl_sv_compile_2op_is_broken(aTHX_ sv, &sop, "re", &pad);
7923 sop->op_private |= OPpREFCOUNTED;
7924 /* re_dup will OpREFCNT_inc */
7925 OpREFCNT_set(sop, 1);
7928 n = add_data(pRExC_state, 3, "nop");
7929 RExC_rxi->data->data[n] = (void*)rop;
7930 RExC_rxi->data->data[n+1] = (void*)sop;
7931 RExC_rxi->data->data[n+2] = (void*)pad;
7934 else { /* First pass */
7935 if (PL_reginterp_cnt < ++RExC_seen_evals
7937 /* No compiled RE interpolated, has runtime
7938 components ===> unsafe. */
7939 FAIL("Eval-group not allowed at runtime, use re 'eval'");
7940 if (PL_tainting && PL_tainted)
7941 FAIL("Eval-group in insecure regular expression");
7942 #if PERL_VERSION > 8
7943 if (IN_PERL_COMPILETIME)
7948 nextchar(pRExC_state);
7950 ret = reg_node(pRExC_state, LOGICAL);
7953 REGTAIL(pRExC_state, ret, reganode(pRExC_state, EVAL, n));
7954 /* deal with the length of this later - MJD */
7957 ret = reganode(pRExC_state, EVAL, n);
7958 Set_Node_Length(ret, RExC_parse - parse_start + 1);
7959 Set_Node_Offset(ret, parse_start);
7962 case '(': /* (?(?{...})...) and (?(?=...)...) */
7965 if (RExC_parse[0] == '?') { /* (?(?...)) */
7966 if (RExC_parse[1] == '=' || RExC_parse[1] == '!'
7967 || RExC_parse[1] == '<'
7968 || RExC_parse[1] == '{') { /* Lookahead or eval. */
7971 ret = reg_node(pRExC_state, LOGICAL);
7974 REGTAIL(pRExC_state, ret, reg(pRExC_state, 1, &flag,depth+1));
7978 else if ( RExC_parse[0] == '<' /* (?(<NAME>)...) */
7979 || RExC_parse[0] == '\'' ) /* (?('NAME')...) */
7981 char ch = RExC_parse[0] == '<' ? '>' : '\'';
7982 char *name_start= RExC_parse++;
7984 SV *sv_dat=reg_scan_name(pRExC_state,
7985 SIZE_ONLY ? REG_RSN_RETURN_NULL : REG_RSN_RETURN_DATA);
7986 if (RExC_parse == name_start || *RExC_parse != ch)
7987 vFAIL2("Sequence (?(%c... not terminated",
7988 (ch == '>' ? '<' : ch));
7991 num = add_data( pRExC_state, 1, "S" );
7992 RExC_rxi->data->data[num]=(void*)sv_dat;
7993 SvREFCNT_inc_simple_void(sv_dat);
7995 ret = reganode(pRExC_state,NGROUPP,num);
7996 goto insert_if_check_paren;
7998 else if (RExC_parse[0] == 'D' &&
7999 RExC_parse[1] == 'E' &&
8000 RExC_parse[2] == 'F' &&
8001 RExC_parse[3] == 'I' &&
8002 RExC_parse[4] == 'N' &&
8003 RExC_parse[5] == 'E')
8005 ret = reganode(pRExC_state,DEFINEP,0);
8008 goto insert_if_check_paren;
8010 else if (RExC_parse[0] == 'R') {
8013 if (RExC_parse[0] >= '1' && RExC_parse[0] <= '9' ) {
8014 parno = atoi(RExC_parse++);
8015 while (isDIGIT(*RExC_parse))
8017 } else if (RExC_parse[0] == '&') {
8020 sv_dat = reg_scan_name(pRExC_state,
8021 SIZE_ONLY ? REG_RSN_RETURN_NULL : REG_RSN_RETURN_DATA);
8022 parno = sv_dat ? *((I32 *)SvPVX(sv_dat)) : 0;
8024 ret = reganode(pRExC_state,INSUBP,parno);
8025 goto insert_if_check_paren;
8027 else if (RExC_parse[0] >= '1' && RExC_parse[0] <= '9' ) {
8030 parno = atoi(RExC_parse++);
8032 while (isDIGIT(*RExC_parse))
8034 ret = reganode(pRExC_state, GROUPP, parno);
8036 insert_if_check_paren:
8037 if ((c = *nextchar(pRExC_state)) != ')')
8038 vFAIL("Switch condition not recognized");
8040 REGTAIL(pRExC_state, ret, reganode(pRExC_state, IFTHEN, 0));
8041 br = regbranch(pRExC_state, &flags, 1,depth+1);
8043 br = reganode(pRExC_state, LONGJMP, 0);
8045 REGTAIL(pRExC_state, br, reganode(pRExC_state, LONGJMP, 0));
8046 c = *nextchar(pRExC_state);
8051 vFAIL("(?(DEFINE)....) does not allow branches");
8052 lastbr = reganode(pRExC_state, IFTHEN, 0); /* Fake one for optimizer. */
8053 regbranch(pRExC_state, &flags, 1,depth+1);
8054 REGTAIL(pRExC_state, ret, lastbr);
8057 c = *nextchar(pRExC_state);
8062 vFAIL("Switch (?(condition)... contains too many branches");
8063 ender = reg_node(pRExC_state, TAIL);
8064 REGTAIL(pRExC_state, br, ender);
8066 REGTAIL(pRExC_state, lastbr, ender);
8067 REGTAIL(pRExC_state, NEXTOPER(NEXTOPER(lastbr)), ender);
8070 REGTAIL(pRExC_state, ret, ender);
8071 RExC_size++; /* XXX WHY do we need this?!!
8072 For large programs it seems to be required
8073 but I can't figure out why. -- dmq*/
8077 vFAIL2("Unknown switch condition (?(%.2s", RExC_parse);
8081 RExC_parse--; /* for vFAIL to print correctly */
8082 vFAIL("Sequence (? incomplete");
8084 case DEFAULT_PAT_MOD: /* Use default flags with the exceptions
8086 has_use_defaults = TRUE;
8087 STD_PMMOD_FLAGS_CLEAR(&RExC_flags);
8088 set_regex_charset(&RExC_flags, (RExC_utf8 || RExC_uni_semantics)
8089 ? REGEX_UNICODE_CHARSET
8090 : REGEX_DEPENDS_CHARSET);
8094 parse_flags: /* (?i) */
8096 U32 posflags = 0, negflags = 0;
8097 U32 *flagsp = &posflags;
8098 char has_charset_modifier = '\0';
8099 regex_charset cs = get_regex_charset(RExC_flags);
8100 if (cs == REGEX_DEPENDS_CHARSET
8101 && (RExC_utf8 || RExC_uni_semantics))
8103 cs = REGEX_UNICODE_CHARSET;
8106 while (*RExC_parse) {
8107 /* && strchr("iogcmsx", *RExC_parse) */
8108 /* (?g), (?gc) and (?o) are useless here
8109 and must be globally applied -- japhy */
8110 switch (*RExC_parse) {
8111 CASE_STD_PMMOD_FLAGS_PARSE_SET(flagsp);
8112 case LOCALE_PAT_MOD:
8113 if (has_charset_modifier) {
8114 goto excess_modifier;
8116 else if (flagsp == &negflags) {
8119 cs = REGEX_LOCALE_CHARSET;
8120 has_charset_modifier = LOCALE_PAT_MOD;
8121 RExC_contains_locale = 1;
8123 case UNICODE_PAT_MOD:
8124 if (has_charset_modifier) {
8125 goto excess_modifier;
8127 else if (flagsp == &negflags) {
8130 cs = REGEX_UNICODE_CHARSET;
8131 has_charset_modifier = UNICODE_PAT_MOD;
8133 case ASCII_RESTRICT_PAT_MOD:
8134 if (flagsp == &negflags) {
8137 if (has_charset_modifier) {
8138 if (cs != REGEX_ASCII_RESTRICTED_CHARSET) {
8139 goto excess_modifier;
8141 /* Doubled modifier implies more restricted */
8142 cs = REGEX_ASCII_MORE_RESTRICTED_CHARSET;
8145 cs = REGEX_ASCII_RESTRICTED_CHARSET;
8147 has_charset_modifier = ASCII_RESTRICT_PAT_MOD;
8149 case DEPENDS_PAT_MOD:
8150 if (has_use_defaults) {
8151 goto fail_modifiers;
8153 else if (flagsp == &negflags) {
8156 else if (has_charset_modifier) {
8157 goto excess_modifier;
8160 /* The dual charset means unicode semantics if the
8161 * pattern (or target, not known until runtime) are
8162 * utf8, or something in the pattern indicates unicode
8164 cs = (RExC_utf8 || RExC_uni_semantics)
8165 ? REGEX_UNICODE_CHARSET
8166 : REGEX_DEPENDS_CHARSET;
8167 has_charset_modifier = DEPENDS_PAT_MOD;
8171 if (has_charset_modifier == ASCII_RESTRICT_PAT_MOD) {
8172 vFAIL2("Regexp modifier \"%c\" may appear a maximum of twice", ASCII_RESTRICT_PAT_MOD);
8174 else if (has_charset_modifier == *(RExC_parse - 1)) {
8175 vFAIL2("Regexp modifier \"%c\" may not appear twice", *(RExC_parse - 1));
8178 vFAIL3("Regexp modifiers \"%c\" and \"%c\" are mutually exclusive", has_charset_modifier, *(RExC_parse - 1));
8183 vFAIL2("Regexp modifier \"%c\" may not appear after the \"-\"", *(RExC_parse - 1));
8185 case ONCE_PAT_MOD: /* 'o' */
8186 case GLOBAL_PAT_MOD: /* 'g' */
8187 if (SIZE_ONLY && ckWARN(WARN_REGEXP)) {
8188 const I32 wflagbit = *RExC_parse == 'o' ? WASTED_O : WASTED_G;
8189 if (! (wastedflags & wflagbit) ) {
8190 wastedflags |= wflagbit;
8193 "Useless (%s%c) - %suse /%c modifier",
8194 flagsp == &negflags ? "?-" : "?",
8196 flagsp == &negflags ? "don't " : "",
8203 case CONTINUE_PAT_MOD: /* 'c' */
8204 if (SIZE_ONLY && ckWARN(WARN_REGEXP)) {
8205 if (! (wastedflags & WASTED_C) ) {
8206 wastedflags |= WASTED_GC;
8209 "Useless (%sc) - %suse /gc modifier",
8210 flagsp == &negflags ? "?-" : "?",
8211 flagsp == &negflags ? "don't " : ""
8216 case KEEPCOPY_PAT_MOD: /* 'p' */
8217 if (flagsp == &negflags) {
8219 ckWARNreg(RExC_parse + 1,"Useless use of (?-p)");
8221 *flagsp |= RXf_PMf_KEEPCOPY;
8225 /* A flag is a default iff it is following a minus, so
8226 * if there is a minus, it means will be trying to
8227 * re-specify a default which is an error */
8228 if (has_use_defaults || flagsp == &negflags) {
8231 vFAIL3("Sequence (%.*s...) not recognized", RExC_parse-seqstart, seqstart);
8235 wastedflags = 0; /* reset so (?g-c) warns twice */
8241 RExC_flags |= posflags;
8242 RExC_flags &= ~negflags;
8243 set_regex_charset(&RExC_flags, cs);
8245 oregflags |= posflags;
8246 oregflags &= ~negflags;
8247 set_regex_charset(&oregflags, cs);
8249 nextchar(pRExC_state);
8260 vFAIL3("Sequence (%.*s...) not recognized", RExC_parse-seqstart, seqstart);
8265 }} /* one for the default block, one for the switch */
8272 ret = reganode(pRExC_state, OPEN, parno);
8275 RExC_nestroot = parno;
8276 if (RExC_seen & REG_SEEN_RECURSE
8277 && !RExC_open_parens[parno-1])
8279 DEBUG_OPTIMISE_MORE_r(PerlIO_printf(Perl_debug_log,
8280 "Setting open paren #%"IVdf" to %d\n",
8281 (IV)parno, REG_NODE_NUM(ret)));
8282 RExC_open_parens[parno-1]= ret;
8285 Set_Node_Length(ret, 1); /* MJD */
8286 Set_Node_Offset(ret, RExC_parse); /* MJD */
8294 /* Pick up the branches, linking them together. */
8295 parse_start = RExC_parse; /* MJD */
8296 br = regbranch(pRExC_state, &flags, 1,depth+1);
8298 /* branch_len = (paren != 0); */
8302 if (*RExC_parse == '|') {
8303 if (!SIZE_ONLY && RExC_extralen) {
8304 reginsert(pRExC_state, BRANCHJ, br, depth+1);
8307 reginsert(pRExC_state, BRANCH, br, depth+1);
8308 Set_Node_Length(br, paren != 0);
8309 Set_Node_Offset_To_R(br-RExC_emit_start, parse_start-RExC_start);
8313 RExC_extralen += 1; /* For BRANCHJ-BRANCH. */
8315 else if (paren == ':') {
8316 *flagp |= flags&SIMPLE;
8318 if (is_open) { /* Starts with OPEN. */
8319 REGTAIL(pRExC_state, ret, br); /* OPEN -> first. */
8321 else if (paren != '?') /* Not Conditional */
8323 *flagp |= flags & (SPSTART | HASWIDTH | POSTPONED);
8325 while (*RExC_parse == '|') {
8326 if (!SIZE_ONLY && RExC_extralen) {
8327 ender = reganode(pRExC_state, LONGJMP,0);
8328 REGTAIL(pRExC_state, NEXTOPER(NEXTOPER(lastbr)), ender); /* Append to the previous. */
8331 RExC_extralen += 2; /* Account for LONGJMP. */
8332 nextchar(pRExC_state);
8334 if (RExC_npar > after_freeze)
8335 after_freeze = RExC_npar;
8336 RExC_npar = freeze_paren;
8338 br = regbranch(pRExC_state, &flags, 0, depth+1);
8342 REGTAIL(pRExC_state, lastbr, br); /* BRANCH -> BRANCH. */
8344 *flagp |= flags & (SPSTART | HASWIDTH | POSTPONED);
8347 if (have_branch || paren != ':') {
8348 /* Make a closing node, and hook it on the end. */
8351 ender = reg_node(pRExC_state, TAIL);
8354 ender = reganode(pRExC_state, CLOSE, parno);
8355 if (!SIZE_ONLY && RExC_seen & REG_SEEN_RECURSE) {
8356 DEBUG_OPTIMISE_MORE_r(PerlIO_printf(Perl_debug_log,
8357 "Setting close paren #%"IVdf" to %d\n",
8358 (IV)parno, REG_NODE_NUM(ender)));
8359 RExC_close_parens[parno-1]= ender;
8360 if (RExC_nestroot == parno)
8363 Set_Node_Offset(ender,RExC_parse+1); /* MJD */
8364 Set_Node_Length(ender,1); /* MJD */
8370 *flagp &= ~HASWIDTH;
8373 ender = reg_node(pRExC_state, SUCCEED);
8376 ender = reg_node(pRExC_state, END);
8378 assert(!RExC_opend); /* there can only be one! */
8383 REGTAIL(pRExC_state, lastbr, ender);
8385 if (have_branch && !SIZE_ONLY) {
8387 RExC_seen |= REG_TOP_LEVEL_BRANCHES;
8389 /* Hook the tails of the branches to the closing node. */
8390 for (br = ret; br; br = regnext(br)) {
8391 const U8 op = PL_regkind[OP(br)];
8393 REGTAIL_STUDY(pRExC_state, NEXTOPER(br), ender);
8395 else if (op == BRANCHJ) {
8396 REGTAIL_STUDY(pRExC_state, NEXTOPER(NEXTOPER(br)), ender);
8404 static const char parens[] = "=!<,>";
8406 if (paren && (p = strchr(parens, paren))) {
8407 U8 node = ((p - parens) % 2) ? UNLESSM : IFMATCH;
8408 int flag = (p - parens) > 1;
8411 node = SUSPEND, flag = 0;
8412 reginsert(pRExC_state, node,ret, depth+1);
8413 Set_Node_Cur_Length(ret);
8414 Set_Node_Offset(ret, parse_start + 1);
8416 REGTAIL_STUDY(pRExC_state, ret, reg_node(pRExC_state, TAIL));
8420 /* Check for proper termination. */
8422 RExC_flags = oregflags;
8423 if (RExC_parse >= RExC_end || *nextchar(pRExC_state) != ')') {
8424 RExC_parse = oregcomp_parse;
8425 vFAIL("Unmatched (");
8428 else if (!paren && RExC_parse < RExC_end) {
8429 if (*RExC_parse == ')') {
8431 vFAIL("Unmatched )");
8434 FAIL("Junk on end of regexp"); /* "Can't happen". */
8438 if (RExC_in_lookbehind) {
8439 RExC_in_lookbehind--;
8441 if (after_freeze > RExC_npar)
8442 RExC_npar = after_freeze;
8447 - regbranch - one alternative of an | operator
8449 * Implements the concatenation operator.
8452 S_regbranch(pTHX_ RExC_state_t *pRExC_state, I32 *flagp, I32 first, U32 depth)
8455 register regnode *ret;
8456 register regnode *chain = NULL;
8457 register regnode *latest;
8458 I32 flags = 0, c = 0;
8459 GET_RE_DEBUG_FLAGS_DECL;
8461 PERL_ARGS_ASSERT_REGBRANCH;
8463 DEBUG_PARSE("brnc");
8468 if (!SIZE_ONLY && RExC_extralen)
8469 ret = reganode(pRExC_state, BRANCHJ,0);
8471 ret = reg_node(pRExC_state, BRANCH);
8472 Set_Node_Length(ret, 1);
8476 if (!first && SIZE_ONLY)
8477 RExC_extralen += 1; /* BRANCHJ */
8479 *flagp = WORST; /* Tentatively. */
8482 nextchar(pRExC_state);
8483 while (RExC_parse < RExC_end && *RExC_parse != '|' && *RExC_parse != ')') {
8485 latest = regpiece(pRExC_state, &flags,depth+1);
8486 if (latest == NULL) {
8487 if (flags & TRYAGAIN)
8491 else if (ret == NULL)
8493 *flagp |= flags&(HASWIDTH|POSTPONED);
8494 if (chain == NULL) /* First piece. */
8495 *flagp |= flags&SPSTART;
8498 REGTAIL(pRExC_state, chain, latest);
8503 if (chain == NULL) { /* Loop ran zero times. */
8504 chain = reg_node(pRExC_state, NOTHING);
8509 *flagp |= flags&SIMPLE;
8516 - regpiece - something followed by possible [*+?]
8518 * Note that the branching code sequences used for ? and the general cases
8519 * of * and + are somewhat optimized: they use the same NOTHING node as
8520 * both the endmarker for their branch list and the body of the last branch.
8521 * It might seem that this node could be dispensed with entirely, but the
8522 * endmarker role is not redundant.
8525 S_regpiece(pTHX_ RExC_state_t *pRExC_state, I32 *flagp, U32 depth)
8528 register regnode *ret;
8530 register char *next;
8532 const char * const origparse = RExC_parse;
8534 I32 max = REG_INFTY;
8535 #ifdef RE_TRACK_PATTERN_OFFSETS
8538 const char *maxpos = NULL;
8539 GET_RE_DEBUG_FLAGS_DECL;
8541 PERL_ARGS_ASSERT_REGPIECE;
8543 DEBUG_PARSE("piec");
8545 ret = regatom(pRExC_state, &flags,depth+1);
8547 if (flags & TRYAGAIN)
8554 if (op == '{' && regcurly(RExC_parse)) {
8556 #ifdef RE_TRACK_PATTERN_OFFSETS
8557 parse_start = RExC_parse; /* MJD */
8559 next = RExC_parse + 1;
8560 while (isDIGIT(*next) || *next == ',') {
8569 if (*next == '}') { /* got one */
8573 min = atoi(RExC_parse);
8577 maxpos = RExC_parse;
8579 if (!max && *maxpos != '0')
8580 max = REG_INFTY; /* meaning "infinity" */
8581 else if (max >= REG_INFTY)
8582 vFAIL2("Quantifier in {,} bigger than %d", REG_INFTY - 1);
8584 nextchar(pRExC_state);
8587 if ((flags&SIMPLE)) {
8588 RExC_naughty += 2 + RExC_naughty / 2;
8589 reginsert(pRExC_state, CURLY, ret, depth+1);
8590 Set_Node_Offset(ret, parse_start+1); /* MJD */
8591 Set_Node_Cur_Length(ret);
8594 regnode * const w = reg_node(pRExC_state, WHILEM);
8597 REGTAIL(pRExC_state, ret, w);
8598 if (!SIZE_ONLY && RExC_extralen) {
8599 reginsert(pRExC_state, LONGJMP,ret, depth+1);
8600 reginsert(pRExC_state, NOTHING,ret, depth+1);
8601 NEXT_OFF(ret) = 3; /* Go over LONGJMP. */
8603 reginsert(pRExC_state, CURLYX,ret, depth+1);
8605 Set_Node_Offset(ret, parse_start+1);
8606 Set_Node_Length(ret,
8607 op == '{' ? (RExC_parse - parse_start) : 1);
8609 if (!SIZE_ONLY && RExC_extralen)
8610 NEXT_OFF(ret) = 3; /* Go over NOTHING to LONGJMP. */
8611 REGTAIL(pRExC_state, ret, reg_node(pRExC_state, NOTHING));
8613 RExC_whilem_seen++, RExC_extralen += 3;
8614 RExC_naughty += 4 + RExC_naughty; /* compound interest */
8623 vFAIL("Can't do {n,m} with n > m");
8625 ARG1_SET(ret, (U16)min);
8626 ARG2_SET(ret, (U16)max);
8638 #if 0 /* Now runtime fix should be reliable. */
8640 /* if this is reinstated, don't forget to put this back into perldiag:
8642 =item Regexp *+ operand could be empty at {#} in regex m/%s/
8644 (F) The part of the regexp subject to either the * or + quantifier
8645 could match an empty string. The {#} shows in the regular
8646 expression about where the problem was discovered.
8650 if (!(flags&HASWIDTH) && op != '?')
8651 vFAIL("Regexp *+ operand could be empty");
8654 #ifdef RE_TRACK_PATTERN_OFFSETS
8655 parse_start = RExC_parse;
8657 nextchar(pRExC_state);
8659 *flagp = (op != '+') ? (WORST|SPSTART|HASWIDTH) : (WORST|HASWIDTH);
8661 if (op == '*' && (flags&SIMPLE)) {
8662 reginsert(pRExC_state, STAR, ret, depth+1);
8666 else if (op == '*') {
8670 else if (op == '+' && (flags&SIMPLE)) {
8671 reginsert(pRExC_state, PLUS, ret, depth+1);
8675 else if (op == '+') {
8679 else if (op == '?') {
8684 if (!SIZE_ONLY && !(flags&(HASWIDTH|POSTPONED)) && max > REG_INFTY/3) {
8685 ckWARN3reg(RExC_parse,
8686 "%.*s matches null string many times",
8687 (int)(RExC_parse >= origparse ? RExC_parse - origparse : 0),
8691 if (RExC_parse < RExC_end && *RExC_parse == '?') {
8692 nextchar(pRExC_state);
8693 reginsert(pRExC_state, MINMOD, ret, depth+1);
8694 REGTAIL(pRExC_state, ret, ret + NODE_STEP_REGNODE);
8696 #ifndef REG_ALLOW_MINMOD_SUSPEND
8699 if (RExC_parse < RExC_end && *RExC_parse == '+') {
8701 nextchar(pRExC_state);
8702 ender = reg_node(pRExC_state, SUCCEED);
8703 REGTAIL(pRExC_state, ret, ender);
8704 reginsert(pRExC_state, SUSPEND, ret, depth+1);
8706 ender = reg_node(pRExC_state, TAIL);
8707 REGTAIL(pRExC_state, ret, ender);
8711 if (RExC_parse < RExC_end && ISMULT2(RExC_parse)) {
8713 vFAIL("Nested quantifiers");
8720 /* reg_namedseq(pRExC_state,UVp, UV depth)
8722 This is expected to be called by a parser routine that has
8723 recognized '\N' and needs to handle the rest. RExC_parse is
8724 expected to point at the first char following the N at the time
8727 The \N may be inside (indicated by valuep not being NULL) or outside a
8730 \N may begin either a named sequence, or if outside a character class, mean
8731 to match a non-newline. For non single-quoted regexes, the tokenizer has
8732 attempted to decide which, and in the case of a named sequence converted it
8733 into one of the forms: \N{} (if the sequence is null), or \N{U+c1.c2...},
8734 where c1... are the characters in the sequence. For single-quoted regexes,
8735 the tokenizer passes the \N sequence through unchanged; this code will not
8736 attempt to determine this nor expand those. The net effect is that if the
8737 beginning of the passed-in pattern isn't '{U+' or there is no '}', it
8738 signals that this \N occurrence means to match a non-newline.
8740 Only the \N{U+...} form should occur in a character class, for the same
8741 reason that '.' inside a character class means to just match a period: it
8742 just doesn't make sense.
8744 If valuep is non-null then it is assumed that we are parsing inside
8745 of a charclass definition and the first codepoint in the resolved
8746 string is returned via *valuep and the routine will return NULL.
8747 In this mode if a multichar string is returned from the charnames
8748 handler, a warning will be issued, and only the first char in the
8749 sequence will be examined. If the string returned is zero length
8750 then the value of *valuep is undefined and NON-NULL will
8751 be returned to indicate failure. (This will NOT be a valid pointer
8754 If valuep is null then it is assumed that we are parsing normal text and a
8755 new EXACT node is inserted into the program containing the resolved string,
8756 and a pointer to the new node is returned. But if the string is zero length
8757 a NOTHING node is emitted instead.
8759 On success RExC_parse is set to the char following the endbrace.
8760 Parsing failures will generate a fatal error via vFAIL(...)
8763 S_reg_namedseq(pTHX_ RExC_state_t *pRExC_state, UV *valuep, I32 *flagp, U32 depth)
8765 char * endbrace; /* '}' following the name */
8766 regnode *ret = NULL;
8769 GET_RE_DEBUG_FLAGS_DECL;
8771 PERL_ARGS_ASSERT_REG_NAMEDSEQ;
8775 /* The [^\n] meaning of \N ignores spaces and comments under the /x
8776 * modifier. The other meaning does not */
8777 p = (RExC_flags & RXf_PMf_EXTENDED)
8778 ? regwhite( pRExC_state, RExC_parse )
8781 /* Disambiguate between \N meaning a named character versus \N meaning
8782 * [^\n]. The former is assumed when it can't be the latter. */
8783 if (*p != '{' || regcurly(p)) {
8786 /* no bare \N in a charclass */
8787 vFAIL("\\N in a character class must be a named character: \\N{...}");
8789 nextchar(pRExC_state);
8790 ret = reg_node(pRExC_state, REG_ANY);
8791 *flagp |= HASWIDTH|SIMPLE;
8794 Set_Node_Length(ret, 1); /* MJD */
8798 /* Here, we have decided it should be a named sequence */
8800 /* The test above made sure that the next real character is a '{', but
8801 * under the /x modifier, it could be separated by space (or a comment and
8802 * \n) and this is not allowed (for consistency with \x{...} and the
8803 * tokenizer handling of \N{NAME}). */
8804 if (*RExC_parse != '{') {
8805 vFAIL("Missing braces on \\N{}");
8808 RExC_parse++; /* Skip past the '{' */
8810 if (! (endbrace = strchr(RExC_parse, '}')) /* no trailing brace */
8811 || ! (endbrace == RExC_parse /* nothing between the {} */
8812 || (endbrace - RExC_parse >= 2 /* U+ (bad hex is checked below */
8813 && strnEQ(RExC_parse, "U+", 2)))) /* for a better error msg) */
8815 if (endbrace) RExC_parse = endbrace; /* position msg's '<--HERE' */
8816 vFAIL("\\N{NAME} must be resolved by the lexer");
8819 if (endbrace == RExC_parse) { /* empty: \N{} */
8821 RExC_parse = endbrace + 1;
8822 return reg_node(pRExC_state,NOTHING);
8826 ckWARNreg(RExC_parse,
8827 "Ignoring zero length \\N{} in character class"
8829 RExC_parse = endbrace + 1;
8832 return (regnode *) &RExC_parse; /* Invalid regnode pointer */
8835 REQUIRE_UTF8; /* named sequences imply Unicode semantics */
8836 RExC_parse += 2; /* Skip past the 'U+' */
8838 if (valuep) { /* In a bracketed char class */
8839 /* We only pay attention to the first char of
8840 multichar strings being returned. I kinda wonder
8841 if this makes sense as it does change the behaviour
8842 from earlier versions, OTOH that behaviour was broken
8843 as well. XXX Solution is to recharacterize as
8844 [rest-of-class]|multi1|multi2... */
8846 STRLEN length_of_hex;
8847 I32 flags = PERL_SCAN_ALLOW_UNDERSCORES
8848 | PERL_SCAN_DISALLOW_PREFIX
8849 | (SIZE_ONLY ? PERL_SCAN_SILENT_ILLDIGIT : 0);
8851 char * endchar = RExC_parse + strcspn(RExC_parse, ".}");
8852 if (endchar < endbrace) {
8853 ckWARNreg(endchar, "Using just the first character returned by \\N{} in character class");
8856 length_of_hex = (STRLEN)(endchar - RExC_parse);
8857 *valuep = grok_hex(RExC_parse, &length_of_hex, &flags, NULL);
8859 /* The tokenizer should have guaranteed validity, but it's possible to
8860 * bypass it by using single quoting, so check */
8861 if (length_of_hex == 0
8862 || length_of_hex != (STRLEN)(endchar - RExC_parse) )
8864 RExC_parse += length_of_hex; /* Includes all the valid */
8865 RExC_parse += (RExC_orig_utf8) /* point to after 1st invalid */
8866 ? UTF8SKIP(RExC_parse)
8868 /* Guard against malformed utf8 */
8869 if (RExC_parse >= endchar) RExC_parse = endchar;
8870 vFAIL("Invalid hexadecimal number in \\N{U+...}");
8873 RExC_parse = endbrace + 1;
8874 if (endchar == endbrace) return NULL;
8876 ret = (regnode *) &RExC_parse; /* Invalid regnode pointer */
8878 else { /* Not a char class */
8880 /* What is done here is to convert this to a sub-pattern of the form
8881 * (?:\x{char1}\x{char2}...)
8882 * and then call reg recursively. That way, it retains its atomicness,
8883 * while not having to worry about special handling that some code
8884 * points may have. toke.c has converted the original Unicode values
8885 * to native, so that we can just pass on the hex values unchanged. We
8886 * do have to set a flag to keep recoding from happening in the
8889 SV * substitute_parse = newSVpvn_flags("?:", 2, SVf_UTF8|SVs_TEMP);
8891 char *endchar; /* Points to '.' or '}' ending cur char in the input
8893 char *orig_end = RExC_end;
8895 while (RExC_parse < endbrace) {
8897 /* Code points are separated by dots. If none, there is only one
8898 * code point, and is terminated by the brace */
8899 endchar = RExC_parse + strcspn(RExC_parse, ".}");
8901 /* Convert to notation the rest of the code understands */
8902 sv_catpv(substitute_parse, "\\x{");
8903 sv_catpvn(substitute_parse, RExC_parse, endchar - RExC_parse);
8904 sv_catpv(substitute_parse, "}");
8906 /* Point to the beginning of the next character in the sequence. */
8907 RExC_parse = endchar + 1;
8909 sv_catpv(substitute_parse, ")");
8911 RExC_parse = SvPV(substitute_parse, len);
8913 /* Don't allow empty number */
8915 vFAIL("Invalid hexadecimal number in \\N{U+...}");
8917 RExC_end = RExC_parse + len;
8919 /* The values are Unicode, and therefore not subject to recoding */
8920 RExC_override_recoding = 1;
8922 ret = reg(pRExC_state, 1, flagp, depth+1);
8924 RExC_parse = endbrace;
8925 RExC_end = orig_end;
8926 RExC_override_recoding = 0;
8928 nextchar(pRExC_state);
8938 * It returns the code point in utf8 for the value in *encp.
8939 * value: a code value in the source encoding
8940 * encp: a pointer to an Encode object
8942 * If the result from Encode is not a single character,
8943 * it returns U+FFFD (Replacement character) and sets *encp to NULL.
8946 S_reg_recode(pTHX_ const char value, SV **encp)
8949 SV * const sv = newSVpvn_flags(&value, numlen, SVs_TEMP);
8950 const char * const s = *encp ? sv_recode_to_utf8(sv, *encp) : SvPVX(sv);
8951 const STRLEN newlen = SvCUR(sv);
8952 UV uv = UNICODE_REPLACEMENT;
8954 PERL_ARGS_ASSERT_REG_RECODE;
8958 ? utf8n_to_uvchr((U8*)s, newlen, &numlen, UTF8_ALLOW_DEFAULT)
8961 if (!newlen || numlen != newlen) {
8962 uv = UNICODE_REPLACEMENT;
8970 - regatom - the lowest level
8972 Try to identify anything special at the start of the pattern. If there
8973 is, then handle it as required. This may involve generating a single regop,
8974 such as for an assertion; or it may involve recursing, such as to
8975 handle a () structure.
8977 If the string doesn't start with something special then we gobble up
8978 as much literal text as we can.
8980 Once we have been able to handle whatever type of thing started the
8981 sequence, we return.
8983 Note: we have to be careful with escapes, as they can be both literal
8984 and special, and in the case of \10 and friends can either, depending
8985 on context. Specifically there are two separate switches for handling
8986 escape sequences, with the one for handling literal escapes requiring
8987 a dummy entry for all of the special escapes that are actually handled
8992 S_regatom(pTHX_ RExC_state_t *pRExC_state, I32 *flagp, U32 depth)
8995 register regnode *ret = NULL;
8997 char *parse_start = RExC_parse;
8999 GET_RE_DEBUG_FLAGS_DECL;
9000 DEBUG_PARSE("atom");
9001 *flagp = WORST; /* Tentatively. */
9003 PERL_ARGS_ASSERT_REGATOM;
9006 switch ((U8)*RExC_parse) {
9008 RExC_seen_zerolen++;
9009 nextchar(pRExC_state);
9010 if (RExC_flags & RXf_PMf_MULTILINE)
9011 ret = reg_node(pRExC_state, MBOL);
9012 else if (RExC_flags & RXf_PMf_SINGLELINE)
9013 ret = reg_node(pRExC_state, SBOL);
9015 ret = reg_node(pRExC_state, BOL);
9016 Set_Node_Length(ret, 1); /* MJD */
9019 nextchar(pRExC_state);
9021 RExC_seen_zerolen++;
9022 if (RExC_flags & RXf_PMf_MULTILINE)
9023 ret = reg_node(pRExC_state, MEOL);
9024 else if (RExC_flags & RXf_PMf_SINGLELINE)
9025 ret = reg_node(pRExC_state, SEOL);
9027 ret = reg_node(pRExC_state, EOL);
9028 Set_Node_Length(ret, 1); /* MJD */
9031 nextchar(pRExC_state);
9032 if (RExC_flags & RXf_PMf_SINGLELINE)
9033 ret = reg_node(pRExC_state, SANY);
9035 ret = reg_node(pRExC_state, REG_ANY);
9036 *flagp |= HASWIDTH|SIMPLE;
9038 Set_Node_Length(ret, 1); /* MJD */
9042 char * const oregcomp_parse = ++RExC_parse;
9043 ret = regclass(pRExC_state,depth+1);
9044 if (*RExC_parse != ']') {
9045 RExC_parse = oregcomp_parse;
9046 vFAIL("Unmatched [");
9048 nextchar(pRExC_state);
9049 *flagp |= HASWIDTH|SIMPLE;
9050 Set_Node_Length(ret, RExC_parse - oregcomp_parse + 1); /* MJD */
9054 nextchar(pRExC_state);
9055 ret = reg(pRExC_state, 1, &flags,depth+1);
9057 if (flags & TRYAGAIN) {
9058 if (RExC_parse == RExC_end) {
9059 /* Make parent create an empty node if needed. */
9067 *flagp |= flags&(HASWIDTH|SPSTART|SIMPLE|POSTPONED);
9071 if (flags & TRYAGAIN) {
9075 vFAIL("Internal urp");
9076 /* Supposed to be caught earlier. */
9079 if (!regcurly(RExC_parse)) {
9088 vFAIL("Quantifier follows nothing");
9093 This switch handles escape sequences that resolve to some kind
9094 of special regop and not to literal text. Escape sequnces that
9095 resolve to literal text are handled below in the switch marked
9098 Every entry in this switch *must* have a corresponding entry
9099 in the literal escape switch. However, the opposite is not
9100 required, as the default for this switch is to jump to the
9101 literal text handling code.
9103 switch ((U8)*++RExC_parse) {
9104 /* Special Escapes */
9106 RExC_seen_zerolen++;
9107 ret = reg_node(pRExC_state, SBOL);
9109 goto finish_meta_pat;
9111 ret = reg_node(pRExC_state, GPOS);
9112 RExC_seen |= REG_SEEN_GPOS;
9114 goto finish_meta_pat;
9116 RExC_seen_zerolen++;
9117 ret = reg_node(pRExC_state, KEEPS);
9119 /* XXX:dmq : disabling in-place substitution seems to
9120 * be necessary here to avoid cases of memory corruption, as
9121 * with: C<$_="x" x 80; s/x\K/y/> -- rgs
9123 RExC_seen |= REG_SEEN_LOOKBEHIND;
9124 goto finish_meta_pat;
9126 ret = reg_node(pRExC_state, SEOL);
9128 RExC_seen_zerolen++; /* Do not optimize RE away */
9129 goto finish_meta_pat;
9131 ret = reg_node(pRExC_state, EOS);
9133 RExC_seen_zerolen++; /* Do not optimize RE away */
9134 goto finish_meta_pat;
9136 ret = reg_node(pRExC_state, CANY);
9137 RExC_seen |= REG_SEEN_CANY;
9138 *flagp |= HASWIDTH|SIMPLE;
9139 goto finish_meta_pat;
9141 ret = reg_node(pRExC_state, CLUMP);
9143 goto finish_meta_pat;
9145 switch (get_regex_charset(RExC_flags)) {
9146 case REGEX_LOCALE_CHARSET:
9149 case REGEX_UNICODE_CHARSET:
9152 case REGEX_ASCII_RESTRICTED_CHARSET:
9153 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9156 case REGEX_DEPENDS_CHARSET:
9162 ret = reg_node(pRExC_state, op);
9163 *flagp |= HASWIDTH|SIMPLE;
9164 goto finish_meta_pat;
9166 switch (get_regex_charset(RExC_flags)) {
9167 case REGEX_LOCALE_CHARSET:
9170 case REGEX_UNICODE_CHARSET:
9173 case REGEX_ASCII_RESTRICTED_CHARSET:
9174 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9177 case REGEX_DEPENDS_CHARSET:
9183 ret = reg_node(pRExC_state, op);
9184 *flagp |= HASWIDTH|SIMPLE;
9185 goto finish_meta_pat;
9187 RExC_seen_zerolen++;
9188 RExC_seen |= REG_SEEN_LOOKBEHIND;
9189 switch (get_regex_charset(RExC_flags)) {
9190 case REGEX_LOCALE_CHARSET:
9193 case REGEX_UNICODE_CHARSET:
9196 case REGEX_ASCII_RESTRICTED_CHARSET:
9197 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9200 case REGEX_DEPENDS_CHARSET:
9206 ret = reg_node(pRExC_state, op);
9207 FLAGS(ret) = get_regex_charset(RExC_flags);
9209 if (! SIZE_ONLY && (U8) *(RExC_parse + 1) == '{') {
9210 ckWARNregdep(RExC_parse, "\"\\b{\" is deprecated; use \"\\b\\{\" instead");
9212 goto finish_meta_pat;
9214 RExC_seen_zerolen++;
9215 RExC_seen |= REG_SEEN_LOOKBEHIND;
9216 switch (get_regex_charset(RExC_flags)) {
9217 case REGEX_LOCALE_CHARSET:
9220 case REGEX_UNICODE_CHARSET:
9223 case REGEX_ASCII_RESTRICTED_CHARSET:
9224 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9227 case REGEX_DEPENDS_CHARSET:
9233 ret = reg_node(pRExC_state, op);
9234 FLAGS(ret) = get_regex_charset(RExC_flags);
9236 if (! SIZE_ONLY && (U8) *(RExC_parse + 1) == '{') {
9237 ckWARNregdep(RExC_parse, "\"\\B{\" is deprecated; use \"\\B\\{\" instead");
9239 goto finish_meta_pat;
9241 switch (get_regex_charset(RExC_flags)) {
9242 case REGEX_LOCALE_CHARSET:
9245 case REGEX_UNICODE_CHARSET:
9248 case REGEX_ASCII_RESTRICTED_CHARSET:
9249 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9252 case REGEX_DEPENDS_CHARSET:
9258 ret = reg_node(pRExC_state, op);
9259 *flagp |= HASWIDTH|SIMPLE;
9260 goto finish_meta_pat;
9262 switch (get_regex_charset(RExC_flags)) {
9263 case REGEX_LOCALE_CHARSET:
9266 case REGEX_UNICODE_CHARSET:
9269 case REGEX_ASCII_RESTRICTED_CHARSET:
9270 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9273 case REGEX_DEPENDS_CHARSET:
9279 ret = reg_node(pRExC_state, op);
9280 *flagp |= HASWIDTH|SIMPLE;
9281 goto finish_meta_pat;
9283 switch (get_regex_charset(RExC_flags)) {
9284 case REGEX_LOCALE_CHARSET:
9287 case REGEX_ASCII_RESTRICTED_CHARSET:
9288 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9291 case REGEX_DEPENDS_CHARSET: /* No difference between these */
9292 case REGEX_UNICODE_CHARSET:
9298 ret = reg_node(pRExC_state, op);
9299 *flagp |= HASWIDTH|SIMPLE;
9300 goto finish_meta_pat;
9302 switch (get_regex_charset(RExC_flags)) {
9303 case REGEX_LOCALE_CHARSET:
9306 case REGEX_ASCII_RESTRICTED_CHARSET:
9307 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
9310 case REGEX_DEPENDS_CHARSET: /* No difference between these */
9311 case REGEX_UNICODE_CHARSET:
9317 ret = reg_node(pRExC_state, op);
9318 *flagp |= HASWIDTH|SIMPLE;
9319 goto finish_meta_pat;
9321 ret = reg_node(pRExC_state, LNBREAK);
9322 *flagp |= HASWIDTH|SIMPLE;
9323 goto finish_meta_pat;
9325 ret = reg_node(pRExC_state, HORIZWS);
9326 *flagp |= HASWIDTH|SIMPLE;
9327 goto finish_meta_pat;
9329 ret = reg_node(pRExC_state, NHORIZWS);
9330 *flagp |= HASWIDTH|SIMPLE;
9331 goto finish_meta_pat;
9333 ret = reg_node(pRExC_state, VERTWS);
9334 *flagp |= HASWIDTH|SIMPLE;
9335 goto finish_meta_pat;
9337 ret = reg_node(pRExC_state, NVERTWS);
9338 *flagp |= HASWIDTH|SIMPLE;
9340 nextchar(pRExC_state);
9341 Set_Node_Length(ret, 2); /* MJD */
9346 char* const oldregxend = RExC_end;
9348 char* parse_start = RExC_parse - 2;
9351 if (RExC_parse[1] == '{') {
9352 /* a lovely hack--pretend we saw [\pX] instead */
9353 RExC_end = strchr(RExC_parse, '}');
9355 const U8 c = (U8)*RExC_parse;
9357 RExC_end = oldregxend;
9358 vFAIL2("Missing right brace on \\%c{}", c);
9363 RExC_end = RExC_parse + 2;
9364 if (RExC_end > oldregxend)
9365 RExC_end = oldregxend;
9369 ret = regclass(pRExC_state,depth+1);
9371 RExC_end = oldregxend;
9374 Set_Node_Offset(ret, parse_start + 2);
9375 Set_Node_Cur_Length(ret);
9376 nextchar(pRExC_state);
9377 *flagp |= HASWIDTH|SIMPLE;
9381 /* Handle \N and \N{NAME} here and not below because it can be
9382 multicharacter. join_exact() will join them up later on.
9383 Also this makes sure that things like /\N{BLAH}+/ and
9384 \N{BLAH} being multi char Just Happen. dmq*/
9386 ret= reg_namedseq(pRExC_state, NULL, flagp, depth);
9388 case 'k': /* Handle \k<NAME> and \k'NAME' */
9391 char ch= RExC_parse[1];
9392 if (ch != '<' && ch != '\'' && ch != '{') {
9394 vFAIL2("Sequence %.2s... not terminated",parse_start);
9396 /* this pretty much dupes the code for (?P=...) in reg(), if
9397 you change this make sure you change that */
9398 char* name_start = (RExC_parse += 2);
9400 SV *sv_dat = reg_scan_name(pRExC_state,
9401 SIZE_ONLY ? REG_RSN_RETURN_NULL : REG_RSN_RETURN_DATA);
9402 ch= (ch == '<') ? '>' : (ch == '{') ? '}' : '\'';
9403 if (RExC_parse == name_start || *RExC_parse != ch)
9404 vFAIL2("Sequence %.3s... not terminated",parse_start);
9407 num = add_data( pRExC_state, 1, "S" );
9408 RExC_rxi->data->data[num]=(void*)sv_dat;
9409 SvREFCNT_inc_simple_void(sv_dat);
9413 ret = reganode(pRExC_state,
9416 : (MORE_ASCII_RESTRICTED)
9418 : (AT_LEAST_UNI_SEMANTICS)
9426 /* override incorrect value set in reganode MJD */
9427 Set_Node_Offset(ret, parse_start+1);
9428 Set_Node_Cur_Length(ret); /* MJD */
9429 nextchar(pRExC_state);
9435 case '1': case '2': case '3': case '4':
9436 case '5': case '6': case '7': case '8': case '9':
9439 bool isg = *RExC_parse == 'g';
9444 if (*RExC_parse == '{') {
9448 if (*RExC_parse == '-') {
9452 if (hasbrace && !isDIGIT(*RExC_parse)) {
9453 if (isrel) RExC_parse--;
9455 goto parse_named_seq;
9457 num = atoi(RExC_parse);
9458 if (isg && num == 0)
9459 vFAIL("Reference to invalid group 0");
9461 num = RExC_npar - num;
9463 vFAIL("Reference to nonexistent or unclosed group");
9465 if (!isg && num > 9 && num >= RExC_npar)
9468 char * const parse_start = RExC_parse - 1; /* MJD */
9469 while (isDIGIT(*RExC_parse))
9471 if (parse_start == RExC_parse - 1)
9472 vFAIL("Unterminated \\g... pattern");
9474 if (*RExC_parse != '}')
9475 vFAIL("Unterminated \\g{...} pattern");
9479 if (num > (I32)RExC_rx->nparens)
9480 vFAIL("Reference to nonexistent group");
9483 ret = reganode(pRExC_state,
9486 : (MORE_ASCII_RESTRICTED)
9488 : (AT_LEAST_UNI_SEMANTICS)
9496 /* override incorrect value set in reganode MJD */
9497 Set_Node_Offset(ret, parse_start+1);
9498 Set_Node_Cur_Length(ret); /* MJD */
9500 nextchar(pRExC_state);
9505 if (RExC_parse >= RExC_end)
9506 FAIL("Trailing \\");
9509 /* Do not generate "unrecognized" warnings here, we fall
9510 back into the quick-grab loop below */
9517 if (RExC_flags & RXf_PMf_EXTENDED) {
9518 if ( reg_skipcomment( pRExC_state ) )
9525 parse_start = RExC_parse - 1;
9530 register STRLEN len;
9535 U8 tmpbuf[UTF8_MAXBYTES_CASE+1], *foldbuf;
9538 /* Is this a LATIN LOWER CASE SHARP S in an EXACTFU node? If so,
9539 * it is folded to 'ss' even if not utf8 */
9540 bool is_exactfu_sharp_s;
9543 node_type = ((! FOLD) ? EXACT
9546 : (MORE_ASCII_RESTRICTED)
9548 : (AT_LEAST_UNI_SEMANTICS)
9551 ret = reg_node(pRExC_state, node_type);
9554 /* XXX The node can hold up to 255 bytes, yet this only goes to
9555 * 127. I (khw) do not know why. Keeping it somewhat less than
9556 * 255 allows us to not have to worry about overflow due to
9557 * converting to utf8 and fold expansion, but that value is
9558 * 255-UTF8_MAXBYTES_CASE. join_exact() may join adjacent nodes
9559 * split up by this limit into a single one using the real max of
9560 * 255. Even at 127, this breaks under rare circumstances. If
9561 * folding, we do not want to split a node at a character that is a
9562 * non-final in a multi-char fold, as an input string could just
9563 * happen to want to match across the node boundary. The join
9564 * would solve that problem if the join actually happens. But a
9565 * series of more than two nodes in a row each of 127 would cause
9566 * the first join to succeed to get to 254, but then there wouldn't
9567 * be room for the next one, which could at be one of those split
9568 * multi-char folds. I don't know of any fool-proof solution. One
9569 * could back off to end with only a code point that isn't such a
9570 * non-final, but it is possible for there not to be any in the
9572 for (len = 0, p = RExC_parse - 1;
9573 len < 127 && p < RExC_end;
9576 char * const oldp = p;
9578 if (RExC_flags & RXf_PMf_EXTENDED)
9579 p = regwhite( pRExC_state, p );
9590 /* Literal Escapes Switch
9592 This switch is meant to handle escape sequences that
9593 resolve to a literal character.
9595 Every escape sequence that represents something
9596 else, like an assertion or a char class, is handled
9597 in the switch marked 'Special Escapes' above in this
9598 routine, but also has an entry here as anything that
9599 isn't explicitly mentioned here will be treated as
9600 an unescaped equivalent literal.
9604 /* These are all the special escapes. */
9605 case 'A': /* Start assertion */
9606 case 'b': case 'B': /* Word-boundary assertion*/
9607 case 'C': /* Single char !DANGEROUS! */
9608 case 'd': case 'D': /* digit class */
9609 case 'g': case 'G': /* generic-backref, pos assertion */
9610 case 'h': case 'H': /* HORIZWS */
9611 case 'k': case 'K': /* named backref, keep marker */
9612 case 'N': /* named char sequence */
9613 case 'p': case 'P': /* Unicode property */
9614 case 'R': /* LNBREAK */
9615 case 's': case 'S': /* space class */
9616 case 'v': case 'V': /* VERTWS */
9617 case 'w': case 'W': /* word class */
9618 case 'X': /* eXtended Unicode "combining character sequence" */
9619 case 'z': case 'Z': /* End of line/string assertion */
9623 /* Anything after here is an escape that resolves to a
9624 literal. (Except digits, which may or may not)
9643 ender = ASCII_TO_NATIVE('\033');
9647 ender = ASCII_TO_NATIVE('\007');
9652 STRLEN brace_len = len;
9654 const char* error_msg;
9656 bool valid = grok_bslash_o(p,
9663 RExC_parse = p; /* going to die anyway; point
9664 to exact spot of failure */
9671 if (PL_encoding && ender < 0x100) {
9672 goto recode_encoding;
9681 char* const e = strchr(p, '}');
9685 vFAIL("Missing right brace on \\x{}");
9688 I32 flags = PERL_SCAN_ALLOW_UNDERSCORES
9689 | PERL_SCAN_DISALLOW_PREFIX;
9690 STRLEN numlen = e - p - 1;
9691 ender = grok_hex(p + 1, &numlen, &flags, NULL);
9698 I32 flags = PERL_SCAN_DISALLOW_PREFIX;
9700 ender = grok_hex(p, &numlen, &flags, NULL);
9703 if (PL_encoding && ender < 0x100)
9704 goto recode_encoding;
9708 ender = grok_bslash_c(*p++, UTF, SIZE_ONLY);
9710 case '0': case '1': case '2': case '3':case '4':
9711 case '5': case '6': case '7': case '8':case '9':
9713 (isDIGIT(p[1]) && atoi(p) >= RExC_npar))
9715 I32 flags = PERL_SCAN_SILENT_ILLDIGIT;
9717 ender = grok_oct(p, &numlen, &flags, NULL);
9727 if (PL_encoding && ender < 0x100)
9728 goto recode_encoding;
9731 if (! RExC_override_recoding) {
9732 SV* enc = PL_encoding;
9733 ender = reg_recode((const char)(U8)ender, &enc);
9734 if (!enc && SIZE_ONLY)
9735 ckWARNreg(p, "Invalid escape in the specified encoding");
9741 FAIL("Trailing \\");
9744 if (!SIZE_ONLY&& isALPHA(*p)) {
9745 /* Include any { following the alpha to emphasize
9746 * that it could be part of an escape at some point
9748 int len = (*(p + 1) == '{') ? 2 : 1;
9749 ckWARN3reg(p + len, "Unrecognized escape \\%.*s passed through", len, p);
9751 goto normal_default;
9756 if (UTF8_IS_START(*p) && UTF) {
9758 ender = utf8n_to_uvchr((U8*)p, RExC_end - p,
9759 &numlen, UTF8_ALLOW_DEFAULT);
9765 } /* End of switch on the literal */
9767 is_exactfu_sharp_s = (node_type == EXACTFU
9768 && ender == LATIN_SMALL_LETTER_SHARP_S);
9769 if ( RExC_flags & RXf_PMf_EXTENDED)
9770 p = regwhite( pRExC_state, p );
9771 if ((UTF && FOLD) || is_exactfu_sharp_s) {
9772 /* Prime the casefolded buffer. Locale rules, which apply
9773 * only to code points < 256, aren't known until execution,
9774 * so for them, just output the original character using
9775 * utf8. If we start to fold non-UTF patterns, be sure to
9776 * update join_exact() */
9777 if (LOC && ender < 256) {
9778 if (UNI_IS_INVARIANT(ender)) {
9779 *tmpbuf = (U8) ender;
9782 *tmpbuf = UTF8_TWO_BYTE_HI(ender);
9783 *(tmpbuf + 1) = UTF8_TWO_BYTE_LO(ender);
9787 else if (isASCII(ender)) { /* Note: Here can't also be LOC
9789 ender = toLOWER(ender);
9790 *tmpbuf = (U8) ender;
9793 else if (! MORE_ASCII_RESTRICTED && ! LOC) {
9795 /* Locale and /aa require more selectivity about the
9796 * fold, so are handled below. Otherwise, here, just
9798 ender = toFOLD_uni(ender, tmpbuf, &foldlen);
9801 /* Under locale rules or /aa we are not to mix,
9802 * respectively, ords < 256 or ASCII with non-. So
9803 * reject folds that mix them, using only the
9804 * non-folded code point. So do the fold to a
9805 * temporary, and inspect each character in it. */
9806 U8 trialbuf[UTF8_MAXBYTES_CASE+1];
9808 UV tmpender = toFOLD_uni(ender, trialbuf, &foldlen);
9809 U8* e = s + foldlen;
9810 bool fold_ok = TRUE;
9814 || (LOC && (UTF8_IS_INVARIANT(*s)
9815 || UTF8_IS_DOWNGRADEABLE_START(*s))))
9823 Copy(trialbuf, tmpbuf, foldlen, U8);
9827 uvuni_to_utf8(tmpbuf, ender);
9828 foldlen = UNISKIP(ender);
9832 if (p < RExC_end && ISMULT2(p)) { /* Back off on ?+*. */
9835 else if (UTF || is_exactfu_sharp_s) {
9837 /* Emit all the Unicode characters. */
9839 for (foldbuf = tmpbuf;
9841 foldlen -= numlen) {
9843 /* tmpbuf has been constructed by us, so we
9844 * know it is valid utf8 */
9845 ender = valid_utf8_to_uvchr(foldbuf, &numlen);
9847 const STRLEN unilen = reguni(pRExC_state, ender, s);
9850 /* In EBCDIC the numlen
9851 * and unilen can differ. */
9853 if (numlen >= foldlen)
9857 break; /* "Can't happen." */
9861 const STRLEN unilen = reguni(pRExC_state, ender, s);
9870 REGC((char)ender, s++);
9874 if (UTF || is_exactfu_sharp_s) {
9876 /* Emit all the Unicode characters. */
9878 for (foldbuf = tmpbuf;
9880 foldlen -= numlen) {
9881 ender = valid_utf8_to_uvchr(foldbuf, &numlen);
9883 const STRLEN unilen = reguni(pRExC_state, ender, s);
9886 /* In EBCDIC the numlen
9887 * and unilen can differ. */
9889 if (numlen >= foldlen)
9897 const STRLEN unilen = reguni(pRExC_state, ender, s);
9906 REGC((char)ender, s++);
9909 loopdone: /* Jumped to when encounters something that shouldn't be in
9912 Set_Node_Cur_Length(ret); /* MJD */
9913 nextchar(pRExC_state);
9915 /* len is STRLEN which is unsigned, need to copy to signed */
9918 vFAIL("Internal disaster");
9922 if (len == 1 && UNI_IS_INVARIANT(ender))
9926 RExC_size += STR_SZ(len);
9929 RExC_emit += STR_SZ(len);
9937 /* Jumped to when an unrecognized character set is encountered */
9939 Perl_croak(aTHX_ "panic: Unknown regex character set encoding: %u", get_regex_charset(RExC_flags));
9944 S_regwhite( RExC_state_t *pRExC_state, char *p )
9946 const char *e = RExC_end;
9948 PERL_ARGS_ASSERT_REGWHITE;
9953 else if (*p == '#') {
9962 RExC_seen |= REG_SEEN_RUN_ON_COMMENT;
9970 /* Parse POSIX character classes: [[:foo:]], [[=foo=]], [[.foo.]].
9971 Character classes ([:foo:]) can also be negated ([:^foo:]).
9972 Returns a named class id (ANYOF_XXX) if successful, -1 otherwise.
9973 Equivalence classes ([=foo=]) and composites ([.foo.]) are parsed,
9974 but trigger failures because they are currently unimplemented. */
9976 #define POSIXCC_DONE(c) ((c) == ':')
9977 #define POSIXCC_NOTYET(c) ((c) == '=' || (c) == '.')
9978 #define POSIXCC(c) (POSIXCC_DONE(c) || POSIXCC_NOTYET(c))
9981 S_regpposixcc(pTHX_ RExC_state_t *pRExC_state, I32 value)
9984 I32 namedclass = OOB_NAMEDCLASS;
9986 PERL_ARGS_ASSERT_REGPPOSIXCC;
9988 if (value == '[' && RExC_parse + 1 < RExC_end &&
9989 /* I smell either [: or [= or [. -- POSIX has been here, right? */
9990 POSIXCC(UCHARAT(RExC_parse))) {
9991 const char c = UCHARAT(RExC_parse);
9992 char* const s = RExC_parse++;
9994 while (RExC_parse < RExC_end && UCHARAT(RExC_parse) != c)
9996 if (RExC_parse == RExC_end)
9997 /* Grandfather lone [:, [=, [. */
10000 const char* const t = RExC_parse++; /* skip over the c */
10003 if (UCHARAT(RExC_parse) == ']') {
10004 const char *posixcc = s + 1;
10005 RExC_parse++; /* skip over the ending ] */
10008 const I32 complement = *posixcc == '^' ? *posixcc++ : 0;
10009 const I32 skip = t - posixcc;
10011 /* Initially switch on the length of the name. */
10014 if (memEQ(posixcc, "word", 4)) /* this is not POSIX, this is the Perl \w */
10015 namedclass = complement ? ANYOF_NALNUM : ANYOF_ALNUM;
10018 /* Names all of length 5. */
10019 /* alnum alpha ascii blank cntrl digit graph lower
10020 print punct space upper */
10021 /* Offset 4 gives the best switch position. */
10022 switch (posixcc[4]) {
10024 if (memEQ(posixcc, "alph", 4)) /* alpha */
10025 namedclass = complement ? ANYOF_NALPHA : ANYOF_ALPHA;
10028 if (memEQ(posixcc, "spac", 4)) /* space */
10029 namedclass = complement ? ANYOF_NPSXSPC : ANYOF_PSXSPC;
10032 if (memEQ(posixcc, "grap", 4)) /* graph */
10033 namedclass = complement ? ANYOF_NGRAPH : ANYOF_GRAPH;
10036 if (memEQ(posixcc, "asci", 4)) /* ascii */
10037 namedclass = complement ? ANYOF_NASCII : ANYOF_ASCII;
10040 if (memEQ(posixcc, "blan", 4)) /* blank */
10041 namedclass = complement ? ANYOF_NBLANK : ANYOF_BLANK;
10044 if (memEQ(posixcc, "cntr", 4)) /* cntrl */
10045 namedclass = complement ? ANYOF_NCNTRL : ANYOF_CNTRL;
10048 if (memEQ(posixcc, "alnu", 4)) /* alnum */
10049 namedclass = complement ? ANYOF_NALNUMC : ANYOF_ALNUMC;
10052 if (memEQ(posixcc, "lowe", 4)) /* lower */
10053 namedclass = complement ? ANYOF_NLOWER : ANYOF_LOWER;
10054 else if (memEQ(posixcc, "uppe", 4)) /* upper */
10055 namedclass = complement ? ANYOF_NUPPER : ANYOF_UPPER;
10058 if (memEQ(posixcc, "digi", 4)) /* digit */
10059 namedclass = complement ? ANYOF_NDIGIT : ANYOF_DIGIT;
10060 else if (memEQ(posixcc, "prin", 4)) /* print */
10061 namedclass = complement ? ANYOF_NPRINT : ANYOF_PRINT;
10062 else if (memEQ(posixcc, "punc", 4)) /* punct */
10063 namedclass = complement ? ANYOF_NPUNCT : ANYOF_PUNCT;
10068 if (memEQ(posixcc, "xdigit", 6))
10069 namedclass = complement ? ANYOF_NXDIGIT : ANYOF_XDIGIT;
10073 if (namedclass == OOB_NAMEDCLASS)
10074 Simple_vFAIL3("POSIX class [:%.*s:] unknown",
10076 assert (posixcc[skip] == ':');
10077 assert (posixcc[skip+1] == ']');
10078 } else if (!SIZE_ONLY) {
10079 /* [[=foo=]] and [[.foo.]] are still future. */
10081 /* adjust RExC_parse so the warning shows after
10082 the class closes */
10083 while (UCHARAT(RExC_parse) && UCHARAT(RExC_parse) != ']')
10085 Simple_vFAIL3("POSIX syntax [%c %c] is reserved for future extensions", c, c);
10088 /* Maternal grandfather:
10089 * "[:" ending in ":" but not in ":]" */
10099 S_checkposixcc(pTHX_ RExC_state_t *pRExC_state)
10103 PERL_ARGS_ASSERT_CHECKPOSIXCC;
10105 if (POSIXCC(UCHARAT(RExC_parse))) {
10106 const char *s = RExC_parse;
10107 const char c = *s++;
10109 while (isALNUM(*s))
10111 if (*s && c == *s && s[1] == ']') {
10113 "POSIX syntax [%c %c] belongs inside character classes",
10116 /* [[=foo=]] and [[.foo.]] are still future. */
10117 if (POSIXCC_NOTYET(c)) {
10118 /* adjust RExC_parse so the error shows after
10119 the class closes */
10120 while (UCHARAT(RExC_parse) && UCHARAT(RExC_parse++) != ']')
10122 Simple_vFAIL3("POSIX syntax [%c %c] is reserved for future extensions", c, c);
10128 /* Generate the code to add a full posix character <class> to the bracketed
10129 * character class given by <node>. (<node> is needed only under locale rules)
10130 * destlist is the inversion list for non-locale rules that this class is
10132 * sourcelist is the ASCII-range inversion list to add under /a rules
10133 * Xsourcelist is the full Unicode range list to use otherwise. */
10134 #define DO_POSIX(node, class, destlist, sourcelist, Xsourcelist) \
10136 SV* scratch_list = NULL; \
10138 /* Set this class in the node for runtime matching */ \
10139 ANYOF_CLASS_SET(node, class); \
10141 /* For above Latin1 code points, we use the full Unicode range */ \
10142 _invlist_intersection(PL_AboveLatin1, \
10145 /* And set the output to it, adding instead if there already is an \
10146 * output. Checking if <destlist> is NULL first saves an extra \
10147 * clone. Its reference count will be decremented at the next \
10148 * union, etc, or if this is the only instance, at the end of the \
10150 if (! destlist) { \
10151 destlist = scratch_list; \
10154 _invlist_union(destlist, scratch_list, &destlist); \
10155 SvREFCNT_dec(scratch_list); \
10159 /* For non-locale, just add it to any existing list */ \
10160 _invlist_union(destlist, \
10161 (AT_LEAST_ASCII_RESTRICTED) \
10167 /* Like DO_POSIX, but matches the complement of <sourcelist> and <Xsourcelist>.
10169 #define DO_N_POSIX(node, class, destlist, sourcelist, Xsourcelist) \
10171 SV* scratch_list = NULL; \
10172 ANYOF_CLASS_SET(node, class); \
10173 _invlist_subtract(PL_AboveLatin1, Xsourcelist, &scratch_list); \
10174 if (! destlist) { \
10175 destlist = scratch_list; \
10178 _invlist_union(destlist, scratch_list, &destlist); \
10179 SvREFCNT_dec(scratch_list); \
10183 _invlist_union_complement_2nd(destlist, \
10184 (AT_LEAST_ASCII_RESTRICTED) \
10188 /* Under /d, everything in the upper half of the Latin1 range \
10189 * matches this complement */ \
10190 if (DEPENDS_SEMANTICS) { \
10191 ANYOF_FLAGS(node) |= ANYOF_NON_UTF8_LATIN1_ALL; \
10195 /* Generate the code to add a posix character <class> to the bracketed
10196 * character class given by <node>. (<node> is needed only under locale rules)
10197 * destlist is the inversion list for non-locale rules that this class is
10199 * sourcelist is the ASCII-range inversion list to add under /a rules
10200 * l1_sourcelist is the Latin1 range list to use otherwise.
10201 * Xpropertyname is the name to add to <run_time_list> of the property to
10202 * specify the code points above Latin1 that will have to be
10203 * determined at run-time
10204 * run_time_list is a SV* that contains text names of properties that are to
10205 * be computed at run time. This concatenates <Xpropertyname>
10206 * to it, apppropriately
10207 * This is essentially DO_POSIX, but we know only the Latin1 values at compile
10209 #define DO_POSIX_LATIN1_ONLY_KNOWN(node, class, destlist, sourcelist, \
10210 l1_sourcelist, Xpropertyname, run_time_list) \
10211 /* If not /a matching, there are going to be code points we will have \
10212 * to defer to runtime to look-up */ \
10213 if (! AT_LEAST_ASCII_RESTRICTED) { \
10214 Perl_sv_catpvf(aTHX_ run_time_list, "+utf8::%s\n", Xpropertyname); \
10217 ANYOF_CLASS_SET(node, class); \
10220 _invlist_union(destlist, \
10221 (AT_LEAST_ASCII_RESTRICTED) \
10227 /* Like DO_POSIX_LATIN1_ONLY_KNOWN, but for the complement. A combination of
10228 * this and DO_N_POSIX */
10229 #define DO_N_POSIX_LATIN1_ONLY_KNOWN(node, class, destlist, sourcelist, \
10230 l1_sourcelist, Xpropertyname, run_time_list) \
10231 if (AT_LEAST_ASCII_RESTRICTED) { \
10232 _invlist_union_complement_2nd(destlist, sourcelist, &destlist); \
10235 Perl_sv_catpvf(aTHX_ run_time_list, "!utf8::%s\n", Xpropertyname); \
10237 ANYOF_CLASS_SET(node, namedclass); \
10240 SV* scratch_list = NULL; \
10241 _invlist_subtract(PL_Latin1, l1_sourcelist, &scratch_list); \
10242 if (! destlist) { \
10243 destlist = scratch_list; \
10246 _invlist_union(destlist, scratch_list, &destlist); \
10247 SvREFCNT_dec(scratch_list); \
10249 if (DEPENDS_SEMANTICS) { \
10250 ANYOF_FLAGS(node) |= ANYOF_NON_UTF8_LATIN1_ALL; \
10256 S_set_regclass_bit_fold(pTHX_ RExC_state_t *pRExC_state, regnode* node, const U8 value, SV** invlist_ptr, AV** alternate_ptr)
10259 /* Handle the setting of folds in the bitmap for non-locale ANYOF nodes.
10260 * Locale folding is done at run-time, so this function should not be
10261 * called for nodes that are for locales.
10263 * This function sets the bit corresponding to the fold of the input
10264 * 'value', if not already set. The fold of 'f' is 'F', and the fold of
10267 * It also knows about the characters that are in the bitmap that have
10268 * folds that are matchable only outside it, and sets the appropriate lists
10271 * It returns the number of bits that actually changed from 0 to 1 */
10276 PERL_ARGS_ASSERT_SET_REGCLASS_BIT_FOLD;
10278 fold = (AT_LEAST_UNI_SEMANTICS) ? PL_fold_latin1[value]
10281 /* It assumes the bit for 'value' has already been set */
10282 if (fold != value && ! ANYOF_BITMAP_TEST(node, fold)) {
10283 ANYOF_BITMAP_SET(node, fold);
10286 if (_HAS_NONLATIN1_FOLD_CLOSURE_ONLY_FOR_USE_BY_REGCOMP_DOT_C_AND_REGEXEC_DOT_C(value) && (! isASCII(value) || ! MORE_ASCII_RESTRICTED)) {
10287 /* Certain Latin1 characters have matches outside the bitmap. To get
10288 * here, 'value' is one of those characters. None of these matches is
10289 * valid for ASCII characters under /aa, which have been excluded by
10290 * the 'if' above. The matches fall into three categories:
10291 * 1) They are singly folded-to or -from an above 255 character, as
10292 * LATIN SMALL LETTER Y WITH DIAERESIS and LATIN CAPITAL LETTER Y
10294 * 2) They are part of a multi-char fold with another character in the
10295 * bitmap, only LATIN SMALL LETTER SHARP S => "ss" fits that bill;
10296 * 3) They are part of a multi-char fold with a character not in the
10297 * bitmap, such as various ligatures.
10298 * We aren't dealing fully with multi-char folds, except we do deal
10299 * with the pattern containing a character that has a multi-char fold
10300 * (not so much the inverse).
10301 * For types 1) and 3), the matches only happen when the target string
10302 * is utf8; that's not true for 2), and we set a flag for it.
10304 * The code below adds to the passed in inversion list the single fold
10305 * closures for 'value'. The values are hard-coded here so that an
10306 * innocent-looking character class, like /[ks]/i won't have to go out
10307 * to disk to find the possible matches. XXX It would be better to
10308 * generate these via regen, in case a new version of the Unicode
10309 * standard adds new mappings, though that is not really likely. */
10314 *invlist_ptr = add_cp_to_invlist(*invlist_ptr, 0x212A);
10318 /* LATIN SMALL LETTER LONG S */
10319 *invlist_ptr = add_cp_to_invlist(*invlist_ptr, 0x017F);
10322 *invlist_ptr = add_cp_to_invlist(*invlist_ptr,
10323 GREEK_SMALL_LETTER_MU);
10324 *invlist_ptr = add_cp_to_invlist(*invlist_ptr,
10325 GREEK_CAPITAL_LETTER_MU);
10327 case LATIN_CAPITAL_LETTER_A_WITH_RING_ABOVE:
10328 case LATIN_SMALL_LETTER_A_WITH_RING_ABOVE:
10329 /* ANGSTROM SIGN */
10330 *invlist_ptr = add_cp_to_invlist(*invlist_ptr, 0x212B);
10331 if (DEPENDS_SEMANTICS) { /* See DEPENDS comment below */
10332 *invlist_ptr = add_cp_to_invlist(*invlist_ptr,
10333 PL_fold_latin1[value]);
10336 case LATIN_SMALL_LETTER_Y_WITH_DIAERESIS:
10337 *invlist_ptr = add_cp_to_invlist(*invlist_ptr,
10338 LATIN_CAPITAL_LETTER_Y_WITH_DIAERESIS);
10340 case LATIN_SMALL_LETTER_SHARP_S:
10341 *invlist_ptr = add_cp_to_invlist(*invlist_ptr,
10342 LATIN_CAPITAL_LETTER_SHARP_S);
10344 /* Under /a, /d, and /u, this can match the two chars "ss" */
10345 if (! MORE_ASCII_RESTRICTED) {
10346 add_alternate(alternate_ptr, (U8 *) "ss", 2);
10348 /* And under /u or /a, it can match even if the target is
10350 if (AT_LEAST_UNI_SEMANTICS) {
10351 ANYOF_FLAGS(node) |= ANYOF_NONBITMAP_NON_UTF8;
10355 case 'F': case 'f':
10356 case 'I': case 'i':
10357 case 'L': case 'l':
10358 case 'T': case 't':
10359 case 'A': case 'a':
10360 case 'H': case 'h':
10361 case 'J': case 'j':
10362 case 'N': case 'n':
10363 case 'W': case 'w':
10364 case 'Y': case 'y':
10365 /* These all are targets of multi-character folds from code
10366 * points that require UTF8 to express, so they can't match
10367 * unless the target string is in UTF-8, so no action here is
10368 * necessary, as regexec.c properly handles the general case
10369 * for UTF-8 matching */
10372 /* Use deprecated warning to increase the chances of this
10374 ckWARN2regdep(RExC_parse, "Perl folding rules are not up-to-date for 0x%x; please use the perlbug utility to report;", value);
10378 else if (DEPENDS_SEMANTICS
10379 && ! isASCII(value)
10380 && PL_fold_latin1[value] != value)
10382 /* Under DEPENDS rules, non-ASCII Latin1 characters match their
10383 * folds only when the target string is in UTF-8. We add the fold
10384 * here to the list of things to match outside the bitmap, which
10385 * won't be looked at unless it is UTF8 (or else if something else
10386 * says to look even if not utf8, but those things better not happen
10387 * under DEPENDS semantics. */
10388 *invlist_ptr = add_cp_to_invlist(*invlist_ptr, PL_fold_latin1[value]);
10395 PERL_STATIC_INLINE U8
10396 S_set_regclass_bit(pTHX_ RExC_state_t *pRExC_state, regnode* node, const U8 value, SV** invlist_ptr, AV** alternate_ptr)
10398 /* This inline function sets a bit in the bitmap if not already set, and if
10399 * appropriate, its fold, returning the number of bits that actually
10400 * changed from 0 to 1 */
10404 PERL_ARGS_ASSERT_SET_REGCLASS_BIT;
10406 if (ANYOF_BITMAP_TEST(node, value)) { /* Already set */
10410 ANYOF_BITMAP_SET(node, value);
10413 if (FOLD && ! LOC) { /* Locale folds aren't known until runtime */
10414 stored += set_regclass_bit_fold(pRExC_state, node, value, invlist_ptr, alternate_ptr);
10421 S_add_alternate(pTHX_ AV** alternate_ptr, U8* string, STRLEN len)
10423 /* Adds input 'string' with length 'len' to the ANYOF node's unicode
10424 * alternate list, pointed to by 'alternate_ptr'. This is an array of
10425 * the multi-character folds of characters in the node */
10428 PERL_ARGS_ASSERT_ADD_ALTERNATE;
10430 if (! *alternate_ptr) {
10431 *alternate_ptr = newAV();
10433 sv = newSVpvn_utf8((char*)string, len, TRUE);
10434 av_push(*alternate_ptr, sv);
10439 parse a class specification and produce either an ANYOF node that
10440 matches the pattern or perhaps will be optimized into an EXACTish node
10441 instead. The node contains a bit map for the first 256 characters, with the
10442 corresponding bit set if that character is in the list. For characters
10443 above 255, a range list is used */
10446 S_regclass(pTHX_ RExC_state_t *pRExC_state, U32 depth)
10449 register UV nextvalue;
10450 register IV prevvalue = OOB_UNICODE;
10451 register IV range = 0;
10452 UV value = 0; /* XXX:dmq: needs to be referenceable (unfortunately) */
10453 register regnode *ret;
10456 char *rangebegin = NULL;
10457 bool need_class = 0;
10458 bool allow_full_fold = TRUE; /* Assume wants multi-char folding */
10460 STRLEN initial_listsv_len = 0; /* Kind of a kludge to see if it is more
10461 than just initialized. */
10462 SV* properties = NULL; /* Code points that match \p{} \P{} */
10463 UV element_count = 0; /* Number of distinct elements in the class.
10464 Optimizations may be possible if this is tiny */
10467 /* Unicode properties are stored in a swash; this holds the current one
10468 * being parsed. If this swash is the only above-latin1 component of the
10469 * character class, an optimization is to pass it directly on to the
10470 * execution engine. Otherwise, it is set to NULL to indicate that there
10471 * are other things in the class that have to be dealt with at execution
10473 SV* swash = NULL; /* Code points that match \p{} \P{} */
10475 /* Set if a component of this character class is user-defined; just passed
10476 * on to the engine */
10477 UV has_user_defined_property = 0;
10479 /* code points this node matches that can't be stored in the bitmap */
10480 SV* nonbitmap = NULL;
10482 /* The items that are to match that aren't stored in the bitmap, but are a
10483 * result of things that are stored there. This is the fold closure of
10484 * such a character, either because it has DEPENDS semantics and shouldn't
10485 * be matched unless the target string is utf8, or is a code point that is
10486 * too large for the bit map, as for example, the fold of the MICRO SIGN is
10487 * above 255. This all is solely for performance reasons. By having this
10488 * code know the outside-the-bitmap folds that the bitmapped characters are
10489 * involved with, we don't have to go out to disk to find the list of
10490 * matches, unless the character class includes code points that aren't
10491 * storable in the bit map. That means that a character class with an 's'
10492 * in it, for example, doesn't need to go out to disk to find everything
10493 * that matches. A 2nd list is used so that the 'nonbitmap' list is kept
10494 * empty unless there is something whose fold we don't know about, and will
10495 * have to go out to the disk to find. */
10496 SV* l1_fold_invlist = NULL;
10498 /* List of multi-character folds that are matched by this node */
10499 AV* unicode_alternate = NULL;
10501 UV literal_endpoint = 0;
10503 UV stored = 0; /* how many chars stored in the bitmap */
10505 regnode * const orig_emit = RExC_emit; /* Save the original RExC_emit in
10506 case we need to change the emitted regop to an EXACT. */
10507 const char * orig_parse = RExC_parse;
10508 GET_RE_DEBUG_FLAGS_DECL;
10510 PERL_ARGS_ASSERT_REGCLASS;
10512 PERL_UNUSED_ARG(depth);
10515 DEBUG_PARSE("clas");
10517 /* Assume we are going to generate an ANYOF node. */
10518 ret = reganode(pRExC_state, ANYOF, 0);
10522 ANYOF_FLAGS(ret) = 0;
10525 if (UCHARAT(RExC_parse) == '^') { /* Complement of range. */
10529 ANYOF_FLAGS(ret) |= ANYOF_INVERT;
10531 /* We have decided to not allow multi-char folds in inverted character
10532 * classes, due to the confusion that can happen, especially with
10533 * classes that are designed for a non-Unicode world: You have the
10534 * peculiar case that:
10535 "s s" =~ /^[^\xDF]+$/i => Y
10536 "ss" =~ /^[^\xDF]+$/i => N
10538 * See [perl #89750] */
10539 allow_full_fold = FALSE;
10543 RExC_size += ANYOF_SKIP;
10544 listsv = &PL_sv_undef; /* For code scanners: listsv always non-NULL. */
10547 RExC_emit += ANYOF_SKIP;
10549 ANYOF_FLAGS(ret) |= ANYOF_LOCALE;
10551 ANYOF_BITMAP_ZERO(ret);
10552 listsv = newSVpvs("# comment\n");
10553 initial_listsv_len = SvCUR(listsv);
10556 nextvalue = RExC_parse < RExC_end ? UCHARAT(RExC_parse) : 0;
10558 if (!SIZE_ONLY && POSIXCC(nextvalue))
10559 checkposixcc(pRExC_state);
10561 /* allow 1st char to be ] (allowing it to be - is dealt with later) */
10562 if (UCHARAT(RExC_parse) == ']')
10563 goto charclassloop;
10566 while (RExC_parse < RExC_end && UCHARAT(RExC_parse) != ']') {
10570 namedclass = OOB_NAMEDCLASS; /* initialize as illegal */
10573 rangebegin = RExC_parse;
10577 value = utf8n_to_uvchr((U8*)RExC_parse,
10578 RExC_end - RExC_parse,
10579 &numlen, UTF8_ALLOW_DEFAULT);
10580 RExC_parse += numlen;
10583 value = UCHARAT(RExC_parse++);
10585 nextvalue = RExC_parse < RExC_end ? UCHARAT(RExC_parse) : 0;
10586 if (value == '[' && POSIXCC(nextvalue))
10587 namedclass = regpposixcc(pRExC_state, value);
10588 else if (value == '\\') {
10590 value = utf8n_to_uvchr((U8*)RExC_parse,
10591 RExC_end - RExC_parse,
10592 &numlen, UTF8_ALLOW_DEFAULT);
10593 RExC_parse += numlen;
10596 value = UCHARAT(RExC_parse++);
10597 /* Some compilers cannot handle switching on 64-bit integer
10598 * values, therefore value cannot be an UV. Yes, this will
10599 * be a problem later if we want switch on Unicode.
10600 * A similar issue a little bit later when switching on
10601 * namedclass. --jhi */
10602 switch ((I32)value) {
10603 case 'w': namedclass = ANYOF_ALNUM; break;
10604 case 'W': namedclass = ANYOF_NALNUM; break;
10605 case 's': namedclass = ANYOF_SPACE; break;
10606 case 'S': namedclass = ANYOF_NSPACE; break;
10607 case 'd': namedclass = ANYOF_DIGIT; break;
10608 case 'D': namedclass = ANYOF_NDIGIT; break;
10609 case 'v': namedclass = ANYOF_VERTWS; break;
10610 case 'V': namedclass = ANYOF_NVERTWS; break;
10611 case 'h': namedclass = ANYOF_HORIZWS; break;
10612 case 'H': namedclass = ANYOF_NHORIZWS; break;
10613 case 'N': /* Handle \N{NAME} in class */
10615 /* We only pay attention to the first char of
10616 multichar strings being returned. I kinda wonder
10617 if this makes sense as it does change the behaviour
10618 from earlier versions, OTOH that behaviour was broken
10620 UV v; /* value is register so we cant & it /grrr */
10621 if (reg_namedseq(pRExC_state, &v, NULL, depth)) {
10631 if (RExC_parse >= RExC_end)
10632 vFAIL2("Empty \\%c{}", (U8)value);
10633 if (*RExC_parse == '{') {
10634 const U8 c = (U8)value;
10635 e = strchr(RExC_parse++, '}');
10637 vFAIL2("Missing right brace on \\%c{}", c);
10638 while (isSPACE(UCHARAT(RExC_parse)))
10640 if (e == RExC_parse)
10641 vFAIL2("Empty \\%c{}", c);
10642 n = e - RExC_parse;
10643 while (isSPACE(UCHARAT(RExC_parse + n - 1)))
10654 if (UCHARAT(RExC_parse) == '^') {
10657 value = value == 'p' ? 'P' : 'p'; /* toggle */
10658 while (isSPACE(UCHARAT(RExC_parse))) {
10663 /* Try to get the definition of the property into
10664 * <invlist>. If /i is in effect, the effective property
10665 * will have its name be <__NAME_i>. The design is
10666 * discussed in commit
10667 * 2f833f5208e26b208886e51e09e2c072b5eabb46 */
10668 Newx(name, n + sizeof("_i__\n"), char);
10670 sprintf(name, "%s%.*s%s\n",
10671 (FOLD) ? "__" : "",
10677 /* Look up the property name, and get its swash and
10678 * inversion list, if the property is found */
10680 SvREFCNT_dec(swash);
10682 swash = _core_swash_init("utf8", name, &PL_sv_undef,
10685 TRUE, /* this routine will handle
10686 undefined properties */
10687 NULL, FALSE /* No inversion list */
10691 || ! SvTYPE(SvRV(swash)) == SVt_PVHV
10693 hv_fetchs(MUTABLE_HV(SvRV(swash)),
10695 || ! (invlist = *invlistsvp))
10698 SvREFCNT_dec(swash);
10702 /* Here didn't find it. It could be a user-defined
10703 * property that will be available at run-time. Add it
10704 * to the list to look up then */
10705 Perl_sv_catpvf(aTHX_ listsv, "%cutf8::%s\n",
10706 (value == 'p' ? '+' : '!'),
10708 has_user_defined_property = 1;
10710 /* We don't know yet, so have to assume that the
10711 * property could match something in the Latin1 range,
10712 * hence something that isn't utf8 */
10713 ANYOF_FLAGS(ret) |= ANYOF_NONBITMAP_NON_UTF8;
10717 /* Here, did get the swash and its inversion list. If
10718 * the swash is from a user-defined property, then this
10719 * whole character class should be regarded as such */
10720 SV** user_defined_svp =
10721 hv_fetchs(MUTABLE_HV(SvRV(swash)),
10722 "USER_DEFINED", FALSE);
10723 if (user_defined_svp) {
10724 has_user_defined_property
10725 |= SvUV(*user_defined_svp);
10728 /* Invert if asking for the complement */
10729 if (value == 'P') {
10730 _invlist_union_complement_2nd(properties, invlist, &properties);
10732 /* The swash can't be used as-is, because we've
10733 * inverted things; delay removing it to here after
10734 * have copied its invlist above */
10735 SvREFCNT_dec(swash);
10739 _invlist_union(properties, invlist, &properties);
10744 RExC_parse = e + 1;
10745 namedclass = ANYOF_MAX; /* no official name, but it's named */
10747 /* \p means they want Unicode semantics */
10748 RExC_uni_semantics = 1;
10751 case 'n': value = '\n'; break;
10752 case 'r': value = '\r'; break;
10753 case 't': value = '\t'; break;
10754 case 'f': value = '\f'; break;
10755 case 'b': value = '\b'; break;
10756 case 'e': value = ASCII_TO_NATIVE('\033');break;
10757 case 'a': value = ASCII_TO_NATIVE('\007');break;
10759 RExC_parse--; /* function expects to be pointed at the 'o' */
10761 const char* error_msg;
10762 bool valid = grok_bslash_o(RExC_parse,
10767 RExC_parse += numlen;
10772 if (PL_encoding && value < 0x100) {
10773 goto recode_encoding;
10777 if (*RExC_parse == '{') {
10778 I32 flags = PERL_SCAN_ALLOW_UNDERSCORES
10779 | PERL_SCAN_DISALLOW_PREFIX;
10780 char * const e = strchr(RExC_parse++, '}');
10782 vFAIL("Missing right brace on \\x{}");
10784 numlen = e - RExC_parse;
10785 value = grok_hex(RExC_parse, &numlen, &flags, NULL);
10786 RExC_parse = e + 1;
10789 I32 flags = PERL_SCAN_DISALLOW_PREFIX;
10791 value = grok_hex(RExC_parse, &numlen, &flags, NULL);
10792 RExC_parse += numlen;
10794 if (PL_encoding && value < 0x100)
10795 goto recode_encoding;
10798 value = grok_bslash_c(*RExC_parse++, UTF, SIZE_ONLY);
10800 case '0': case '1': case '2': case '3': case '4':
10801 case '5': case '6': case '7':
10803 /* Take 1-3 octal digits */
10804 I32 flags = PERL_SCAN_SILENT_ILLDIGIT;
10806 value = grok_oct(--RExC_parse, &numlen, &flags, NULL);
10807 RExC_parse += numlen;
10808 if (PL_encoding && value < 0x100)
10809 goto recode_encoding;
10813 if (! RExC_override_recoding) {
10814 SV* enc = PL_encoding;
10815 value = reg_recode((const char)(U8)value, &enc);
10816 if (!enc && SIZE_ONLY)
10817 ckWARNreg(RExC_parse,
10818 "Invalid escape in the specified encoding");
10822 /* Allow \_ to not give an error */
10823 if (!SIZE_ONLY && isALNUM(value) && value != '_') {
10824 ckWARN2reg(RExC_parse,
10825 "Unrecognized escape \\%c in character class passed through",
10830 } /* end of \blah */
10833 literal_endpoint++;
10836 if (namedclass > OOB_NAMEDCLASS) { /* this is a named class \blah */
10838 /* What matches in a locale is not known until runtime, so need to
10839 * (one time per class) allocate extra space to pass to regexec.
10840 * The space will contain a bit for each named class that is to be
10841 * matched against. This isn't needed for \p{} and pseudo-classes,
10842 * as they are not affected by locale, and hence are dealt with
10844 if (LOC && namedclass < ANYOF_MAX && ! need_class) {
10847 RExC_size += ANYOF_CLASS_SKIP - ANYOF_SKIP;
10850 RExC_emit += ANYOF_CLASS_SKIP - ANYOF_SKIP;
10851 ANYOF_CLASS_ZERO(ret);
10853 ANYOF_FLAGS(ret) |= ANYOF_CLASS;
10856 /* a bad range like a-\d, a-[:digit:]. The '-' is taken as a
10857 * literal, as is the character that began the false range, i.e.
10858 * the 'a' in the examples */
10862 RExC_parse >= rangebegin ?
10863 RExC_parse - rangebegin : 0;
10864 ckWARN4reg(RExC_parse,
10865 "False [] range \"%*.*s\"",
10869 set_regclass_bit(pRExC_state, ret, '-', &l1_fold_invlist, &unicode_alternate);
10870 if (prevvalue < 256) {
10872 set_regclass_bit(pRExC_state, ret, (U8) prevvalue, &l1_fold_invlist, &unicode_alternate);
10875 nonbitmap = add_cp_to_invlist(nonbitmap, prevvalue);
10879 range = 0; /* this was not a true range */
10884 /* Possible truncation here but in some 64-bit environments
10885 * the compiler gets heartburn about switch on 64-bit values.
10886 * A similar issue a little earlier when switching on value.
10888 switch ((I32)namedclass) {
10890 case ANYOF_ALNUMC: /* C's alnum, in contrast to \w */
10891 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10892 PL_PosixAlnum, PL_L1PosixAlnum, "XPosixAlnum", listsv);
10894 case ANYOF_NALNUMC:
10895 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10896 PL_PosixAlnum, PL_L1PosixAlnum, "XPosixAlnum", listsv);
10899 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10900 PL_PosixAlpha, PL_L1PosixAlpha, "XPosixAlpha", listsv);
10903 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10904 PL_PosixAlpha, PL_L1PosixAlpha, "XPosixAlpha", listsv);
10908 ANYOF_CLASS_SET(ret, namedclass);
10911 _invlist_union(properties, PL_ASCII, &properties);
10916 ANYOF_CLASS_SET(ret, namedclass);
10919 _invlist_union_complement_2nd(properties,
10920 PL_ASCII, &properties);
10921 if (DEPENDS_SEMANTICS) {
10922 ANYOF_FLAGS(ret) |= ANYOF_NON_UTF8_LATIN1_ALL;
10927 DO_POSIX(ret, namedclass, properties,
10928 PL_PosixBlank, PL_XPosixBlank);
10931 DO_N_POSIX(ret, namedclass, properties,
10932 PL_PosixBlank, PL_XPosixBlank);
10935 DO_POSIX(ret, namedclass, properties,
10936 PL_PosixCntrl, PL_XPosixCntrl);
10939 DO_N_POSIX(ret, namedclass, properties,
10940 PL_PosixCntrl, PL_XPosixCntrl);
10943 /* Ignore the compiler warning for this macro, planned to
10944 * be eliminated later */
10945 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10946 PL_PosixDigit, PL_PosixDigit, "XPosixDigit", listsv);
10949 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10950 PL_PosixDigit, PL_PosixDigit, "XPosixDigit", listsv);
10953 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10954 PL_PosixGraph, PL_L1PosixGraph, "XPosixGraph", listsv);
10957 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10958 PL_PosixGraph, PL_L1PosixGraph, "XPosixGraph", listsv);
10960 case ANYOF_HORIZWS:
10961 /* For these, we use the nonbitmap, as /d doesn't make a
10962 * difference in what these match. There would be problems
10963 * if these characters had folds other than themselves, as
10964 * nonbitmap is subject to folding. It turns out that \h
10965 * is just a synonym for XPosixBlank */
10966 _invlist_union(nonbitmap, PL_XPosixBlank, &nonbitmap);
10968 case ANYOF_NHORIZWS:
10969 _invlist_union_complement_2nd(nonbitmap,
10970 PL_XPosixBlank, &nonbitmap);
10974 { /* These require special handling, as they differ under
10975 folding, matching Cased there (which in the ASCII range
10976 is the same as Alpha */
10982 if (FOLD && ! LOC) {
10983 ascii_source = PL_PosixAlpha;
10984 l1_source = PL_L1Cased;
10988 ascii_source = PL_PosixLower;
10989 l1_source = PL_L1PosixLower;
10990 Xname = "XPosixLower";
10992 if (namedclass == ANYOF_LOWER) {
10993 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
10994 ascii_source, l1_source, Xname, listsv);
10997 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass,
10998 properties, ascii_source, l1_source, Xname, listsv);
11003 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
11004 PL_PosixPrint, PL_L1PosixPrint, "XPosixPrint", listsv);
11007 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
11008 PL_PosixPrint, PL_L1PosixPrint, "XPosixPrint", listsv);
11011 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
11012 PL_PosixPunct, PL_L1PosixPunct, "XPosixPunct", listsv);
11015 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
11016 PL_PosixPunct, PL_L1PosixPunct, "XPosixPunct", listsv);
11019 DO_POSIX(ret, namedclass, properties,
11020 PL_PosixSpace, PL_XPosixSpace);
11022 case ANYOF_NPSXSPC:
11023 DO_N_POSIX(ret, namedclass, properties,
11024 PL_PosixSpace, PL_XPosixSpace);
11027 DO_POSIX(ret, namedclass, properties,
11028 PL_PerlSpace, PL_XPerlSpace);
11031 DO_N_POSIX(ret, namedclass, properties,
11032 PL_PerlSpace, PL_XPerlSpace);
11034 case ANYOF_UPPER: /* Same as LOWER, above */
11041 if (FOLD && ! LOC) {
11042 ascii_source = PL_PosixAlpha;
11043 l1_source = PL_L1Cased;
11047 ascii_source = PL_PosixUpper;
11048 l1_source = PL_L1PosixUpper;
11049 Xname = "XPosixUpper";
11051 if (namedclass == ANYOF_UPPER) {
11052 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
11053 ascii_source, l1_source, Xname, listsv);
11056 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass,
11057 properties, ascii_source, l1_source, Xname, listsv);
11061 case ANYOF_ALNUM: /* Really is 'Word' */
11062 DO_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
11063 PL_PosixWord, PL_L1PosixWord, "XPosixWord", listsv);
11066 DO_N_POSIX_LATIN1_ONLY_KNOWN(ret, namedclass, properties,
11067 PL_PosixWord, PL_L1PosixWord, "XPosixWord", listsv);
11070 /* For these, we use the nonbitmap, as /d doesn't make a
11071 * difference in what these match. There would be problems
11072 * if these characters had folds other than themselves, as
11073 * nonbitmap is subject to folding */
11074 _invlist_union(nonbitmap, PL_VertSpace, &nonbitmap);
11076 case ANYOF_NVERTWS:
11077 _invlist_union_complement_2nd(nonbitmap,
11078 PL_VertSpace, &nonbitmap);
11081 DO_POSIX(ret, namedclass, properties,
11082 PL_PosixXDigit, PL_XPosixXDigit);
11084 case ANYOF_NXDIGIT:
11085 DO_N_POSIX(ret, namedclass, properties,
11086 PL_PosixXDigit, PL_XPosixXDigit);
11089 /* this is to handle \p and \P */
11092 vFAIL("Invalid [::] class");
11098 } /* end of namedclass \blah */
11101 if (prevvalue > (IV)value) /* b-a */ {
11102 const int w = RExC_parse - rangebegin;
11103 Simple_vFAIL4("Invalid [] range \"%*.*s\"", w, w, rangebegin);
11104 range = 0; /* not a valid range */
11108 prevvalue = value; /* save the beginning of the range */
11109 if (RExC_parse+1 < RExC_end
11110 && *RExC_parse == '-'
11111 && RExC_parse[1] != ']')
11115 /* a bad range like \w-, [:word:]- ? */
11116 if (namedclass > OOB_NAMEDCLASS) {
11117 if (ckWARN(WARN_REGEXP)) {
11119 RExC_parse >= rangebegin ?
11120 RExC_parse - rangebegin : 0;
11122 "False [] range \"%*.*s\"",
11127 set_regclass_bit(pRExC_state, ret, '-', &l1_fold_invlist, &unicode_alternate);
11129 range = 1; /* yeah, it's a range! */
11130 continue; /* but do it the next time */
11134 /* non-Latin1 code point implies unicode semantics. Must be set in
11135 * pass1 so is there for the whole of pass 2 */
11137 RExC_uni_semantics = 1;
11140 /* now is the next time */
11142 if (prevvalue < 256) {
11143 const IV ceilvalue = value < 256 ? value : 255;
11146 /* In EBCDIC [\x89-\x91] should include
11147 * the \x8e but [i-j] should not. */
11148 if (literal_endpoint == 2 &&
11149 ((isLOWER(prevvalue) && isLOWER(ceilvalue)) ||
11150 (isUPPER(prevvalue) && isUPPER(ceilvalue))))
11152 if (isLOWER(prevvalue)) {
11153 for (i = prevvalue; i <= ceilvalue; i++)
11154 if (isLOWER(i) && !ANYOF_BITMAP_TEST(ret,i)) {
11156 set_regclass_bit(pRExC_state, ret, (U8) i, &l1_fold_invlist, &unicode_alternate);
11159 for (i = prevvalue; i <= ceilvalue; i++)
11160 if (isUPPER(i) && !ANYOF_BITMAP_TEST(ret,i)) {
11162 set_regclass_bit(pRExC_state, ret, (U8) i, &l1_fold_invlist, &unicode_alternate);
11168 for (i = prevvalue; i <= ceilvalue; i++) {
11169 stored += set_regclass_bit(pRExC_state, ret, (U8) i, &l1_fold_invlist, &unicode_alternate);
11173 const UV prevnatvalue = NATIVE_TO_UNI(prevvalue);
11174 const UV natvalue = NATIVE_TO_UNI(value);
11175 nonbitmap = add_range_to_invlist(nonbitmap, prevnatvalue, natvalue);
11178 literal_endpoint = 0;
11182 range = 0; /* this range (if it was one) is done now */
11189 /****** !SIZE_ONLY AFTER HERE *********/
11191 /* If folding and there are code points above 255, we calculate all
11192 * characters that could fold to or from the ones already on the list */
11193 if (FOLD && nonbitmap) {
11194 UV start, end; /* End points of code point ranges */
11196 SV* fold_intersection = NULL;
11198 /* This is a list of all the characters that participate in folds
11199 * (except marks, etc in multi-char folds */
11200 if (! PL_utf8_foldable) {
11201 SV* swash = swash_init("utf8", "Cased", &PL_sv_undef, 1, 0);
11202 PL_utf8_foldable = _swash_to_invlist(swash);
11203 SvREFCNT_dec(swash);
11206 /* This is a hash that for a particular fold gives all characters
11207 * that are involved in it */
11208 if (! PL_utf8_foldclosures) {
11210 /* If we were unable to find any folds, then we likely won't be
11211 * able to find the closures. So just create an empty list.
11212 * Folding will effectively be restricted to the non-Unicode rules
11213 * hard-coded into Perl. (This case happens legitimately during
11214 * compilation of Perl itself before the Unicode tables are
11216 if (invlist_len(PL_utf8_foldable) == 0) {
11217 PL_utf8_foldclosures = newHV();
11219 /* If the folds haven't been read in, call a fold function
11221 if (! PL_utf8_tofold) {
11222 U8 dummy[UTF8_MAXBYTES+1];
11225 /* This particular string is above \xff in both UTF-8 and
11227 to_utf8_fold((U8*) "\xC8\x80", dummy, &dummy_len);
11228 assert(PL_utf8_tofold); /* Verify that worked */
11230 PL_utf8_foldclosures = _swash_inversion_hash(PL_utf8_tofold);
11234 /* Only the characters in this class that participate in folds need be
11235 * checked. Get the intersection of this class and all the possible
11236 * characters that are foldable. This can quickly narrow down a large
11238 _invlist_intersection(PL_utf8_foldable, nonbitmap, &fold_intersection);
11240 /* Now look at the foldable characters in this class individually */
11241 invlist_iterinit(fold_intersection);
11242 while (invlist_iternext(fold_intersection, &start, &end)) {
11245 /* Look at every character in the range */
11246 for (j = start; j <= end; j++) {
11249 U8 foldbuf[UTF8_MAXBYTES_CASE+1];
11252 _to_uni_fold_flags(j, foldbuf, &foldlen, allow_full_fold);
11254 if (foldlen > (STRLEN)UNISKIP(f)) {
11256 /* Any multicharacter foldings (disallowed in lookbehind
11257 * patterns) require the following transform: [ABCDEF] ->
11258 * (?:[ABCabcDEFd]|pq|rst) where E folds into "pq" and F
11259 * folds into "rst", all other characters fold to single
11260 * characters. We save away these multicharacter foldings,
11261 * to be later saved as part of the additional "s" data. */
11262 if (! RExC_in_lookbehind) {
11264 U8* e = foldbuf + foldlen;
11266 /* If any of the folded characters of this are in the
11267 * Latin1 range, tell the regex engine that this can
11268 * match a non-utf8 target string. The only multi-byte
11269 * fold whose source is in the Latin1 range (U+00DF)
11270 * applies only when the target string is utf8, or
11271 * under unicode rules */
11272 if (j > 255 || AT_LEAST_UNI_SEMANTICS) {
11275 /* Can't mix ascii with non- under /aa */
11276 if (MORE_ASCII_RESTRICTED
11277 && (isASCII(*loc) != isASCII(j)))
11279 goto end_multi_fold;
11281 if (UTF8_IS_INVARIANT(*loc)
11282 || UTF8_IS_DOWNGRADEABLE_START(*loc))
11284 /* Can't mix above and below 256 under LOC
11287 goto end_multi_fold;
11290 |= ANYOF_NONBITMAP_NON_UTF8;
11293 loc += UTF8SKIP(loc);
11297 add_alternate(&unicode_alternate, foldbuf, foldlen);
11301 /* This is special-cased, as it is the only letter which
11302 * has both a multi-fold and single-fold in Latin1. All
11303 * the other chars that have single and multi-folds are
11304 * always in utf8, and the utf8 folding algorithm catches
11306 if (! LOC && j == LATIN_CAPITAL_LETTER_SHARP_S) {
11307 stored += set_regclass_bit(pRExC_state,
11309 LATIN_SMALL_LETTER_SHARP_S,
11310 &l1_fold_invlist, &unicode_alternate);
11314 /* Single character fold. Add everything in its fold
11315 * closure to the list that this node should match */
11318 /* The fold closures data structure is a hash with the keys
11319 * being every character that is folded to, like 'k', and
11320 * the values each an array of everything that folds to its
11321 * key. e.g. [ 'k', 'K', KELVIN_SIGN ] */
11322 if ((listp = hv_fetch(PL_utf8_foldclosures,
11323 (char *) foldbuf, foldlen, FALSE)))
11325 AV* list = (AV*) *listp;
11327 for (k = 0; k <= av_len(list); k++) {
11328 SV** c_p = av_fetch(list, k, FALSE);
11331 Perl_croak(aTHX_ "panic: invalid PL_utf8_foldclosures structure");
11335 /* /aa doesn't allow folds between ASCII and non-;
11336 * /l doesn't allow them between above and below
11338 if ((MORE_ASCII_RESTRICTED
11339 && (isASCII(c) != isASCII(j)))
11340 || (LOC && ((c < 256) != (j < 256))))
11345 if (c < 256 && AT_LEAST_UNI_SEMANTICS) {
11346 stored += set_regclass_bit(pRExC_state,
11349 &l1_fold_invlist, &unicode_alternate);
11351 /* It may be that the code point is already in
11352 * this range or already in the bitmap, in
11353 * which case we need do nothing */
11354 else if ((c < start || c > end)
11356 || ! ANYOF_BITMAP_TEST(ret, c)))
11358 nonbitmap = add_cp_to_invlist(nonbitmap, c);
11365 SvREFCNT_dec(fold_intersection);
11368 /* Combine the two lists into one. */
11369 if (l1_fold_invlist) {
11371 _invlist_union(nonbitmap, l1_fold_invlist, &nonbitmap);
11372 SvREFCNT_dec(l1_fold_invlist);
11375 nonbitmap = l1_fold_invlist;
11379 /* And combine the result (if any) with any inversion list from properties.
11380 * The lists are kept separate up to now because we don't want to fold the
11384 _invlist_union(nonbitmap, properties, &nonbitmap);
11385 SvREFCNT_dec(properties);
11388 nonbitmap = properties;
11392 /* Here, <nonbitmap> contains all the code points we can determine at
11393 * compile time that we haven't put into the bitmap. Go through it, and
11394 * for things that belong in the bitmap, put them there, and delete from
11398 /* Above-ASCII code points in /d have to stay in <nonbitmap>, as they
11399 * possibly only should match when the target string is UTF-8 */
11400 UV max_cp_to_set = (DEPENDS_SEMANTICS) ? 127 : 255;
11402 /* This gets set if we actually need to modify things */
11403 bool change_invlist = FALSE;
11407 /* Start looking through <nonbitmap> */
11408 invlist_iterinit(nonbitmap);
11409 while (invlist_iternext(nonbitmap, &start, &end)) {
11413 /* Quit if are above what we should change */
11414 if (start > max_cp_to_set) {
11418 change_invlist = TRUE;
11420 /* Set all the bits in the range, up to the max that we are doing */
11421 high = (end < max_cp_to_set) ? end : max_cp_to_set;
11422 for (i = start; i <= (int) high; i++) {
11423 if (! ANYOF_BITMAP_TEST(ret, i)) {
11424 ANYOF_BITMAP_SET(ret, i);
11432 /* Done with loop; remove any code points that are in the bitmap from
11434 if (change_invlist) {
11435 _invlist_subtract(nonbitmap,
11436 (DEPENDS_SEMANTICS)
11442 /* If have completely emptied it, remove it completely */
11443 if (invlist_len(nonbitmap) == 0) {
11444 SvREFCNT_dec(nonbitmap);
11449 /* Here, we have calculated what code points should be in the character
11450 * class. <nonbitmap> does not overlap the bitmap except possibly in the
11451 * case of DEPENDS rules.
11453 * Now we can see about various optimizations. Fold calculation (which we
11454 * did above) needs to take place before inversion. Otherwise /[^k]/i
11455 * would invert to include K, which under /i would match k, which it
11458 /* Optimize inverted simple patterns (e.g. [^a-z]). Note that we haven't
11459 * set the FOLD flag yet, so this does optimize those. It doesn't
11460 * optimize locale. Doing so perhaps could be done as long as there is
11461 * nothing like \w in it; some thought also would have to be given to the
11462 * interaction with above 0x100 chars */
11463 if ((ANYOF_FLAGS(ret) & ANYOF_INVERT)
11465 && ! unicode_alternate
11466 /* In case of /d, there are some things that should match only when in
11467 * not in the bitmap, i.e., they require UTF8 to match. These are
11468 * listed in nonbitmap, but if ANYOF_NONBITMAP_NON_UTF8 is set in this
11469 * case, they don't require UTF8, so can invert here */
11471 || ! DEPENDS_SEMANTICS
11472 || (ANYOF_FLAGS(ret) & ANYOF_NONBITMAP_NON_UTF8))
11473 && SvCUR(listsv) == initial_listsv_len)
11477 for (i = 0; i < 256; ++i) {
11478 if (ANYOF_BITMAP_TEST(ret, i)) {
11479 ANYOF_BITMAP_CLEAR(ret, i);
11482 ANYOF_BITMAP_SET(ret, i);
11487 /* The inversion means that everything above 255 is matched */
11488 ANYOF_FLAGS(ret) |= ANYOF_UNICODE_ALL;
11491 /* Here, also has things outside the bitmap that may overlap with
11492 * the bitmap. We have to sync them up, so that they get inverted
11493 * in both places. Earlier, we removed all overlaps except in the
11494 * case of /d rules, so no syncing is needed except for this case
11496 SV *remove_list = NULL;
11498 if (DEPENDS_SEMANTICS) {
11501 /* Set the bits that correspond to the ones that aren't in the
11502 * bitmap. Otherwise, when we invert, we'll miss these.
11503 * Earlier, we removed from the nonbitmap all code points
11504 * < 128, so there is no extra work here */
11505 invlist_iterinit(nonbitmap);
11506 while (invlist_iternext(nonbitmap, &start, &end)) {
11507 if (start > 255) { /* The bit map goes to 255 */
11513 for (i = start; i <= (int) end; ++i) {
11514 ANYOF_BITMAP_SET(ret, i);
11521 /* Now invert both the bitmap and the nonbitmap. Anything in the
11522 * bitmap has to also be removed from the non-bitmap, but again,
11523 * there should not be overlap unless is /d rules. */
11524 _invlist_invert(nonbitmap);
11526 /* Any swash can't be used as-is, because we've inverted things */
11528 SvREFCNT_dec(swash);
11532 for (i = 0; i < 256; ++i) {
11533 if (ANYOF_BITMAP_TEST(ret, i)) {
11534 ANYOF_BITMAP_CLEAR(ret, i);
11535 if (DEPENDS_SEMANTICS) {
11536 if (! remove_list) {
11537 remove_list = _new_invlist(2);
11539 remove_list = add_cp_to_invlist(remove_list, i);
11543 ANYOF_BITMAP_SET(ret, i);
11549 /* And do the removal */
11550 if (DEPENDS_SEMANTICS) {
11552 _invlist_subtract(nonbitmap, remove_list, &nonbitmap);
11553 SvREFCNT_dec(remove_list);
11557 /* There is no overlap for non-/d, so just delete anything
11559 _invlist_intersection(nonbitmap, PL_AboveLatin1, &nonbitmap);
11563 stored = 256 - stored;
11565 /* Clear the invert flag since have just done it here */
11566 ANYOF_FLAGS(ret) &= ~ANYOF_INVERT;
11569 /* Folding in the bitmap is taken care of above, but not for locale (for
11570 * which we have to wait to see what folding is in effect at runtime), and
11571 * for some things not in the bitmap (only the upper latin folds in this
11572 * case, as all other single-char folding has been set above). Set
11573 * run-time fold flag for these */
11575 || (DEPENDS_SEMANTICS
11577 && ! (ANYOF_FLAGS(ret) & ANYOF_NONBITMAP_NON_UTF8))
11578 || unicode_alternate))
11580 ANYOF_FLAGS(ret) |= ANYOF_LOC_NONBITMAP_FOLD;
11583 /* A single character class can be "optimized" into an EXACTish node.
11584 * Note that since we don't currently count how many characters there are
11585 * outside the bitmap, we are XXX missing optimization possibilities for
11586 * them. This optimization can't happen unless this is a truly single
11587 * character class, which means that it can't be an inversion into a
11588 * many-character class, and there must be no possibility of there being
11589 * things outside the bitmap. 'stored' (only) for locales doesn't include
11590 * \w, etc, so have to make a special test that they aren't present
11592 * Similarly A 2-character class of the very special form like [bB] can be
11593 * optimized into an EXACTFish node, but only for non-locales, and for
11594 * characters which only have the two folds; so things like 'fF' and 'Ii'
11595 * wouldn't work because they are part of the fold of 'LATIN SMALL LIGATURE
11598 && ! unicode_alternate
11599 && SvCUR(listsv) == initial_listsv_len
11600 && ! (ANYOF_FLAGS(ret) & (ANYOF_INVERT|ANYOF_UNICODE_ALL))
11601 && (((stored == 1 && ((! (ANYOF_FLAGS(ret) & ANYOF_LOCALE))
11602 || (! ANYOF_CLASS_TEST_ANY_SET(ret)))))
11603 || (stored == 2 && ((! (ANYOF_FLAGS(ret) & ANYOF_LOCALE))
11604 && (! _HAS_NONLATIN1_FOLD_CLOSURE_ONLY_FOR_USE_BY_REGCOMP_DOT_C_AND_REGEXEC_DOT_C(value))
11605 /* If the latest code point has a fold whose
11606 * bit is set, it must be the only other one */
11607 && ((prevvalue = PL_fold_latin1[value]) != (IV)value)
11608 && ANYOF_BITMAP_TEST(ret, prevvalue)))))
11610 /* Note that the information needed to decide to do this optimization
11611 * is not currently available until the 2nd pass, and that the actually
11612 * used EXACTish node takes less space than the calculated ANYOF node,
11613 * and hence the amount of space calculated in the first pass is larger
11614 * than actually used, so this optimization doesn't gain us any space.
11615 * But an EXACT node is faster than an ANYOF node, and can be combined
11616 * with any adjacent EXACT nodes later by the optimizer for further
11617 * gains. The speed of executing an EXACTF is similar to an ANYOF
11618 * node, so the optimization advantage comes from the ability to join
11619 * it to adjacent EXACT nodes */
11621 const char * cur_parse= RExC_parse;
11623 RExC_emit = (regnode *)orig_emit;
11624 RExC_parse = (char *)orig_parse;
11628 /* A locale node with one point can be folded; all the other cases
11629 * with folding will have two points, since we calculate them above
11631 if (ANYOF_FLAGS(ret) & ANYOF_LOC_NONBITMAP_FOLD) {
11638 else { /* else 2 chars in the bit map: the folds of each other */
11640 /* Use the folded value, which for the cases where we get here,
11641 * is just the lower case of the current one (which may resolve to
11642 * itself, or to the other one */
11643 value = toLOWER_LATIN1(value);
11645 /* To join adjacent nodes, they must be the exact EXACTish type.
11646 * Try to use the most likely type, by using EXACTFA if possible,
11647 * then EXACTFU if the regex calls for it, or is required because
11648 * the character is non-ASCII. (If <value> is ASCII, its fold is
11649 * also ASCII for the cases where we get here.) */
11650 if (MORE_ASCII_RESTRICTED && isASCII(value)) {
11653 else if (AT_LEAST_UNI_SEMANTICS || !isASCII(value)) {
11656 else { /* Otherwise, more likely to be EXACTF type */
11661 ret = reg_node(pRExC_state, op);
11662 RExC_parse = (char *)cur_parse;
11663 if (UTF && ! NATIVE_IS_INVARIANT(value)) {
11664 *STRING(ret)= UTF8_EIGHT_BIT_HI((U8) value);
11665 *(STRING(ret) + 1)= UTF8_EIGHT_BIT_LO((U8) value);
11667 RExC_emit += STR_SZ(2);
11670 *STRING(ret)= (char)value;
11672 RExC_emit += STR_SZ(1);
11674 SvREFCNT_dec(listsv);
11678 /* If there is a swash and more than one element, we can't use the swash in
11679 * the optimization below. */
11680 if (swash && element_count > 1) {
11681 SvREFCNT_dec(swash);
11685 && SvCUR(listsv) == initial_listsv_len
11686 && ! unicode_alternate)
11688 ARG_SET(ret, ANYOF_NONBITMAP_EMPTY);
11689 SvREFCNT_dec(listsv);
11690 SvREFCNT_dec(unicode_alternate);
11693 /* av[0] stores the character class description in its textual form:
11694 * used later (regexec.c:Perl_regclass_swash()) to initialize the
11695 * appropriate swash, and is also useful for dumping the regnode.
11696 * av[1] if NULL, is a placeholder to later contain the swash computed
11697 * from av[0]. But if no further computation need be done, the
11698 * swash is stored there now.
11699 * av[2] stores the multicharacter foldings, used later in
11700 * regexec.c:S_reginclass().
11701 * av[3] stores the nonbitmap inversion list for use in addition or
11702 * instead of av[0]; not used if av[1] isn't NULL
11703 * av[4] is set if any component of the class is from a user-defined
11704 * property; not used if av[1] isn't NULL */
11705 AV * const av = newAV();
11708 av_store(av, 0, (SvCUR(listsv) == initial_listsv_len)
11712 av_store(av, 1, swash);
11713 SvREFCNT_dec(nonbitmap);
11716 av_store(av, 1, NULL);
11718 av_store(av, 3, nonbitmap);
11719 av_store(av, 4, newSVuv(has_user_defined_property));
11723 /* Store any computed multi-char folds only if we are allowing
11725 if (allow_full_fold) {
11726 av_store(av, 2, MUTABLE_SV(unicode_alternate));
11727 if (unicode_alternate) { /* This node is variable length */
11732 av_store(av, 2, NULL);
11734 rv = newRV_noinc(MUTABLE_SV(av));
11735 n = add_data(pRExC_state, 1, "s");
11736 RExC_rxi->data->data[n] = (void*)rv;
11743 /* reg_skipcomment()
11745 Absorbs an /x style # comments from the input stream.
11746 Returns true if there is more text remaining in the stream.
11747 Will set the REG_SEEN_RUN_ON_COMMENT flag if the comment
11748 terminates the pattern without including a newline.
11750 Note its the callers responsibility to ensure that we are
11751 actually in /x mode
11756 S_reg_skipcomment(pTHX_ RExC_state_t *pRExC_state)
11760 PERL_ARGS_ASSERT_REG_SKIPCOMMENT;
11762 while (RExC_parse < RExC_end)
11763 if (*RExC_parse++ == '\n') {
11768 /* we ran off the end of the pattern without ending
11769 the comment, so we have to add an \n when wrapping */
11770 RExC_seen |= REG_SEEN_RUN_ON_COMMENT;
11778 Advances the parse position, and optionally absorbs
11779 "whitespace" from the inputstream.
11781 Without /x "whitespace" means (?#...) style comments only,
11782 with /x this means (?#...) and # comments and whitespace proper.
11784 Returns the RExC_parse point from BEFORE the scan occurs.
11786 This is the /x friendly way of saying RExC_parse++.
11790 S_nextchar(pTHX_ RExC_state_t *pRExC_state)
11792 char* const retval = RExC_parse++;
11794 PERL_ARGS_ASSERT_NEXTCHAR;
11797 if (RExC_end - RExC_parse >= 3
11798 && *RExC_parse == '('
11799 && RExC_parse[1] == '?'
11800 && RExC_parse[2] == '#')
11802 while (*RExC_parse != ')') {
11803 if (RExC_parse == RExC_end)
11804 FAIL("Sequence (?#... not terminated");
11810 if (RExC_flags & RXf_PMf_EXTENDED) {
11811 if (isSPACE(*RExC_parse)) {
11815 else if (*RExC_parse == '#') {
11816 if ( reg_skipcomment( pRExC_state ) )
11825 - reg_node - emit a node
11827 STATIC regnode * /* Location. */
11828 S_reg_node(pTHX_ RExC_state_t *pRExC_state, U8 op)
11831 register regnode *ptr;
11832 regnode * const ret = RExC_emit;
11833 GET_RE_DEBUG_FLAGS_DECL;
11835 PERL_ARGS_ASSERT_REG_NODE;
11838 SIZE_ALIGN(RExC_size);
11842 if (RExC_emit >= RExC_emit_bound)
11843 Perl_croak(aTHX_ "panic: reg_node overrun trying to emit %d, %p>=%p",
11844 op, RExC_emit, RExC_emit_bound);
11846 NODE_ALIGN_FILL(ret);
11848 FILL_ADVANCE_NODE(ptr, op);
11849 REH_CALL_COMP_NODE_HOOK(pRExC_state->rx, (ptr) - 1);
11850 #ifdef RE_TRACK_PATTERN_OFFSETS
11851 if (RExC_offsets) { /* MJD */
11852 MJD_OFFSET_DEBUG(("%s:%d: (op %s) %s %"UVuf" (len %"UVuf") (max %"UVuf").\n",
11853 "reg_node", __LINE__,
11855 (UV)(RExC_emit - RExC_emit_start) > RExC_offsets[0]
11856 ? "Overwriting end of array!\n" : "OK",
11857 (UV)(RExC_emit - RExC_emit_start),
11858 (UV)(RExC_parse - RExC_start),
11859 (UV)RExC_offsets[0]));
11860 Set_Node_Offset(RExC_emit, RExC_parse + (op == END));
11868 - reganode - emit a node with an argument
11870 STATIC regnode * /* Location. */
11871 S_reganode(pTHX_ RExC_state_t *pRExC_state, U8 op, U32 arg)
11874 register regnode *ptr;
11875 regnode * const ret = RExC_emit;
11876 GET_RE_DEBUG_FLAGS_DECL;
11878 PERL_ARGS_ASSERT_REGANODE;
11881 SIZE_ALIGN(RExC_size);
11886 assert(2==regarglen[op]+1);
11888 Anything larger than this has to allocate the extra amount.
11889 If we changed this to be:
11891 RExC_size += (1 + regarglen[op]);
11893 then it wouldn't matter. Its not clear what side effect
11894 might come from that so its not done so far.
11899 if (RExC_emit >= RExC_emit_bound)
11900 Perl_croak(aTHX_ "panic: reg_node overrun trying to emit %d, %p>=%p",
11901 op, RExC_emit, RExC_emit_bound);
11903 NODE_ALIGN_FILL(ret);
11905 FILL_ADVANCE_NODE_ARG(ptr, op, arg);
11906 REH_CALL_COMP_NODE_HOOK(pRExC_state->rx, (ptr) - 2);
11907 #ifdef RE_TRACK_PATTERN_OFFSETS
11908 if (RExC_offsets) { /* MJD */
11909 MJD_OFFSET_DEBUG(("%s(%d): (op %s) %s %"UVuf" <- %"UVuf" (max %"UVuf").\n",
11913 (UV)(RExC_emit - RExC_emit_start) > RExC_offsets[0] ?
11914 "Overwriting end of array!\n" : "OK",
11915 (UV)(RExC_emit - RExC_emit_start),
11916 (UV)(RExC_parse - RExC_start),
11917 (UV)RExC_offsets[0]));
11918 Set_Cur_Node_Offset;
11926 - reguni - emit (if appropriate) a Unicode character
11929 S_reguni(pTHX_ const RExC_state_t *pRExC_state, UV uv, char* s)
11933 PERL_ARGS_ASSERT_REGUNI;
11935 return SIZE_ONLY ? UNISKIP(uv) : (uvchr_to_utf8((U8*)s, uv) - (U8*)s);
11939 - reginsert - insert an operator in front of already-emitted operand
11941 * Means relocating the operand.
11944 S_reginsert(pTHX_ RExC_state_t *pRExC_state, U8 op, regnode *opnd, U32 depth)
11947 register regnode *src;
11948 register regnode *dst;
11949 register regnode *place;
11950 const int offset = regarglen[(U8)op];
11951 const int size = NODE_STEP_REGNODE + offset;
11952 GET_RE_DEBUG_FLAGS_DECL;
11954 PERL_ARGS_ASSERT_REGINSERT;
11955 PERL_UNUSED_ARG(depth);
11956 /* (PL_regkind[(U8)op] == CURLY ? EXTRA_STEP_2ARGS : 0); */
11957 DEBUG_PARSE_FMT("inst"," - %s",PL_reg_name[op]);
11966 if (RExC_open_parens) {
11968 /*DEBUG_PARSE_FMT("inst"," - %"IVdf, (IV)RExC_npar);*/
11969 for ( paren=0 ; paren < RExC_npar ; paren++ ) {
11970 if ( RExC_open_parens[paren] >= opnd ) {
11971 /*DEBUG_PARSE_FMT("open"," - %d",size);*/
11972 RExC_open_parens[paren] += size;
11974 /*DEBUG_PARSE_FMT("open"," - %s","ok");*/
11976 if ( RExC_close_parens[paren] >= opnd ) {
11977 /*DEBUG_PARSE_FMT("close"," - %d",size);*/
11978 RExC_close_parens[paren] += size;
11980 /*DEBUG_PARSE_FMT("close"," - %s","ok");*/
11985 while (src > opnd) {
11986 StructCopy(--src, --dst, regnode);
11987 #ifdef RE_TRACK_PATTERN_OFFSETS
11988 if (RExC_offsets) { /* MJD 20010112 */
11989 MJD_OFFSET_DEBUG(("%s(%d): (op %s) %s copy %"UVuf" -> %"UVuf" (max %"UVuf").\n",
11993 (UV)(dst - RExC_emit_start) > RExC_offsets[0]
11994 ? "Overwriting end of array!\n" : "OK",
11995 (UV)(src - RExC_emit_start),
11996 (UV)(dst - RExC_emit_start),
11997 (UV)RExC_offsets[0]));
11998 Set_Node_Offset_To_R(dst-RExC_emit_start, Node_Offset(src));
11999 Set_Node_Length_To_R(dst-RExC_emit_start, Node_Length(src));
12005 place = opnd; /* Op node, where operand used to be. */
12006 #ifdef RE_TRACK_PATTERN_OFFSETS
12007 if (RExC_offsets) { /* MJD */
12008 MJD_OFFSET_DEBUG(("%s(%d): (op %s) %s %"UVuf" <- %"UVuf" (max %"UVuf").\n",
12012 (UV)(place - RExC_emit_start) > RExC_offsets[0]
12013 ? "Overwriting end of array!\n" : "OK",
12014 (UV)(place - RExC_emit_start),
12015 (UV)(RExC_parse - RExC_start),
12016 (UV)RExC_offsets[0]));
12017 Set_Node_Offset(place, RExC_parse);
12018 Set_Node_Length(place, 1);
12021 src = NEXTOPER(place);
12022 FILL_ADVANCE_NODE(place, op);
12023 REH_CALL_COMP_NODE_HOOK(pRExC_state->rx, (place) - 1);
12024 Zero(src, offset, regnode);
12028 - regtail - set the next-pointer at the end of a node chain of p to val.
12029 - SEE ALSO: regtail_study
12031 /* TODO: All three parms should be const */
12033 S_regtail(pTHX_ RExC_state_t *pRExC_state, regnode *p, const regnode *val,U32 depth)
12036 register regnode *scan;
12037 GET_RE_DEBUG_FLAGS_DECL;
12039 PERL_ARGS_ASSERT_REGTAIL;
12041 PERL_UNUSED_ARG(depth);
12047 /* Find last node. */
12050 regnode * const temp = regnext(scan);
12052 SV * const mysv=sv_newmortal();
12053 DEBUG_PARSE_MSG((scan==p ? "tail" : ""));
12054 regprop(RExC_rx, mysv, scan);
12055 PerlIO_printf(Perl_debug_log, "~ %s (%d) %s %s\n",
12056 SvPV_nolen_const(mysv), REG_NODE_NUM(scan),
12057 (temp == NULL ? "->" : ""),
12058 (temp == NULL ? PL_reg_name[OP(val)] : "")
12066 if (reg_off_by_arg[OP(scan)]) {
12067 ARG_SET(scan, val - scan);
12070 NEXT_OFF(scan) = val - scan;
12076 - regtail_study - set the next-pointer at the end of a node chain of p to val.
12077 - Look for optimizable sequences at the same time.
12078 - currently only looks for EXACT chains.
12080 This is experimental code. The idea is to use this routine to perform
12081 in place optimizations on branches and groups as they are constructed,
12082 with the long term intention of removing optimization from study_chunk so
12083 that it is purely analytical.
12085 Currently only used when in DEBUG mode. The macro REGTAIL_STUDY() is used
12086 to control which is which.
12089 /* TODO: All four parms should be const */
12092 S_regtail_study(pTHX_ RExC_state_t *pRExC_state, regnode *p, const regnode *val,U32 depth)
12095 register regnode *scan;
12097 #ifdef EXPERIMENTAL_INPLACESCAN
12100 GET_RE_DEBUG_FLAGS_DECL;
12102 PERL_ARGS_ASSERT_REGTAIL_STUDY;
12108 /* Find last node. */
12112 regnode * const temp = regnext(scan);
12113 #ifdef EXPERIMENTAL_INPLACESCAN
12114 if (PL_regkind[OP(scan)] == EXACT) {
12115 bool has_exactf_sharp_s; /* Unexamined in this routine */
12116 if (join_exact(pRExC_state,scan,&min, &has_exactf_sharp_s, 1,val,depth+1))
12121 switch (OP(scan)) {
12127 case EXACTFU_TRICKYFOLD:
12129 if( exact == PSEUDO )
12131 else if ( exact != OP(scan) )
12140 SV * const mysv=sv_newmortal();
12141 DEBUG_PARSE_MSG((scan==p ? "tsdy" : ""));
12142 regprop(RExC_rx, mysv, scan);
12143 PerlIO_printf(Perl_debug_log, "~ %s (%d) -> %s\n",
12144 SvPV_nolen_const(mysv),
12145 REG_NODE_NUM(scan),
12146 PL_reg_name[exact]);
12153 SV * const mysv_val=sv_newmortal();
12154 DEBUG_PARSE_MSG("");
12155 regprop(RExC_rx, mysv_val, val);
12156 PerlIO_printf(Perl_debug_log, "~ attach to %s (%"IVdf") offset to %"IVdf"\n",
12157 SvPV_nolen_const(mysv_val),
12158 (IV)REG_NODE_NUM(val),
12162 if (reg_off_by_arg[OP(scan)]) {
12163 ARG_SET(scan, val - scan);
12166 NEXT_OFF(scan) = val - scan;
12174 - regdump - dump a regexp onto Perl_debug_log in vaguely comprehensible form
12178 S_regdump_extflags(pTHX_ const char *lead, const U32 flags)
12184 for (bit=0; bit<32; bit++) {
12185 if (flags & (1<<bit)) {
12186 if ((1<<bit) & RXf_PMf_CHARSET) { /* Output separately, below */
12189 if (!set++ && lead)
12190 PerlIO_printf(Perl_debug_log, "%s",lead);
12191 PerlIO_printf(Perl_debug_log, "%s ",PL_reg_extflags_name[bit]);
12194 if ((cs = get_regex_charset(flags)) != REGEX_DEPENDS_CHARSET) {
12195 if (!set++ && lead) {
12196 PerlIO_printf(Perl_debug_log, "%s",lead);
12199 case REGEX_UNICODE_CHARSET:
12200 PerlIO_printf(Perl_debug_log, "UNICODE");
12202 case REGEX_LOCALE_CHARSET:
12203 PerlIO_printf(Perl_debug_log, "LOCALE");
12205 case REGEX_ASCII_RESTRICTED_CHARSET:
12206 PerlIO_printf(Perl_debug_log, "ASCII-RESTRICTED");
12208 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
12209 PerlIO_printf(Perl_debug_log, "ASCII-MORE_RESTRICTED");
12212 PerlIO_printf(Perl_debug_log, "UNKNOWN CHARACTER SET");
12218 PerlIO_printf(Perl_debug_log, "\n");
12220 PerlIO_printf(Perl_debug_log, "%s[none-set]\n",lead);
12226 Perl_regdump(pTHX_ const regexp *r)
12230 SV * const sv = sv_newmortal();
12231 SV *dsv= sv_newmortal();
12232 RXi_GET_DECL(r,ri);
12233 GET_RE_DEBUG_FLAGS_DECL;
12235 PERL_ARGS_ASSERT_REGDUMP;
12237 (void)dumpuntil(r, ri->program, ri->program + 1, NULL, NULL, sv, 0, 0);
12239 /* Header fields of interest. */
12240 if (r->anchored_substr) {
12241 RE_PV_QUOTED_DECL(s, 0, dsv, SvPVX_const(r->anchored_substr),
12242 RE_SV_DUMPLEN(r->anchored_substr), 30);
12243 PerlIO_printf(Perl_debug_log,
12244 "anchored %s%s at %"IVdf" ",
12245 s, RE_SV_TAIL(r->anchored_substr),
12246 (IV)r->anchored_offset);
12247 } else if (r->anchored_utf8) {
12248 RE_PV_QUOTED_DECL(s, 1, dsv, SvPVX_const(r->anchored_utf8),
12249 RE_SV_DUMPLEN(r->anchored_utf8), 30);
12250 PerlIO_printf(Perl_debug_log,
12251 "anchored utf8 %s%s at %"IVdf" ",
12252 s, RE_SV_TAIL(r->anchored_utf8),
12253 (IV)r->anchored_offset);
12255 if (r->float_substr) {
12256 RE_PV_QUOTED_DECL(s, 0, dsv, SvPVX_const(r->float_substr),
12257 RE_SV_DUMPLEN(r->float_substr), 30);
12258 PerlIO_printf(Perl_debug_log,
12259 "floating %s%s at %"IVdf"..%"UVuf" ",
12260 s, RE_SV_TAIL(r->float_substr),
12261 (IV)r->float_min_offset, (UV)r->float_max_offset);
12262 } else if (r->float_utf8) {
12263 RE_PV_QUOTED_DECL(s, 1, dsv, SvPVX_const(r->float_utf8),
12264 RE_SV_DUMPLEN(r->float_utf8), 30);
12265 PerlIO_printf(Perl_debug_log,
12266 "floating utf8 %s%s at %"IVdf"..%"UVuf" ",
12267 s, RE_SV_TAIL(r->float_utf8),
12268 (IV)r->float_min_offset, (UV)r->float_max_offset);
12270 if (r->check_substr || r->check_utf8)
12271 PerlIO_printf(Perl_debug_log,
12273 (r->check_substr == r->float_substr
12274 && r->check_utf8 == r->float_utf8
12275 ? "(checking floating" : "(checking anchored"));
12276 if (r->extflags & RXf_NOSCAN)
12277 PerlIO_printf(Perl_debug_log, " noscan");
12278 if (r->extflags & RXf_CHECK_ALL)
12279 PerlIO_printf(Perl_debug_log, " isall");
12280 if (r->check_substr || r->check_utf8)
12281 PerlIO_printf(Perl_debug_log, ") ");
12283 if (ri->regstclass) {
12284 regprop(r, sv, ri->regstclass);
12285 PerlIO_printf(Perl_debug_log, "stclass %s ", SvPVX_const(sv));
12287 if (r->extflags & RXf_ANCH) {
12288 PerlIO_printf(Perl_debug_log, "anchored");
12289 if (r->extflags & RXf_ANCH_BOL)
12290 PerlIO_printf(Perl_debug_log, "(BOL)");
12291 if (r->extflags & RXf_ANCH_MBOL)
12292 PerlIO_printf(Perl_debug_log, "(MBOL)");
12293 if (r->extflags & RXf_ANCH_SBOL)
12294 PerlIO_printf(Perl_debug_log, "(SBOL)");
12295 if (r->extflags & RXf_ANCH_GPOS)
12296 PerlIO_printf(Perl_debug_log, "(GPOS)");
12297 PerlIO_putc(Perl_debug_log, ' ');
12299 if (r->extflags & RXf_GPOS_SEEN)
12300 PerlIO_printf(Perl_debug_log, "GPOS:%"UVuf" ", (UV)r->gofs);
12301 if (r->intflags & PREGf_SKIP)
12302 PerlIO_printf(Perl_debug_log, "plus ");
12303 if (r->intflags & PREGf_IMPLICIT)
12304 PerlIO_printf(Perl_debug_log, "implicit ");
12305 PerlIO_printf(Perl_debug_log, "minlen %"IVdf" ", (IV)r->minlen);
12306 if (r->extflags & RXf_EVAL_SEEN)
12307 PerlIO_printf(Perl_debug_log, "with eval ");
12308 PerlIO_printf(Perl_debug_log, "\n");
12309 DEBUG_FLAGS_r(regdump_extflags("r->extflags: ",r->extflags));
12311 PERL_ARGS_ASSERT_REGDUMP;
12312 PERL_UNUSED_CONTEXT;
12313 PERL_UNUSED_ARG(r);
12314 #endif /* DEBUGGING */
12318 - regprop - printable representation of opcode
12320 #define EMIT_ANYOF_TEST_SEPARATOR(do_sep,sv,flags) \
12323 Perl_sv_catpvf(aTHX_ sv,"%s][%s",PL_colors[1],PL_colors[0]); \
12324 if (flags & ANYOF_INVERT) \
12325 /*make sure the invert info is in each */ \
12326 sv_catpvs(sv, "^"); \
12332 Perl_regprop(pTHX_ const regexp *prog, SV *sv, const regnode *o)
12337 RXi_GET_DECL(prog,progi);
12338 GET_RE_DEBUG_FLAGS_DECL;
12340 PERL_ARGS_ASSERT_REGPROP;
12344 if (OP(o) > REGNODE_MAX) /* regnode.type is unsigned */
12345 /* It would be nice to FAIL() here, but this may be called from
12346 regexec.c, and it would be hard to supply pRExC_state. */
12347 Perl_croak(aTHX_ "Corrupted regexp opcode %d > %d", (int)OP(o), (int)REGNODE_MAX);
12348 sv_catpv(sv, PL_reg_name[OP(o)]); /* Take off const! */
12350 k = PL_regkind[OP(o)];
12353 sv_catpvs(sv, " ");
12354 /* Using is_utf8_string() (via PERL_PV_UNI_DETECT)
12355 * is a crude hack but it may be the best for now since
12356 * we have no flag "this EXACTish node was UTF-8"
12358 pv_pretty(sv, STRING(o), STR_LEN(o), 60, PL_colors[0], PL_colors[1],
12359 PERL_PV_ESCAPE_UNI_DETECT |
12360 PERL_PV_ESCAPE_NONASCII |
12361 PERL_PV_PRETTY_ELLIPSES |
12362 PERL_PV_PRETTY_LTGT |
12363 PERL_PV_PRETTY_NOCLEAR
12365 } else if (k == TRIE) {
12366 /* print the details of the trie in dumpuntil instead, as
12367 * progi->data isn't available here */
12368 const char op = OP(o);
12369 const U32 n = ARG(o);
12370 const reg_ac_data * const ac = IS_TRIE_AC(op) ?
12371 (reg_ac_data *)progi->data->data[n] :
12373 const reg_trie_data * const trie
12374 = (reg_trie_data*)progi->data->data[!IS_TRIE_AC(op) ? n : ac->trie];
12376 Perl_sv_catpvf(aTHX_ sv, "-%s",PL_reg_name[o->flags]);
12377 DEBUG_TRIE_COMPILE_r(
12378 Perl_sv_catpvf(aTHX_ sv,
12379 "<S:%"UVuf"/%"IVdf" W:%"UVuf" L:%"UVuf"/%"UVuf" C:%"UVuf"/%"UVuf">",
12380 (UV)trie->startstate,
12381 (IV)trie->statecount-1, /* -1 because of the unused 0 element */
12382 (UV)trie->wordcount,
12385 (UV)TRIE_CHARCOUNT(trie),
12386 (UV)trie->uniquecharcount
12389 if ( IS_ANYOF_TRIE(op) || trie->bitmap ) {
12391 int rangestart = -1;
12392 U8* bitmap = IS_ANYOF_TRIE(op) ? (U8*)ANYOF_BITMAP(o) : (U8*)TRIE_BITMAP(trie);
12393 sv_catpvs(sv, "[");
12394 for (i = 0; i <= 256; i++) {
12395 if (i < 256 && BITMAP_TEST(bitmap,i)) {
12396 if (rangestart == -1)
12398 } else if (rangestart != -1) {
12399 if (i <= rangestart + 3)
12400 for (; rangestart < i; rangestart++)
12401 put_byte(sv, rangestart);
12403 put_byte(sv, rangestart);
12404 sv_catpvs(sv, "-");
12405 put_byte(sv, i - 1);
12410 sv_catpvs(sv, "]");
12413 } else if (k == CURLY) {
12414 if (OP(o) == CURLYM || OP(o) == CURLYN || OP(o) == CURLYX)
12415 Perl_sv_catpvf(aTHX_ sv, "[%d]", o->flags); /* Parenth number */
12416 Perl_sv_catpvf(aTHX_ sv, " {%d,%d}", ARG1(o), ARG2(o));
12418 else if (k == WHILEM && o->flags) /* Ordinal/of */
12419 Perl_sv_catpvf(aTHX_ sv, "[%d/%d]", o->flags & 0xf, o->flags>>4);
12420 else if (k == REF || k == OPEN || k == CLOSE || k == GROUPP || OP(o)==ACCEPT) {
12421 Perl_sv_catpvf(aTHX_ sv, "%d", (int)ARG(o)); /* Parenth number */
12422 if ( RXp_PAREN_NAMES(prog) ) {
12423 if ( k != REF || (OP(o) < NREF)) {
12424 AV *list= MUTABLE_AV(progi->data->data[progi->name_list_idx]);
12425 SV **name= av_fetch(list, ARG(o), 0 );
12427 Perl_sv_catpvf(aTHX_ sv, " '%"SVf"'", SVfARG(*name));
12430 AV *list= MUTABLE_AV(progi->data->data[ progi->name_list_idx ]);
12431 SV *sv_dat= MUTABLE_SV(progi->data->data[ ARG( o ) ]);
12432 I32 *nums=(I32*)SvPVX(sv_dat);
12433 SV **name= av_fetch(list, nums[0], 0 );
12436 for ( n=0; n<SvIVX(sv_dat); n++ ) {
12437 Perl_sv_catpvf(aTHX_ sv, "%s%"IVdf,
12438 (n ? "," : ""), (IV)nums[n]);
12440 Perl_sv_catpvf(aTHX_ sv, " '%"SVf"'", SVfARG(*name));
12444 } else if (k == GOSUB)
12445 Perl_sv_catpvf(aTHX_ sv, "%d[%+d]", (int)ARG(o),(int)ARG2L(o)); /* Paren and offset */
12446 else if (k == VERB) {
12448 Perl_sv_catpvf(aTHX_ sv, ":%"SVf,
12449 SVfARG((MUTABLE_SV(progi->data->data[ ARG( o ) ]))));
12450 } else if (k == LOGICAL)
12451 Perl_sv_catpvf(aTHX_ sv, "[%d]", o->flags); /* 2: embedded, otherwise 1 */
12452 else if (k == ANYOF) {
12453 int i, rangestart = -1;
12454 const U8 flags = ANYOF_FLAGS(o);
12457 /* Should be synchronized with * ANYOF_ #xdefines in regcomp.h */
12458 static const char * const anyofs[] = {
12491 if (flags & ANYOF_LOCALE)
12492 sv_catpvs(sv, "{loc}");
12493 if (flags & ANYOF_LOC_NONBITMAP_FOLD)
12494 sv_catpvs(sv, "{i}");
12495 Perl_sv_catpvf(aTHX_ sv, "[%s", PL_colors[0]);
12496 if (flags & ANYOF_INVERT)
12497 sv_catpvs(sv, "^");
12499 /* output what the standard cp 0-255 bitmap matches */
12500 for (i = 0; i <= 256; i++) {
12501 if (i < 256 && ANYOF_BITMAP_TEST(o,i)) {
12502 if (rangestart == -1)
12504 } else if (rangestart != -1) {
12505 if (i <= rangestart + 3)
12506 for (; rangestart < i; rangestart++)
12507 put_byte(sv, rangestart);
12509 put_byte(sv, rangestart);
12510 sv_catpvs(sv, "-");
12511 put_byte(sv, i - 1);
12518 EMIT_ANYOF_TEST_SEPARATOR(do_sep,sv,flags);
12519 /* output any special charclass tests (used entirely under use locale) */
12520 if (ANYOF_CLASS_TEST_ANY_SET(o))
12521 for (i = 0; i < (int)(sizeof(anyofs)/sizeof(char*)); i++)
12522 if (ANYOF_CLASS_TEST(o,i)) {
12523 sv_catpv(sv, anyofs[i]);
12527 EMIT_ANYOF_TEST_SEPARATOR(do_sep,sv,flags);
12529 if (flags & ANYOF_NON_UTF8_LATIN1_ALL) {
12530 sv_catpvs(sv, "{non-utf8-latin1-all}");
12533 /* output information about the unicode matching */
12534 if (flags & ANYOF_UNICODE_ALL)
12535 sv_catpvs(sv, "{unicode_all}");
12536 else if (ANYOF_NONBITMAP(o))
12537 sv_catpvs(sv, "{unicode}");
12538 if (flags & ANYOF_NONBITMAP_NON_UTF8)
12539 sv_catpvs(sv, "{outside bitmap}");
12541 if (ANYOF_NONBITMAP(o)) {
12542 SV *lv; /* Set if there is something outside the bit map */
12543 SV * const sw = regclass_swash(prog, o, FALSE, &lv, 0);
12544 bool byte_output = FALSE; /* If something in the bitmap has been
12547 if (lv && lv != &PL_sv_undef) {
12549 U8 s[UTF8_MAXBYTES_CASE+1];
12551 for (i = 0; i <= 256; i++) { /* Look at chars in bitmap */
12552 uvchr_to_utf8(s, i);
12555 && ! ANYOF_BITMAP_TEST(o, i) /* Don't duplicate
12559 && swash_fetch(sw, s, TRUE))
12561 if (rangestart == -1)
12563 } else if (rangestart != -1) {
12564 byte_output = TRUE;
12565 if (i <= rangestart + 3)
12566 for (; rangestart < i; rangestart++) {
12567 put_byte(sv, rangestart);
12570 put_byte(sv, rangestart);
12571 sv_catpvs(sv, "-");
12580 char *s = savesvpv(lv);
12581 char * const origs = s;
12583 while (*s && *s != '\n')
12587 const char * const t = ++s;
12590 sv_catpvs(sv, " ");
12596 /* Truncate very long output */
12597 if (s - origs > 256) {
12598 Perl_sv_catpvf(aTHX_ sv,
12600 (int) (s - origs - 1),
12606 else if (*s == '\t') {
12625 Perl_sv_catpvf(aTHX_ sv, "%s]", PL_colors[1]);
12627 else if (k == BRANCHJ && (OP(o) == UNLESSM || OP(o) == IFMATCH))
12628 Perl_sv_catpvf(aTHX_ sv, "[%d]", -(o->flags));
12630 PERL_UNUSED_CONTEXT;
12631 PERL_UNUSED_ARG(sv);
12632 PERL_UNUSED_ARG(o);
12633 PERL_UNUSED_ARG(prog);
12634 #endif /* DEBUGGING */
12638 Perl_re_intuit_string(pTHX_ REGEXP * const r)
12639 { /* Assume that RE_INTUIT is set */
12641 struct regexp *const prog = (struct regexp *)SvANY(r);
12642 GET_RE_DEBUG_FLAGS_DECL;
12644 PERL_ARGS_ASSERT_RE_INTUIT_STRING;
12645 PERL_UNUSED_CONTEXT;
12649 const char * const s = SvPV_nolen_const(prog->check_substr
12650 ? prog->check_substr : prog->check_utf8);
12652 if (!PL_colorset) reginitcolors();
12653 PerlIO_printf(Perl_debug_log,
12654 "%sUsing REx %ssubstr:%s \"%s%.60s%s%s\"\n",
12656 prog->check_substr ? "" : "utf8 ",
12657 PL_colors[5],PL_colors[0],
12660 (strlen(s) > 60 ? "..." : ""));
12663 return prog->check_substr ? prog->check_substr : prog->check_utf8;
12669 handles refcounting and freeing the perl core regexp structure. When
12670 it is necessary to actually free the structure the first thing it
12671 does is call the 'free' method of the regexp_engine associated to
12672 the regexp, allowing the handling of the void *pprivate; member
12673 first. (This routine is not overridable by extensions, which is why
12674 the extensions free is called first.)
12676 See regdupe and regdupe_internal if you change anything here.
12678 #ifndef PERL_IN_XSUB_RE
12680 Perl_pregfree(pTHX_ REGEXP *r)
12686 Perl_pregfree2(pTHX_ REGEXP *rx)
12689 struct regexp *const r = (struct regexp *)SvANY(rx);
12690 GET_RE_DEBUG_FLAGS_DECL;
12692 PERL_ARGS_ASSERT_PREGFREE2;
12694 if (r->mother_re) {
12695 ReREFCNT_dec(r->mother_re);
12697 CALLREGFREE_PVT(rx); /* free the private data */
12698 SvREFCNT_dec(RXp_PAREN_NAMES(r));
12701 SvREFCNT_dec(r->anchored_substr);
12702 SvREFCNT_dec(r->anchored_utf8);
12703 SvREFCNT_dec(r->float_substr);
12704 SvREFCNT_dec(r->float_utf8);
12705 Safefree(r->substrs);
12707 RX_MATCH_COPY_FREE(rx);
12708 #ifdef PERL_OLD_COPY_ON_WRITE
12709 SvREFCNT_dec(r->saved_copy);
12716 This is a hacky workaround to the structural issue of match results
12717 being stored in the regexp structure which is in turn stored in
12718 PL_curpm/PL_reg_curpm. The problem is that due to qr// the pattern
12719 could be PL_curpm in multiple contexts, and could require multiple
12720 result sets being associated with the pattern simultaneously, such
12721 as when doing a recursive match with (??{$qr})
12723 The solution is to make a lightweight copy of the regexp structure
12724 when a qr// is returned from the code executed by (??{$qr}) this
12725 lightweight copy doesn't actually own any of its data except for
12726 the starp/end and the actual regexp structure itself.
12732 Perl_reg_temp_copy (pTHX_ REGEXP *ret_x, REGEXP *rx)
12734 struct regexp *ret;
12735 struct regexp *const r = (struct regexp *)SvANY(rx);
12736 register const I32 npar = r->nparens+1;
12738 PERL_ARGS_ASSERT_REG_TEMP_COPY;
12741 ret_x = (REGEXP*) newSV_type(SVt_REGEXP);
12742 ret = (struct regexp *)SvANY(ret_x);
12744 (void)ReREFCNT_inc(rx);
12745 /* We can take advantage of the existing "copied buffer" mechanism in SVs
12746 by pointing directly at the buffer, but flagging that the allocated
12747 space in the copy is zero. As we've just done a struct copy, it's now
12748 a case of zero-ing that, rather than copying the current length. */
12749 SvPV_set(ret_x, RX_WRAPPED(rx));
12750 SvFLAGS(ret_x) |= SvFLAGS(rx) & (SVf_POK|SVp_POK|SVf_UTF8);
12751 memcpy(&(ret->xpv_cur), &(r->xpv_cur),
12752 sizeof(regexp) - STRUCT_OFFSET(regexp, xpv_cur));
12753 SvLEN_set(ret_x, 0);
12754 SvSTASH_set(ret_x, NULL);
12755 SvMAGIC_set(ret_x, NULL);
12756 Newx(ret->offs, npar, regexp_paren_pair);
12757 Copy(r->offs, ret->offs, npar, regexp_paren_pair);
12759 Newx(ret->substrs, 1, struct reg_substr_data);
12760 StructCopy(r->substrs, ret->substrs, struct reg_substr_data);
12762 SvREFCNT_inc_void(ret->anchored_substr);
12763 SvREFCNT_inc_void(ret->anchored_utf8);
12764 SvREFCNT_inc_void(ret->float_substr);
12765 SvREFCNT_inc_void(ret->float_utf8);
12767 /* check_substr and check_utf8, if non-NULL, point to either their
12768 anchored or float namesakes, and don't hold a second reference. */
12770 RX_MATCH_COPIED_off(ret_x);
12771 #ifdef PERL_OLD_COPY_ON_WRITE
12772 ret->saved_copy = NULL;
12774 ret->mother_re = rx;
12780 /* regfree_internal()
12782 Free the private data in a regexp. This is overloadable by
12783 extensions. Perl takes care of the regexp structure in pregfree(),
12784 this covers the *pprivate pointer which technically perl doesn't
12785 know about, however of course we have to handle the
12786 regexp_internal structure when no extension is in use.
12788 Note this is called before freeing anything in the regexp
12793 Perl_regfree_internal(pTHX_ REGEXP * const rx)
12796 struct regexp *const r = (struct regexp *)SvANY(rx);
12797 RXi_GET_DECL(r,ri);
12798 GET_RE_DEBUG_FLAGS_DECL;
12800 PERL_ARGS_ASSERT_REGFREE_INTERNAL;
12806 SV *dsv= sv_newmortal();
12807 RE_PV_QUOTED_DECL(s, RX_UTF8(rx),
12808 dsv, RX_PRECOMP(rx), RX_PRELEN(rx), 60);
12809 PerlIO_printf(Perl_debug_log,"%sFreeing REx:%s %s\n",
12810 PL_colors[4],PL_colors[5],s);
12813 #ifdef RE_TRACK_PATTERN_OFFSETS
12815 Safefree(ri->u.offsets); /* 20010421 MJD */
12818 int n = ri->data->count;
12819 PAD* new_comppad = NULL;
12824 /* If you add a ->what type here, update the comment in regcomp.h */
12825 switch (ri->data->what[n]) {
12830 SvREFCNT_dec(MUTABLE_SV(ri->data->data[n]));
12833 Safefree(ri->data->data[n]);
12836 new_comppad = MUTABLE_AV(ri->data->data[n]);
12839 if (new_comppad == NULL)
12840 Perl_croak(aTHX_ "panic: pregfree comppad");
12841 PAD_SAVE_LOCAL(old_comppad,
12842 /* Watch out for global destruction's random ordering. */
12843 (SvTYPE(new_comppad) == SVt_PVAV) ? new_comppad : NULL
12846 refcnt = OpREFCNT_dec((OP_4tree*)ri->data->data[n]);
12849 op_free((OP_4tree*)ri->data->data[n]);
12851 PAD_RESTORE_LOCAL(old_comppad);
12852 SvREFCNT_dec(MUTABLE_SV(new_comppad));
12853 new_comppad = NULL;
12858 { /* Aho Corasick add-on structure for a trie node.
12859 Used in stclass optimization only */
12861 reg_ac_data *aho=(reg_ac_data*)ri->data->data[n];
12863 refcount = --aho->refcount;
12866 PerlMemShared_free(aho->states);
12867 PerlMemShared_free(aho->fail);
12868 /* do this last!!!! */
12869 PerlMemShared_free(ri->data->data[n]);
12870 PerlMemShared_free(ri->regstclass);
12876 /* trie structure. */
12878 reg_trie_data *trie=(reg_trie_data*)ri->data->data[n];
12880 refcount = --trie->refcount;
12883 PerlMemShared_free(trie->charmap);
12884 PerlMemShared_free(trie->states);
12885 PerlMemShared_free(trie->trans);
12887 PerlMemShared_free(trie->bitmap);
12889 PerlMemShared_free(trie->jump);
12890 PerlMemShared_free(trie->wordinfo);
12891 /* do this last!!!! */
12892 PerlMemShared_free(ri->data->data[n]);
12897 Perl_croak(aTHX_ "panic: regfree data code '%c'", ri->data->what[n]);
12900 Safefree(ri->data->what);
12901 Safefree(ri->data);
12907 #define av_dup_inc(s,t) MUTABLE_AV(sv_dup_inc((const SV *)s,t))
12908 #define hv_dup_inc(s,t) MUTABLE_HV(sv_dup_inc((const SV *)s,t))
12909 #define SAVEPVN(p,n) ((p) ? savepvn(p,n) : NULL)
12912 re_dup - duplicate a regexp.
12914 This routine is expected to clone a given regexp structure. It is only
12915 compiled under USE_ITHREADS.
12917 After all of the core data stored in struct regexp is duplicated
12918 the regexp_engine.dupe method is used to copy any private data
12919 stored in the *pprivate pointer. This allows extensions to handle
12920 any duplication it needs to do.
12922 See pregfree() and regfree_internal() if you change anything here.
12924 #if defined(USE_ITHREADS)
12925 #ifndef PERL_IN_XSUB_RE
12927 Perl_re_dup_guts(pTHX_ const REGEXP *sstr, REGEXP *dstr, CLONE_PARAMS *param)
12931 const struct regexp *r = (const struct regexp *)SvANY(sstr);
12932 struct regexp *ret = (struct regexp *)SvANY(dstr);
12934 PERL_ARGS_ASSERT_RE_DUP_GUTS;
12936 npar = r->nparens+1;
12937 Newx(ret->offs, npar, regexp_paren_pair);
12938 Copy(r->offs, ret->offs, npar, regexp_paren_pair);
12940 /* no need to copy these */
12941 Newx(ret->swap, npar, regexp_paren_pair);
12944 if (ret->substrs) {
12945 /* Do it this way to avoid reading from *r after the StructCopy().
12946 That way, if any of the sv_dup_inc()s dislodge *r from the L1
12947 cache, it doesn't matter. */
12948 const bool anchored = r->check_substr
12949 ? r->check_substr == r->anchored_substr
12950 : r->check_utf8 == r->anchored_utf8;
12951 Newx(ret->substrs, 1, struct reg_substr_data);
12952 StructCopy(r->substrs, ret->substrs, struct reg_substr_data);
12954 ret->anchored_substr = sv_dup_inc(ret->anchored_substr, param);
12955 ret->anchored_utf8 = sv_dup_inc(ret->anchored_utf8, param);
12956 ret->float_substr = sv_dup_inc(ret->float_substr, param);
12957 ret->float_utf8 = sv_dup_inc(ret->float_utf8, param);
12959 /* check_substr and check_utf8, if non-NULL, point to either their
12960 anchored or float namesakes, and don't hold a second reference. */
12962 if (ret->check_substr) {
12964 assert(r->check_utf8 == r->anchored_utf8);
12965 ret->check_substr = ret->anchored_substr;
12966 ret->check_utf8 = ret->anchored_utf8;
12968 assert(r->check_substr == r->float_substr);
12969 assert(r->check_utf8 == r->float_utf8);
12970 ret->check_substr = ret->float_substr;
12971 ret->check_utf8 = ret->float_utf8;
12973 } else if (ret->check_utf8) {
12975 ret->check_utf8 = ret->anchored_utf8;
12977 ret->check_utf8 = ret->float_utf8;
12982 RXp_PAREN_NAMES(ret) = hv_dup_inc(RXp_PAREN_NAMES(ret), param);
12985 RXi_SET(ret,CALLREGDUPE_PVT(dstr,param));
12987 if (RX_MATCH_COPIED(dstr))
12988 ret->subbeg = SAVEPVN(ret->subbeg, ret->sublen);
12990 ret->subbeg = NULL;
12991 #ifdef PERL_OLD_COPY_ON_WRITE
12992 ret->saved_copy = NULL;
12995 if (ret->mother_re) {
12996 if (SvPVX_const(dstr) == SvPVX_const(ret->mother_re)) {
12997 /* Our storage points directly to our mother regexp, but that's
12998 1: a buffer in a different thread
12999 2: something we no longer hold a reference on
13000 so we need to copy it locally. */
13001 /* Note we need to use SvCUR(), rather than
13002 SvLEN(), on our mother_re, because it, in
13003 turn, may well be pointing to its own mother_re. */
13004 SvPV_set(dstr, SAVEPVN(SvPVX_const(ret->mother_re),
13005 SvCUR(ret->mother_re)+1));
13006 SvLEN_set(dstr, SvCUR(ret->mother_re)+1);
13008 ret->mother_re = NULL;
13012 #endif /* PERL_IN_XSUB_RE */
13017 This is the internal complement to regdupe() which is used to copy
13018 the structure pointed to by the *pprivate pointer in the regexp.
13019 This is the core version of the extension overridable cloning hook.
13020 The regexp structure being duplicated will be copied by perl prior
13021 to this and will be provided as the regexp *r argument, however
13022 with the /old/ structures pprivate pointer value. Thus this routine
13023 may override any copying normally done by perl.
13025 It returns a pointer to the new regexp_internal structure.
13029 Perl_regdupe_internal(pTHX_ REGEXP * const rx, CLONE_PARAMS *param)
13032 struct regexp *const r = (struct regexp *)SvANY(rx);
13033 regexp_internal *reti;
13035 RXi_GET_DECL(r,ri);
13037 PERL_ARGS_ASSERT_REGDUPE_INTERNAL;
13041 Newxc(reti, sizeof(regexp_internal) + len*sizeof(regnode), char, regexp_internal);
13042 Copy(ri->program, reti->program, len+1, regnode);
13045 reti->regstclass = NULL;
13048 struct reg_data *d;
13049 const int count = ri->data->count;
13052 Newxc(d, sizeof(struct reg_data) + count*sizeof(void *),
13053 char, struct reg_data);
13054 Newx(d->what, count, U8);
13057 for (i = 0; i < count; i++) {
13058 d->what[i] = ri->data->what[i];
13059 switch (d->what[i]) {
13060 /* legal options are one of: sSfpontTua
13061 see also regcomp.h and pregfree() */
13062 case 'a': /* actually an AV, but the dup function is identical. */
13065 case 'p': /* actually an AV, but the dup function is identical. */
13066 case 'u': /* actually an HV, but the dup function is identical. */
13067 d->data[i] = sv_dup_inc((const SV *)ri->data->data[i], param);
13070 /* This is cheating. */
13071 Newx(d->data[i], 1, struct regnode_charclass_class);
13072 StructCopy(ri->data->data[i], d->data[i],
13073 struct regnode_charclass_class);
13074 reti->regstclass = (regnode*)d->data[i];
13077 /* Compiled op trees are readonly and in shared memory,
13078 and can thus be shared without duplication. */
13080 d->data[i] = (void*)OpREFCNT_inc((OP*)ri->data->data[i]);
13084 /* Trie stclasses are readonly and can thus be shared
13085 * without duplication. We free the stclass in pregfree
13086 * when the corresponding reg_ac_data struct is freed.
13088 reti->regstclass= ri->regstclass;
13092 ((reg_trie_data*)ri->data->data[i])->refcount++;
13096 d->data[i] = ri->data->data[i];
13099 Perl_croak(aTHX_ "panic: re_dup unknown data code '%c'", ri->data->what[i]);
13108 reti->name_list_idx = ri->name_list_idx;
13110 #ifdef RE_TRACK_PATTERN_OFFSETS
13111 if (ri->u.offsets) {
13112 Newx(reti->u.offsets, 2*len+1, U32);
13113 Copy(ri->u.offsets, reti->u.offsets, 2*len+1, U32);
13116 SetProgLen(reti,len);
13119 return (void*)reti;
13122 #endif /* USE_ITHREADS */
13124 #ifndef PERL_IN_XSUB_RE
13127 - regnext - dig the "next" pointer out of a node
13130 Perl_regnext(pTHX_ register regnode *p)
13133 register I32 offset;
13138 if (OP(p) > REGNODE_MAX) { /* regnode.type is unsigned */
13139 Perl_croak(aTHX_ "Corrupted regexp opcode %d > %d", (int)OP(p), (int)REGNODE_MAX);
13142 offset = (reg_off_by_arg[OP(p)] ? ARG(p) : NEXT_OFF(p));
13151 S_re_croak2(pTHX_ const char* pat1,const char* pat2,...)
13154 STRLEN l1 = strlen(pat1);
13155 STRLEN l2 = strlen(pat2);
13158 const char *message;
13160 PERL_ARGS_ASSERT_RE_CROAK2;
13166 Copy(pat1, buf, l1 , char);
13167 Copy(pat2, buf + l1, l2 , char);
13168 buf[l1 + l2] = '\n';
13169 buf[l1 + l2 + 1] = '\0';
13171 /* ANSI variant takes additional second argument */
13172 va_start(args, pat2);
13176 msv = vmess(buf, &args);
13178 message = SvPV_const(msv,l1);
13181 Copy(message, buf, l1 , char);
13182 buf[l1-1] = '\0'; /* Overwrite \n */
13183 Perl_croak(aTHX_ "%s", buf);
13186 /* XXX Here's a total kludge. But we need to re-enter for swash routines. */
13188 #ifndef PERL_IN_XSUB_RE
13190 Perl_save_re_context(pTHX)
13194 struct re_save_state *state;
13196 SAVEVPTR(PL_curcop);
13197 SSGROW(SAVESTACK_ALLOC_FOR_RE_SAVE_STATE + 1);
13199 state = (struct re_save_state *)(PL_savestack + PL_savestack_ix);
13200 PL_savestack_ix += SAVESTACK_ALLOC_FOR_RE_SAVE_STATE;
13201 SSPUSHUV(SAVEt_RE_STATE);
13203 Copy(&PL_reg_state, state, 1, struct re_save_state);
13205 PL_reg_start_tmp = 0;
13206 PL_reg_start_tmpl = 0;
13207 PL_reg_oldsaved = NULL;
13208 PL_reg_oldsavedlen = 0;
13209 PL_reg_maxiter = 0;
13210 PL_reg_leftiter = 0;
13211 PL_reg_poscache = NULL;
13212 PL_reg_poscache_size = 0;
13213 #ifdef PERL_OLD_COPY_ON_WRITE
13217 /* Save $1..$n (#18107: UTF-8 s/(\w+)/uc($1)/e); AMS 20021106. */
13219 const REGEXP * const rx = PM_GETRE(PL_curpm);
13222 for (i = 1; i <= RX_NPARENS(rx); i++) {
13223 char digits[TYPE_CHARS(long)];
13224 const STRLEN len = my_snprintf(digits, sizeof(digits), "%lu", (long)i);
13225 GV *const *const gvp
13226 = (GV**)hv_fetch(PL_defstash, digits, len, 0);
13229 GV * const gv = *gvp;
13230 if (SvTYPE(gv) == SVt_PVGV && GvSV(gv))
13240 clear_re(pTHX_ void *r)
13243 ReREFCNT_dec((REGEXP *)r);
13249 S_put_byte(pTHX_ SV *sv, int c)
13251 PERL_ARGS_ASSERT_PUT_BYTE;
13253 /* Our definition of isPRINT() ignores locales, so only bytes that are
13254 not part of UTF-8 are considered printable. I assume that the same
13255 holds for UTF-EBCDIC.
13256 Also, code point 255 is not printable in either (it's E0 in EBCDIC,
13257 which Wikipedia says:
13259 EO, or Eight Ones, is an 8-bit EBCDIC character code represented as all
13260 ones (binary 1111 1111, hexadecimal FF). It is similar, but not
13261 identical, to the ASCII delete (DEL) or rubout control character.
13262 ) So the old condition can be simplified to !isPRINT(c) */
13265 Perl_sv_catpvf(aTHX_ sv, "\\x%02x", c);
13268 Perl_sv_catpvf(aTHX_ sv, "\\x{%x}", c);
13272 const char string = c;
13273 if (c == '-' || c == ']' || c == '\\' || c == '^')
13274 sv_catpvs(sv, "\\");
13275 sv_catpvn(sv, &string, 1);
13280 #define CLEAR_OPTSTART \
13281 if (optstart) STMT_START { \
13282 DEBUG_OPTIMISE_r(PerlIO_printf(Perl_debug_log, " (%"IVdf" nodes)\n", (IV)(node - optstart))); \
13286 #define DUMPUNTIL(b,e) CLEAR_OPTSTART; node=dumpuntil(r,start,(b),(e),last,sv,indent+1,depth+1);
13288 STATIC const regnode *
13289 S_dumpuntil(pTHX_ const regexp *r, const regnode *start, const regnode *node,
13290 const regnode *last, const regnode *plast,
13291 SV* sv, I32 indent, U32 depth)
13294 register U8 op = PSEUDO; /* Arbitrary non-END op. */
13295 register const regnode *next;
13296 const regnode *optstart= NULL;
13298 RXi_GET_DECL(r,ri);
13299 GET_RE_DEBUG_FLAGS_DECL;
13301 PERL_ARGS_ASSERT_DUMPUNTIL;
13303 #ifdef DEBUG_DUMPUNTIL
13304 PerlIO_printf(Perl_debug_log, "--- %d : %d - %d - %d\n",indent,node-start,
13305 last ? last-start : 0,plast ? plast-start : 0);
13308 if (plast && plast < last)
13311 while (PL_regkind[op] != END && (!last || node < last)) {
13312 /* While that wasn't END last time... */
13315 if (op == CLOSE || op == WHILEM)
13317 next = regnext((regnode *)node);
13320 if (OP(node) == OPTIMIZED) {
13321 if (!optstart && RE_DEBUG_FLAG(RE_DEBUG_COMPILE_OPTIMISE))
13328 regprop(r, sv, node);
13329 PerlIO_printf(Perl_debug_log, "%4"IVdf":%*s%s", (IV)(node - start),
13330 (int)(2*indent + 1), "", SvPVX_const(sv));
13332 if (OP(node) != OPTIMIZED) {
13333 if (next == NULL) /* Next ptr. */
13334 PerlIO_printf(Perl_debug_log, " (0)");
13335 else if (PL_regkind[(U8)op] == BRANCH && PL_regkind[OP(next)] != BRANCH )
13336 PerlIO_printf(Perl_debug_log, " (FAIL)");
13338 PerlIO_printf(Perl_debug_log, " (%"IVdf")", (IV)(next - start));
13339 (void)PerlIO_putc(Perl_debug_log, '\n');
13343 if (PL_regkind[(U8)op] == BRANCHJ) {
13346 register const regnode *nnode = (OP(next) == LONGJMP
13347 ? regnext((regnode *)next)
13349 if (last && nnode > last)
13351 DUMPUNTIL(NEXTOPER(NEXTOPER(node)), nnode);
13354 else if (PL_regkind[(U8)op] == BRANCH) {
13356 DUMPUNTIL(NEXTOPER(node), next);
13358 else if ( PL_regkind[(U8)op] == TRIE ) {
13359 const regnode *this_trie = node;
13360 const char op = OP(node);
13361 const U32 n = ARG(node);
13362 const reg_ac_data * const ac = op>=AHOCORASICK ?
13363 (reg_ac_data *)ri->data->data[n] :
13365 const reg_trie_data * const trie =
13366 (reg_trie_data*)ri->data->data[op<AHOCORASICK ? n : ac->trie];
13368 AV *const trie_words = MUTABLE_AV(ri->data->data[n + TRIE_WORDS_OFFSET]);
13370 const regnode *nextbranch= NULL;
13373 for (word_idx= 0; word_idx < (I32)trie->wordcount; word_idx++) {
13374 SV ** const elem_ptr = av_fetch(trie_words,word_idx,0);
13376 PerlIO_printf(Perl_debug_log, "%*s%s ",
13377 (int)(2*(indent+3)), "",
13378 elem_ptr ? pv_pretty(sv, SvPV_nolen_const(*elem_ptr), SvCUR(*elem_ptr), 60,
13379 PL_colors[0], PL_colors[1],
13380 (SvUTF8(*elem_ptr) ? PERL_PV_ESCAPE_UNI : 0) |
13381 PERL_PV_PRETTY_ELLIPSES |
13382 PERL_PV_PRETTY_LTGT
13387 U16 dist= trie->jump[word_idx+1];
13388 PerlIO_printf(Perl_debug_log, "(%"UVuf")\n",
13389 (UV)((dist ? this_trie + dist : next) - start));
13392 nextbranch= this_trie + trie->jump[0];
13393 DUMPUNTIL(this_trie + dist, nextbranch);
13395 if (nextbranch && PL_regkind[OP(nextbranch)]==BRANCH)
13396 nextbranch= regnext((regnode *)nextbranch);
13398 PerlIO_printf(Perl_debug_log, "\n");
13401 if (last && next > last)
13406 else if ( op == CURLY ) { /* "next" might be very big: optimizer */
13407 DUMPUNTIL(NEXTOPER(node) + EXTRA_STEP_2ARGS,
13408 NEXTOPER(node) + EXTRA_STEP_2ARGS + 1);
13410 else if (PL_regkind[(U8)op] == CURLY && op != CURLYX) {
13412 DUMPUNTIL(NEXTOPER(node) + EXTRA_STEP_2ARGS, next);
13414 else if ( op == PLUS || op == STAR) {
13415 DUMPUNTIL(NEXTOPER(node), NEXTOPER(node) + 1);
13417 else if (PL_regkind[(U8)op] == ANYOF) {
13418 /* arglen 1 + class block */
13419 node += 1 + ((ANYOF_FLAGS(node) & ANYOF_CLASS)
13420 ? ANYOF_CLASS_SKIP : ANYOF_SKIP);
13421 node = NEXTOPER(node);
13423 else if (PL_regkind[(U8)op] == EXACT) {
13424 /* Literal string, where present. */
13425 node += NODE_SZ_STR(node) - 1;
13426 node = NEXTOPER(node);
13429 node = NEXTOPER(node);
13430 node += regarglen[(U8)op];
13432 if (op == CURLYX || op == OPEN)
13436 #ifdef DEBUG_DUMPUNTIL
13437 PerlIO_printf(Perl_debug_log, "--- %d\n", (int)indent);
13442 #endif /* DEBUGGING */
13446 * c-indentation-style: bsd
13447 * c-basic-offset: 4
13448 * indent-tabs-mode: t
13451 * ex: set ts=8 sts=4 sw=4 noet: