Skip to content

Commit 8927f96

Browse files
sergeyklaysergeyklay
committed
Fixed issue with incorrectly used YYMARKER and YYCURSOR
Refs: * skvadrik/re2c#197 * #31 * zephir-lang/zephir#1591 * phalcon/cphalcon#13140 Thanks to @skvadrik /cc @chilimatic
1 parent abe5ef5 commit 8927f96

File tree

3 files changed

+175
-33
lines changed

3 files changed

+175
-33
lines changed

‎parser/scanner.re

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
#define YYCTYPE unsigned char
1818
#define YYCURSOR (s->start)
1919
#define YYLIMIT (s->end)
20-
#define YYMARKER q
20+
#define YYMARKER qm
2121

2222
int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
2323

24-
char *q = YYCURSOR, *start = YYCURSOR;
24+
char *start = YYCURSOR, *qm;
2525
int status = XX_SCANNER_RETCODE_IMPOSSIBLE;
2626
int is_constant = 0, j;
2727

@@ -37,7 +37,7 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
3737
token->value = estrndup(start, YYCURSOR - start);
3838
token->len = YYCURSOR - start;
3939
s->active_char += (YYCURSOR - start);
40-
q = YYCURSOR;
40+
qm = YYCURSOR;
4141
return 0;
4242
}
4343
@@ -47,7 +47,7 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
4747
token->value = estrndup(start, YYCURSOR - start);
4848
token->len = YYCURSOR - start;
4949
s->active_char += (YYCURSOR - start);
50-
q = YYCURSOR;
50+
qm = YYCURSOR;
5151
return 0;
5252
}
5353
@@ -479,43 +479,45 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
479479
480480
SCHAR = (['] ([\\][']|[\\].|[\001-\377]\[\\'])* [']);
481481
SCHAR {
482+
start++;
482483
token->opcode = XX_T_CHAR;
483-
token->value = estrndup(q, YYCURSOR - q - 1);
484-
token->len = YYCURSOR - q - 1;
484+
token->value = estrndup(start, YYCURSOR - start - 1);
485+
token->len = YYCURSOR - start - 1;
485486
s->active_char += (YYCURSOR - start);
486-
q = YYCURSOR;
487+
qm = YYCURSOR;
487488
return 0;
488489
}
489490
491+
// interned strings, allowing to instantiate strings
490492
ISTRING = ([~]["] ([\\]["]|[\\].|[\001-\377]\[\\"])* ["]);
491493
ISTRING {
494+
start++; // ~
495+
start++; // "
492496
token->opcode = XX_T_ISTRING;
493-
token->value = estrndup(q, YYCURSOR - q - 1);
494-
token->len = YYCURSOR - q - 1;
497+
token->value = estrndup(start, YYCURSOR - start - 1);
498+
token->len = YYCURSOR - start - 1;
495499
s->active_char += (YYCURSOR - start);
496-
q = YYCURSOR;
500+
qm = YYCURSOR;
497501
return 0;
498502
}
499503
500504
STRING = (["] ([\\]["]|[\\].|[\001-\377]\[\\"])* ["]);
501505
STRING {
506+
start++;
502507
token->opcode = XX_T_STRING;
503-
token->value = estrndup(q, YYCURSOR - q - 1);
504-
token->len = YYCURSOR - q - 1;
505-
s->active_char += (YYCURSOR - start);
506-
q = YYCURSOR;
508+
token->value = estrndup(start, YYCURSOR - start - 1);
509+
token->len = YYCURSOR - start - 1;
510+
s->active_char += (YYCURSOR - start + 1);
511+
qm = YYCURSOR;
507512
return 0;
508513
}
509514
510515
DCOMMENT = ("/**"([^*]+|[*]+[^/*])*[*]*"*/");
511516
DCOMMENT {
512-
// FIXME: https://github.com/skvadrik/re2c/issues/197
513-
if (q[0] == '/') {
514-
q++;
515-
}
517+
start++;
516518
token->opcode = XX_T_COMMENT;
517-
token->value = estrndup(q, YYCURSOR - q - 1);
518-
token->len = YYCURSOR - q - 1;
519+
token->value = estrndup(start, YYCURSOR - start - 1);
520+
token->len = YYCURSOR - start - 1;
519521
{
520522
int k, ch = s->active_char;
521523
for (k = 0; k < (token->len - 1); k++) {
@@ -528,15 +530,15 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
528530
}
529531
s->active_char = ch;
530532
}
531-
q = YYCURSOR;
533+
qm = YYCURSOR;
532534
return 0;
533535
}
534536
535537
COMMENT = ("/*"([^*]+|[*]+[^/*])*[*]*"*/");
536538
COMMENT {
537539
token->opcode = XX_T_IGNORE;
538-
token->value = estrndup(q, YYCURSOR - q - 1);
539-
token->len = YYCURSOR - q - 1;
540+
token->value = estrndup(start, YYCURSOR - start - 1);
541+
token->len = YYCURSOR - start - 1;
540542
{
541543
int k, ch = s->active_char;
542544
for (k = 0; k < (token->len - 1); k++) {
@@ -551,7 +553,7 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
551553
}
552554
efree(token->value);
553555
token->len = 0;
554-
q = YYCURSOR;
556+
qm = YYCURSOR;
555557
return 0;
556558
}
557559
@@ -565,8 +567,8 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
565567
CBLOCK = ("%{"([^}]+|[}]+[^%{])*"}%");
566568
CBLOCK {
567569
token->opcode = XX_T_CBLOCK;
568-
token->value = estrndup(q+1, YYCURSOR - q - 3 );
569-
token->len = YYCURSOR - q - 3;
570+
token->value = estrndup(start+1, YYCURSOR - start - 3 );
571+
token->len = YYCURSOR - start - 3;
570572
{
571573
int k, ch = s->active_char;
572574
for (k = 0; k < (token->len - 1); k++) {
@@ -579,7 +581,7 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
579581
}
580582
s->active_char = ch;
581583
}
582-
q = YYCURSOR;
584+
qm = YYCURSOR;
583585
return 0;
584586
}
585587

@@ -596,7 +598,7 @@ int xx_get_token(xx_scanner_state *s, xx_scanner_token *token) {
596598
token->len = YYCURSOR - start;
597599
s->active_char += (YYCURSOR - start);
598600
}
599-
q = YYCURSOR;
601+
qm = YYCURSOR;
600602

601603
if (token->len > 3 && token->value[0] == '_') {
602604

‎tests/base/types.phpt

Lines changed: 133 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ array(2) {
3333
["final"]=>
3434
int(0)
3535
["definition"]=>
36-
array(4) {
36+
array(5) {
3737
["properties"]=>
38-
array(8) {
38+
array(9) {
3939
[0]=>
4040
array(7) {
4141
["visibility"]=>
@@ -240,7 +240,7 @@ array(2) {
240240
["value"]=>
241241
string(10) "909.999999"
242242
["file"]=>
243-
string(%s) "%s/tests/data/base/types.zep"
243+
string(%d) "%s/tests/data/base/types.zep"
244244
["line"]=>
245245
int(12)
246246
["char"]=>
@@ -280,9 +280,137 @@ array(2) {
280280
["file"]=>
281281
string(%d) "%s/tests/data/base/types.zep"
282282
["line"]=>
283-
int(14)
283+
int(15)
284+
["char"]=>
285+
int(7)
286+
}
287+
[8]=>
288+
array(7) {
289+
["visibility"]=>
290+
array(1) {
291+
[0]=>
292+
string(6) "public"
293+
}
294+
["type"]=>
295+
string(8) "property"
296+
["name"]=>
297+
string(7) "t_char1"
298+
["default"]=>
299+
array(5) {
300+
["type"]=>
301+
string(4) "char"
302+
["value"]=>
303+
string(1) "a"
304+
["file"]=>
305+
string(%d) "%s/tests/data/base/types.zep"
306+
["line"]=>
307+
int(15)
308+
["char"]=>
309+
int(21)
310+
}
311+
["file"]=>
312+
string(%d) "%s/tests/data/base/types.zep"
313+
["line"]=>
314+
int(17)
315+
["char"]=>
316+
int(7)
317+
}
318+
}
319+
["methods"]=>
320+
array(2) {
321+
[0]=>
322+
array(8) {
323+
["visibility"]=>
324+
array(1) {
325+
[0]=>
326+
string(6) "public"
327+
}
328+
["type"]=>
329+
string(6) "method"
330+
["name"]=>
331+
string(10) "someString"
332+
["statements"]=>
333+
array(1) {
334+
[0]=>
335+
array(5) {
336+
["type"]=>
337+
string(6) "return"
338+
["expr"]=>
339+
array(5) {
340+
["type"]=>
341+
string(6) "string"
342+
["value"]=>
343+
string(5) "hello"
344+
["file"]=>
345+
string(%d) "%s/tests/data/base/types.zep"
346+
["line"]=>
347+
int(19)
348+
["char"]=>
349+
int(17)
350+
}
351+
["file"]=>
352+
string(%d) "%s/tests/data/base/types.zep"
353+
["line"]=>
354+
int(20)
355+
["char"]=>
356+
int(2)
357+
}
358+
}
359+
["file"]=>
360+
string(%d) "%s/tests/data/base/types.zep"
361+
["line"]=>
362+
int(17)
363+
["last-line"]=>
364+
int(22)
365+
["char"]=>
366+
int(16)
367+
}
368+
[1]=>
369+
array(8) {
370+
["visibility"]=>
371+
array(1) {
372+
[0]=>
373+
string(6) "public"
374+
}
375+
["type"]=>
376+
string(6) "method"
377+
["name"]=>
378+
string(11) "someIString"
379+
["statements"]=>
380+
array(1) {
381+
[0]=>
382+
array(5) {
383+
["type"]=>
384+
string(6) "return"
385+
["expr"]=>
386+
array(5) {
387+
["type"]=>
388+
string(7) "istring"
389+
["value"]=>
390+
string(5) "hello"
391+
["file"]=>
392+
string(%d) "%s/tests/data/base/types.zep"
393+
["line"]=>
394+
int(24)
395+
["char"]=>
396+
int(16)
397+
}
398+
["file"]=>
399+
string(%d) "%s/tests/data/base/types.zep"
400+
["line"]=>
401+
int(25)
402+
["char"]=>
403+
int(2)
404+
}
405+
}
406+
["file"]=>
407+
string(%d) "%s/tests/data/base/types.zep"
408+
["line"]=>
409+
int(22)
410+
["last-line"]=>
411+
int(26)
284412
["char"]=>
285-
int(1)
413+
int(16)
286414
}
287415
}
288416
["file"]=>

‎tests/data/base/types.zep

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,16 @@ class Test
1111
public t_double1 = -0.000001;
1212
public t_double1 = 909.999999;
1313
public t_double1 = -909.999999;
14+
15+
public t_char1 = 'a';
16+
17+
public function someString()
18+
{
19+
return "hello";
20+
}
21+
22+
public function someIString()
23+
{
24+
return ~"hello";
25+
}
1426
}

0 commit comments

Comments
 (0)