@@ -331,7 +331,7 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
331
331
332
332
/* token-split the line with NULL characters; double NULL is eof */
333
333
334
- /* STATE MACHINE TOKENIZE */
334
+ /* Tokenizer state machine */
335
335
char * cur = linebuf .d .ch ;
336
336
/* space char gives room for termination checks */
337
337
if (llen > 0 )
@@ -381,15 +381,15 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
381
381
int nstrescps = 0 ;
382
382
int nstrleads = 0 ;
383
383
for (; cur [0 ]; cur ++ ) {
384
- /* STATE MACHINE GOES HERE */
384
+ /* main tokenizing loop */
385
385
if (cur [0 ] == '\t' ) {
386
386
cur [0 ] = ' ' ;
387
387
}
388
388
int inside_string = proctok == TOK_STRING || proctok == TOK_TRISTR ;
389
389
if (!inside_string && cur [0 ] == ' ' && cur [1 ] == ' ' ) {
390
390
continue ;
391
391
}
392
- /* SO: single space is a token boundary ... */
392
+ /* single space is a token boundary ... */
393
393
char nxt = * cur ;
394
394
int ignore = 0 ;
395
395
int tokfin = 0 ;
@@ -647,22 +647,28 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
647
647
648
648
/* Line wrapping & printing, oh joy */
649
649
char * tokpos = tokbuf .d .ch ;
650
+ char * ntokpos = tokpos ;
650
651
int nests = 0 ;
651
652
int pptok = TOK_INBETWEEN ;
652
653
int pretok = TOK_INBETWEEN ;
653
654
int postok = toks .d .in [0 ];
654
655
for (int i = 0 ; i < ntoks ; i ++ ) {
656
+ ntokpos += strlen (ntokpos ) + 1 ;
657
+ while (toks .d .in [i + 1 ] == TOK_LCONT && i < ntoks ) {
658
+ ntokpos += strlen (ntokpos ) + 1 ;
659
+ i ++ ;
660
+ }
661
+
655
662
pptok = pretok ;
656
663
pretok = postok ;
657
664
postok = toks .d .in [i + 1 ];
658
- int toklen = strlen ( tokpos );
665
+
659
666
if (pretok == TOK_OBRACE ) {
660
667
nests ++ ;
661
668
}
662
669
663
- if (pretok == TOK_LCONT ) {
664
- /* ignore line breaks */
665
- } else if (pretok == TOK_COMMENT ) {
670
+ if (pretok == TOK_COMMENT ) {
671
+ int toklen = strlen (tokpos );
666
672
char * eos = tokpos + toklen - 1 ;
667
673
char * sos = tokpos ;
668
674
while (* sos == ' ' ) {
@@ -678,13 +684,9 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
678
684
buildpt += strapp (buildpt , "# " );
679
685
}
680
686
buildpt += strapp (buildpt , sos );
681
- splitpoints .d .in [nsplits ] = buildpt - laccum .d .ch ;
682
687
split_ratings .d .in [nsplits ] = SSCORE_COMMENT ;
683
- split_nestings .d .in [nsplits ] = nests ;
684
- nsplits ++ ;
685
688
} else {
686
689
buildpt += strapp (buildpt , tokpos );
687
- splitpoints .d .in [nsplits ] = buildpt - laccum .d .ch ;
688
690
if (pretok == TOK_COMMA && postok != TOK_CBRACE && nests > 0 ) {
689
691
split_ratings .d .in [nsplits ] = 1 ;
690
692
} else if (pretok == TOK_COLON && postok != TOK_CBRACE ) {
@@ -696,11 +698,11 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
696
698
} else {
697
699
split_ratings .d .in [nsplits ] = 0 ;
698
700
}
699
-
700
- split_nestings .d .in [nsplits ] = nests ;
701
- nsplits ++ ;
702
701
}
703
- tokpos += toklen + 1 ;
702
+ splitpoints .d .in [nsplits ] = buildpt - laccum .d .ch ;
703
+ split_nestings .d .in [nsplits ] = nests ;
704
+ nsplits ++ ;
705
+ tokpos = ntokpos ;
704
706
705
707
int space ;
706
708
if (pretok == TOK_COMMENT ) {
@@ -709,8 +711,6 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
709
711
postok == TOK_LABEL ) {
710
712
/* annotation */
711
713
space = 0 ;
712
- } else if (pretok == TOK_LCONT ) {
713
- space = 0 ;
714
714
} else if (pretok == TOK_EQUAL || postok == TOK_EQUAL ) {
715
715
space = (nests == 0 );
716
716
} else if (pretok == TOK_SPECIAL ) {
@@ -795,7 +795,6 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
795
795
for (int rleft = length_left , k = i ; k < nsplits && rleft >= 0 ; k ++ ) {
796
796
/* Estimate segment length, walk further */
797
797
int fr = k > 0 ? splitpoints .d .in [k - 1 ] : 0 ;
798
- // int ofr = k > 1 ? splitpoints.d.in[k - 2] : 0;
799
798
int to = k >= nsplits - 1 ? eoff : splitpoints .d .in [k ];
800
799
int seglen = to - fr ;
801
800
rleft -= seglen ;
0 commit comments