Skip to content

Commit 0acb065

Browse files
author
_
committed
strip line continuation tokens before line split ratings
1 parent b3c2aff commit 0acb065

File tree

1 file changed

+17
-18
lines changed

1 file changed

+17
-18
lines changed

pfa/pfa.c

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
331331

332332
/* token-split the line with NULL characters; double NULL is eof */
333333

334-
/* STATE MACHINE TOKENIZE */
334+
/* Tokenizer state machine */
335335
char *cur = linebuf.d.ch;
336336
/* space char gives room for termination checks */
337337
if (llen > 0)
@@ -381,15 +381,15 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
381381
int nstrescps = 0;
382382
int nstrleads = 0;
383383
for (; cur[0]; cur++) {
384-
/* STATE MACHINE GOES HERE */
384+
/* main tokenizing loop */
385385
if (cur[0] == '\t') {
386386
cur[0] = ' ';
387387
}
388388
int inside_string = proctok == TOK_STRING || proctok == TOK_TRISTR;
389389
if (!inside_string && cur[0] == ' ' && cur[1] == ' ') {
390390
continue;
391391
}
392-
/* SO: single space is a token boundary ... */
392+
/* single space is a token boundary ... */
393393
char nxt = *cur;
394394
int ignore = 0;
395395
int tokfin = 0;
@@ -647,22 +647,28 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
647647

648648
/* Line wrapping & printing, oh joy */
649649
char *tokpos = tokbuf.d.ch;
650+
char *ntokpos = tokpos;
650651
int nests = 0;
651652
int pptok = TOK_INBETWEEN;
652653
int pretok = TOK_INBETWEEN;
653654
int postok = toks.d.in[0];
654655
for (int i = 0; i < ntoks; i++) {
656+
ntokpos += strlen(ntokpos) + 1;
657+
while (toks.d.in[i + 1] == TOK_LCONT && i < ntoks) {
658+
ntokpos += strlen(ntokpos) + 1;
659+
i++;
660+
}
661+
655662
pptok = pretok;
656663
pretok = postok;
657664
postok = toks.d.in[i + 1];
658-
int toklen = strlen(tokpos);
665+
659666
if (pretok == TOK_OBRACE) {
660667
nests++;
661668
}
662669

663-
if (pretok == TOK_LCONT) {
664-
/* ignore line breaks */
665-
} else if (pretok == TOK_COMMENT) {
670+
if (pretok == TOK_COMMENT) {
671+
int toklen = strlen(tokpos);
666672
char *eos = tokpos + toklen - 1;
667673
char *sos = tokpos;
668674
while (*sos == ' ') {
@@ -678,13 +684,9 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
678684
buildpt += strapp(buildpt, "# ");
679685
}
680686
buildpt += strapp(buildpt, sos);
681-
splitpoints.d.in[nsplits] = buildpt - laccum.d.ch;
682687
split_ratings.d.in[nsplits] = SSCORE_COMMENT;
683-
split_nestings.d.in[nsplits] = nests;
684-
nsplits++;
685688
} else {
686689
buildpt += strapp(buildpt, tokpos);
687-
splitpoints.d.in[nsplits] = buildpt - laccum.d.ch;
688690
if (pretok == TOK_COMMA && postok != TOK_CBRACE && nests > 0) {
689691
split_ratings.d.in[nsplits] = 1;
690692
} else if (pretok == TOK_COLON && postok != TOK_CBRACE) {
@@ -696,11 +698,11 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
696698
} else {
697699
split_ratings.d.in[nsplits] = 0;
698700
}
699-
700-
split_nestings.d.in[nsplits] = nests;
701-
nsplits++;
702701
}
703-
tokpos += toklen + 1;
702+
splitpoints.d.in[nsplits] = buildpt - laccum.d.ch;
703+
split_nestings.d.in[nsplits] = nests;
704+
nsplits++;
705+
tokpos = ntokpos;
704706

705707
int space;
706708
if (pretok == TOK_COMMENT) {
@@ -709,8 +711,6 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
709711
postok == TOK_LABEL) {
710712
/* annotation */
711713
space = 0;
712-
} else if (pretok == TOK_LCONT) {
713-
space = 0;
714714
} else if (pretok == TOK_EQUAL || postok == TOK_EQUAL) {
715715
space = (nests == 0);
716716
} else if (pretok == TOK_SPECIAL) {
@@ -795,7 +795,6 @@ static void pyformat(FILE *file, FILE *out, struct vlbuf *origfile,
795795
for (int rleft = length_left, k = i; k < nsplits && rleft >= 0; k++) {
796796
/* Estimate segment length, walk further */
797797
int fr = k > 0 ? splitpoints.d.in[k - 1] : 0;
798-
// int ofr = k > 1 ? splitpoints.d.in[k - 2] : 0;
799798
int to = k >= nsplits - 1 ? eoff : splitpoints.d.in[k];
800799
int seglen = to - fr;
801800
rleft -= seglen;

0 commit comments

Comments
 (0)