Skip to content

Commit 3232b2e

Browse files
gridaphobegridaphobe
committed
normalize fortran variables like gfortran
1 parent 5cb6b08 commit 3232b2e

File tree

13 files changed

+112
-39
lines changed

13 files changed

+112
-39
lines changed

‎src/org/opensolaris/opengrok/analysis/Ctags.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.nio.charset.StandardCharsets;
3535
import java.util.ArrayList;
3636
import java.util.List;
37+
import java.util.function.Function;
3738
import java.util.logging.Level;
3839
import java.util.logging.Logger;
3940
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
@@ -61,6 +62,8 @@ public class Ctags implements Resettable {
6162
private String CTagsExtraOptionsFile = null;
6263
private int tabSize;
6364

65+
private Function<String, String> normalizeIdentifier = str -> str;
66+
6467
private boolean junit_testing = false;
6568

6669
/**
@@ -89,6 +92,10 @@ public void setTabSize(int tabSize) {
8992
this.tabSize = tabSize;
9093
}
9194

95+
public void setNormalizeIdentifier(Function<String, String> normalizeIdentifier) {
96+
this.normalizeIdentifier = normalizeIdentifier;
97+
}
98+
9299
public void setCTagsExtraOptionsFile(String CTagsExtraOptionsFile) {
93100
this.CTagsExtraOptionsFile = CTagsExtraOptionsFile;
94101
}
@@ -380,6 +387,7 @@ public Definitions doCtags(String file) throws IOException,
380387
CtagsReader rdr = new CtagsReader();
381388
rdr.setSplitterSupplier(() -> { return trySplitSource(file); });
382389
rdr.setTabSize(tabSize);
390+
rdr.setNormalizeIdentifier(normalizeIdentifier);
383391
Definitions ret;
384392
try {
385393
ctagsIn.write(file + "\n");
@@ -443,6 +451,7 @@ public void destroy() {
443451

444452
CtagsReader rdr = new CtagsReader();
445453
rdr.setTabSize(tabSize);
454+
rdr.setNormalizeIdentifier(normalizeIdentifier);
446455
try {
447456
readTags(rdr);
448457
} catch (InterruptedException ex) {

‎src/org/opensolaris/opengrok/analysis/CtagsReader.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
package org.opensolaris.opengrok.analysis;
2626

2727
import java.util.EnumMap;
28+
import java.util.function.Function;
2829
import java.util.function.Supplier;
2930
import java.util.logging.Level;
3031
import java.util.logging.Logger;
@@ -93,6 +94,12 @@ public class CtagsReader {
9394

9495
private int tabSize;
9596

97+
private Function<String, String> normalizeIdentifier = str -> str;
98+
99+
public void setNormalizeIdentifier(Function<String, String> normalize) {
100+
normalizeIdentifier = normalize;
101+
}
102+
96103
/**
97104
* This should mimic
98105
* https://github.com/universal-ctags/ctags/blob/master/docs/format.rst or
@@ -209,7 +216,7 @@ public void readLine(String tagLine) {
209216
//log.fine("SKIPPING LINE - NO TAB");
210217
return;
211218
}
212-
String def = tagLine.substring(0, p);
219+
String def = normalizeIdentifier.apply(tagLine.substring(0, p));
213220
int mstart = tagLine.indexOf('\t', p + 1);
214221

215222
String kind = null;
@@ -326,7 +333,7 @@ public void readLine(String tagLine) {
326333
name = arg;
327334
}
328335
if (name != null) {
329-
addTag(defs, cidx.lineno, name, "argument", def.trim() +
336+
addTag(defs, cidx.lineno, normalizeIdentifier.apply(name), "argument", def.trim() +
330337
signature.trim(), null, signature, cidx.lineStart,
331338
cidx.lineEnd);
332339
} else {

‎src/org/opensolaris/opengrok/analysis/JFlexNonXref.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ protected boolean writeSymbol(String symbol, Set<String> keywords, int line,
473473
protected boolean writeSymbol(String symbol, Set<String> keywords, int line,
474474
boolean caseSensitive, boolean isKeyword) throws IOException {
475475
return JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
476-
symbol, keywords, line, caseSensitive, isKeyword);
476+
symbol, symbol, keywords, line, caseSensitive, isKeyword);
477477
}
478478

479479
/**

‎src/org/opensolaris/opengrok/analysis/JFlexSymbolMatcher.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ public abstract class JFlexSymbolMatcher extends JFlexStateStacker
3838
private NonSymbolMatchedListener nonSymbolListener;
3939
private String disjointSpanClassName;
4040

41+
protected String normalizeIdentifier(String id) { return id; }
42+
4143
/**
4244
* Associates the specified listener, replacing the former one.
4345
* @param l defined instance
@@ -97,7 +99,7 @@ protected String getDisjointSpanClassName() {
9799
protected void onSymbolMatched(String str, int start) {
98100
SymbolMatchedListener l = symbolListener;
99101
if (l != null) {
100-
SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, start,
102+
SymbolMatchedEvent evt = new SymbolMatchedEvent(this, str, normalizeIdentifier(str), start,
101103
start + str.length());
102104
l.symbolMatched(evt);
103105
}

‎src/org/opensolaris/opengrok/analysis/JFlexXref.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ public void setFoldingEnabled(boolean foldingEnabled) {
203203
public void symbolMatched(SymbolMatchedEvent evt) {
204204
try {
205205
JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
206-
evt.getStr(), null, matcher.getLineNumber(), false, false);
206+
evt.getStr(), evt.getNormalizedStr(), null, matcher.getLineNumber(), false, false);
207207
} catch (IOException ex) {
208208
throw new RuntimeException(ex);
209209
}
@@ -292,7 +292,7 @@ public void linkageMatched(LinkageMatchedEvent evt) {
292292
break;
293293
case LABELDEF:
294294
// Only PowerShell seems to be using this.
295-
JFlexXrefUtils.writeSameFileLinkSymbol(out, str);
295+
JFlexXrefUtils.writeSameFileLinkSymbol(out, str, str);
296296
break;
297297
case FILELIKE:
298298
out.write("<a href=\"");
@@ -585,7 +585,7 @@ public void startNewLine() throws IOException {
585585
*/
586586
protected void writeKeyword(String symbol, int line) throws IOException {
587587
JFlexXrefUtils.writeSymbol(out, defs, urlPrefix, project,
588-
symbol, null, line, false, true);
588+
symbol, symbol, null, line, false, true);
589589
}
590590

591591
/**

‎src/org/opensolaris/opengrok/analysis/JFlexXrefUtils.java

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,7 @@ public static void writeEMailAddress(Writer out, String address)
229229
* @param urlPrefix a defined instance
230230
* @param project a possibly defined instance or null
231231
* @param symbol the symbol to write
232+
* @param id the symbol to write, normalized according to language-specific conventions
232233
* @param keywords a set of keywords recognized by this analyzer (no links
233234
* will be generated if the symbol is a keyword)
234235
* @param line the line number on which the symbol appears
@@ -240,7 +241,7 @@ public static void writeEMailAddress(Writer out, String address)
240241
* @throws IOException if an error occurs while writing to the stream
241242
*/
242243
public static boolean writeSymbol(Writer out, Definitions defs,
243-
String urlPrefix, Project project, String symbol, Set<String> keywords,
244+
String urlPrefix, Project project, String symbol, String id, Set<String> keywords,
244245
int line, boolean caseSensitive, boolean isKeyword)
245246
throws IOException {
246247
String[] strs = new String[1];
@@ -255,7 +256,7 @@ public static boolean writeSymbol(Writer out, Definitions defs,
255256
return false;
256257
}
257258

258-
if (defs != null && defs.hasDefinitionAt(symbol, line, strs)) {
259+
if (defs != null && defs.hasDefinitionAt(id, line, strs)) {
259260
// This is the definition of the symbol.
260261
String type = strs[0];
261262
String style_class = "d";
@@ -281,15 +282,15 @@ public static boolean writeSymbol(Writer out, Definitions defs,
281282
out.append("<a class=\"");
282283
out.append(style_class);
283284
out.append("\" name=\"");
284-
Util.htmlize(symbol, out);
285+
Util.htmlize(id, out);
285286
out.append("\"/>");
286287
}
287288

288289
// 2) Create a link that searches for all references to this symbol.
289290
out.append("<a href=\"");
290291
out.append(urlPrefix);
291292
out.append("refs=");
292-
Util.qurlencode(symbol, out);
293+
Util.qurlencode(id, out);
293294
appendProject(out, project);
294295
out.append("\" class=\"");
295296
out.append(style_class);
@@ -298,8 +299,8 @@ public static boolean writeSymbol(Writer out, Definitions defs,
298299
out.append(">");
299300
Util.htmlize(symbol, out);
300301
out.append("</a>");
301-
} else if (defs != null && defs.occurrences(symbol) == 1) {
302-
writeSameFileLinkSymbol(out, symbol);
302+
} else if (defs != null && defs.occurrences(id) == 1) {
303+
writeSameFileLinkSymbol(out, symbol, id);
303304
} else {
304305
// This is a symbol that is not defined in this file, or a symbol
305306
// that is defined more than once in this file. In either case, we
@@ -308,7 +309,7 @@ public static boolean writeSymbol(Writer out, Definitions defs,
308309
out.append("<a href=\"");
309310
out.append(urlPrefix);
310311
out.append("defs=");
311-
Util.qurlencode(symbol, out);
312+
Util.qurlencode(id, out);
312313
appendProject(out, project);
313314
out.append("\"");
314315
out.append(" class=\"intelliWindow-symbol\"");
@@ -325,10 +326,11 @@ public static boolean writeSymbol(Writer out, Definitions defs,
325326
* exactly one location in the same file.
326327
* @param out a defined, target instance
327328
* @param symbol the symbol to write
329+
* @param id the symbol to write, normalized according to language-specific conventions
328330
* @throws IOException if {@link Writer#append(java.lang.CharSequence)}
329331
* fails
330332
*/
331-
public static void writeSameFileLinkSymbol(Writer out, String symbol)
333+
public static void writeSameFileLinkSymbol(Writer out, String symbol, String id)
332334
throws IOException {
333335
// This is a reference to a symbol defined exactly once in this file.
334336
String style_class = "d";
@@ -337,7 +339,7 @@ public static void writeSameFileLinkSymbol(Writer out, String symbol)
337339
out.append("<a class=\"");
338340
out.append(style_class);
339341
out.append(" intelliWindow-symbol\" href=\"#");
340-
Util.URIEncode(symbol, out);
342+
Util.URIEncode(id, out);
341343
out.append("\"");
342344
out.append(" data-definition-place=\"defined-in-file\"");
343345
out.append(">");

‎src/org/opensolaris/opengrok/analysis/SymbolMatchedEvent.java

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,19 +36,22 @@ public class SymbolMatchedEvent {
3636

3737
private final Object source;
3838
private final String str;
39+
private final String normalizedStr;
3940
private final int start;
4041
private final int end;
4142

4243
/**
4344
* Initializes an immutable instance of {@link SymbolMatchedEvent}.
4445
* @param source the event source
4546
* @param str the symbol string
47+
* @param normalizedStr the symbol string, normalized according to language-specific conventions
4648
* @param start the symbol start position
4749
* @param end the symbol end position
4850
*/
49-
public SymbolMatchedEvent(Object source, String str, int start, int end) {
51+
public SymbolMatchedEvent(Object source, String str, String normalizedStr, int start, int end) {
5052
this.source = source;
5153
this.str = str;
54+
this.normalizedStr = normalizedStr;
5255
this.start = start;
5356
this.end = end;
5457
}
@@ -69,6 +72,14 @@ public String getStr() {
6972
return str;
7073
}
7174

75+
/**
76+
* Gets the normalized symbol string.
77+
* @return the initial value
78+
*/
79+
public String getNormalizedStr() {
80+
return normalizedStr;
81+
}
82+
7283
/**
7384
* Gets the symbol start position.
7485
* @return the initial value

‎src/org/opensolaris/opengrok/analysis/fortran/FortranAnalyzer.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
package org.opensolaris.opengrok.analysis.fortran;
2525

2626
import java.io.Reader;
27+
import org.opensolaris.opengrok.analysis.Ctags;
2728
import org.opensolaris.opengrok.analysis.FileAnalyzer;
2829
import org.opensolaris.opengrok.analysis.JFlexTokenizer;
2930
import org.opensolaris.opengrok.analysis.JFlexXref;
@@ -36,6 +37,10 @@
3637
*/
3738
public class FortranAnalyzer extends AbstractSourceCodeAnalyzer {
3839

40+
public static String normalizeIdentifier(String id) {
41+
return id.toLowerCase() + "_";
42+
}
43+
3944
FortranAnalyzer(FortranAnalyzerFactory factory) {
4045
super(factory, new JFlexTokenizer(new FortranSymbolTokenizer(
4146
FileAnalyzer.dummyReader)));
@@ -50,4 +55,12 @@ public class FortranAnalyzer extends AbstractSourceCodeAnalyzer {
5055
protected JFlexXref newXref(Reader reader) {
5156
return new JFlexXref(new FortranXref(reader));
5257
}
58+
59+
@Override
60+
public void setCtags(Ctags ctags) {
61+
this.ctags = ctags;
62+
if (this.ctags != null) {
63+
this.ctags.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
64+
}
65+
}
5366
}

‎src/org/opensolaris/opengrok/analysis/fortran/FortranXref.lex

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ import org.opensolaris.opengrok.web.HtmlConsts;
6060
break;
6161
}
6262
}
63+
64+
@Override
65+
protected String normalizeIdentifier(String id) { return FortranAnalyzer.normalizeIdentifier(id); }
6366
%}
6467

6568
File = [a-zA-Z]{FNameChar}* ".inc"

‎src/org/opensolaris/opengrok/search/context/Context.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,20 @@
3737
import java.util.List;
3838
import java.util.Map;
3939
import java.util.TreeMap;
40+
import java.util.function.Function;
4041
import java.util.logging.Level;
4142
import java.util.logging.Logger;
4243

4344
import org.apache.lucene.document.Document;
4445
import org.apache.lucene.index.IndexableField;
4546
import org.apache.lucene.search.IndexSearcher;
4647
import org.apache.lucene.search.Query;
48+
import org.opensolaris.opengrok.analysis.AnalyzerGuru;
4749
import org.opensolaris.opengrok.analysis.Definitions;
4850
import org.opensolaris.opengrok.analysis.FileAnalyzer;
51+
import org.opensolaris.opengrok.analysis.FileAnalyzerFactory;
52+
import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzer;
53+
import org.opensolaris.opengrok.analysis.fortran.FortranAnalyzerFactory;
4954
import org.opensolaris.opengrok.analysis.Scopes;
5055
import org.opensolaris.opengrok.analysis.Scopes.Scope;
5156
import org.opensolaris.opengrok.analysis.plain.PlainAnalyzerFactory;
@@ -416,6 +421,10 @@ public boolean getContext(Reader in, Writer out, String urlPrefix,
416421
String token;
417422
int matchState;
418423
int matchedLines = 0;
424+
FileAnalyzerFactory factory = AnalyzerGuru.find(path);
425+
if (factory instanceof FortranAnalyzerFactory) {
426+
tokens.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
427+
}
419428
while ((token = tokens.yylex()) != null && (!lim ||
420429
matchedLines < limit_max_lines)) {
421430
for (int i = 0; i < m.length; i++) {
@@ -455,6 +464,7 @@ public boolean getContext(Reader in, Writer out, String urlPrefix,
455464
}
456465
}
457466
}
467+
tokens.resetNormalizeIdentifier();
458468
return anything;
459469
}
460470
}

‎src/org/opensolaris/opengrok/search/context/PlainLineTokenizer.lex

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import java.io.Reader;
3232
import java.io.Writer;
3333
import java.util.List;
3434
import java.util.TreeMap;
35+
import java.util.function.Function;
3536
import org.opensolaris.opengrok.search.Hit;
3637
import org.opensolaris.opengrok.web.Util;
3738
import org.opensolaris.opengrok.analysis.Scopes;
@@ -72,6 +73,14 @@ import org.opensolaris.opengrok.analysis.Scopes.Scope;
7273
boolean alt;
7374
Scopes scopes = null;
7475

76+
Function<String, String> normalizeIdentifier = str -> str;
77+
public void setNormalizeIdentifier(Function<String, String> normalizeIdentifier) {
78+
this.normalizeIdentifier = normalizeIdentifier;
79+
}
80+
public void resetNormalizeIdentifier() {
81+
this.normalizeIdentifier = str -> str;
82+
}
83+
7584
/**
7685
* Set the writer that should receive all output
7786
* @param out The new writer to write to
@@ -400,7 +409,13 @@ Printable = [\@\$\%\^\&\-+=\?\.\:]
400409

401410

402411
%%
403-
{Identifier}|{Number}|{Printable} {
412+
{Identifier} {
413+
String text = yytext();
414+
markedContents.append(text);
415+
return normalizeIdentifier.apply(text);
416+
}
417+
418+
{Number}|{Printable} {
404419
String text = yytext();
405420
markedContents.append(text);
406421
return text;

‎test/org/opensolaris/opengrok/analysis/fortran/FortranXrefTest.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ private Definitions getTagsDefinitions() throws IOException {
116116
res, "UTF-8"));
117117

118118
CtagsReader rdr = new CtagsReader();
119+
rdr.setNormalizeIdentifier(FortranAnalyzer::normalizeIdentifier);
119120
String line;
120121
while ((line = in.readLine()) != null) {
121122
rdr.readLine(line);

0 commit comments

Comments
 (0)