-
Notifications
You must be signed in to change notification settings - Fork 0
Tokenizers
michal-kapala edited this page Nov 30, 2023
·
1 revision
Script content tokenizers decompiled from Jitterbit Studio (10.55).
This library performs static evaluation of Jitterbit and JS scripts, Transform.Token tokens are later parsed into expressions.
jitterbit-studio-core-1.0.0-SNAPSHOT\org\jitterbit\integration\data\script\Transform.java
package org.jitterbit.integration.data.script;
import java.util.StringTokenizer;
import java.util.ArrayList;
import org.jitterbit.integration.structure.mapping.PathSeparator;
import org.jitterbit.util.string.Case;
import java.util.Collection;
import javax.annotation.Nullable;
import java.util.Stack;
import com.google.common.collect.UnmodifiableIterator;
import java.util.Iterator;
import com.google.common.collect.Maps;
import com.google.common.base.Preconditions;
import java.util.HashMap;
import com.google.common.collect.Lists;
import java.util.Map;
import java.util.List;
import org.jitterbit.util.string.KongaStringTokenizer;
public class Transform
{
private static final boolean BEFORE = true;
private static final boolean AFTER = false;
private static final String TERMINATOR = "+-*/^=&|\"'\\<>!(),;{}[] \n\r\t";
private KongaStringTokenizer m_tk;
public ErrObject m_err;
private String m_savedTokenStr;
private String m_expr;
private Token currentToken;
private int m_pos;
private int m_nData;
private Token m_savedToken;
private Level m_levelObj;
private final List<Token> m_stackVector;
private final List<Token> m_tokenVector;
private int m_lastId;
private SourceDataElementContainer sourceDataElementContainer;
private final Map<String, Integer> lde_status_map;
public static final int t_Unknown = 400;
public static final int t_DE = 405;
public static final int t_IntC = 402;
public static final int t_FloatC = 403;
public static final int t_StrC = 401;
public static final int t_GlobalDE = 406;
public static final int t_LocalDE = 408;
public static final int t_Set = 320;
public static final int t_PSet = 321;
public static final int t_MSet = 322;
public static final int t_Add = 700;
public static final int t_Sub = 701;
public static final int t_Mul = 800;
public static final int t_Div = 801;
public static final int t_Not = 901;
public static final int t_NE = 600;
public static final int t_EQ = 601;
public static final int t_GT = 602;
public static final int t_GE = 603;
public static final int t_LT = 604;
public static final int t_LE = 605;
public static final int t_LP = 100;
public static final int t_RP = 1000;
public static final int t_LBrace = 0;
public static final int t_RBrace = 1003;
public static final int t_LBracket = 1004;
public static final int t_RBracket = 1005;
public static final int t_StartArray = 1006;
public static final int t_EndArray = 1007;
public static final int t_Negative = 1001;
public static final int t_PostPP = 1010;
public static final int t_PostMM = 1011;
public static final int t_PrePP = 1012;
public static final int t_PreMM = 1013;
public static final int t_Pow = 900;
public static final int t_And = 500;
public static final int t_Or = 407;
public static final int t_Fcn = 101;
public static final int t_Comma = 200;
public static final int t_BoolC = 404;
public static final int t_Marker = 1002;
public static final int t_Semicolon = 310;
public static final int t_Space = 1100;
public static final int t_Comment = 1101;
public static final int t_Trans = -1;
public static final int t_End = -2;
private Map<String, FCN> m_fcnList;
public Transform() {
this.m_stackVector = Lists.newArrayList();
this.m_tokenVector = Lists.newArrayList();
this.sourceDataElementContainer = SourceDataElementContainer.EMPTY;
this.lde_status_map = new HashMap<String, Integer>();
this.createFcnList();
}
public void setSourceDataElementContainer(final SourceDataElementContainer sourceDEs) {
this.sourceDataElementContainer = (SourceDataElementContainer)Preconditions.checkNotNull((Object)sourceDEs);
}
private void createFcnList() {
final FunctionRegistry regestry = FunctionRegistry.getRegistry();
final Iterable<FunctionCategory> cats = (Iterable<FunctionCategory>)regestry.getCategories();
this.m_fcnList = Maps.newHashMapWithExpectedSize(200);
for (final FunctionCategory cat : cats) {
for (final Function f : cat.getFunctions()) {
final String key = f.getName() + "(";
final FCN fcn = f.getGrammar();
this.m_fcnList.put(key.toUpperCase(), fcn);
}
}
}
public static int find_END_TRANS_position(final String expr, final int start) {
final int n = expr.length() - ScriptConstants.BEGIN_TRANS_LENGTH;
if (start >= n) {
return -1;
}
boolean lastCharIsSlash = false;
for (int i = start; i < n; ++i) {
final char c = expr.charAt(i);
if (c == '\"' || c == '\'') {
++i;
while (i < n) {
final char c2 = expr.charAt(i);
if (c2 == '\\') {
++i;
}
else if (c2 == c) {
break;
}
++i;
}
}
else if (lastCharIsSlash) {
lastCharIsSlash = false;
if (c == '/') {
++i;
while (i < n) {
final char c2 = expr.charAt(i);
if (c2 == '\n') {
break;
}
if (c2 == '\r') {
break;
}
++i;
}
}
else if (c == '*') {
++i;
while (i < n) {
final char c2 = expr.charAt(i);
if (c2 == '/' && expr.charAt(i - 1) == '*') {
break;
}
++i;
}
}
}
else if (c == '/') {
lastCharIsSlash = true;
}
else if (c == '<' && expr.substring(i).startsWith("</trans>")) {
return i;
}
}
return -1;
}
private boolean isFunctionTakesNodeArgument(final Token t) {
if (t == null) {
return false;
}
final String str = t.getStr().toLowerCase();
return str.startsWith("getxmlstring(") || str.startsWith("getsourceinstancemap") || str.startsWith("getsourceinstancearray") || str.startsWith("getsourceattrnames") || str.startsWith("getsourceelementnames") || str.startsWith("getsourceinstanceelementmap") || str.startsWith("getsourceinstanceelementarray");
}
private static Token getLastLastToken(final List<Token> list) {
Token t = null;
int i = 2;
for (int n = list.size() - 2; n >= 0; --n) {
t = list.get(n);
if (t.isNoOp()) {
t = null;
}
else if (--i == 0) {
return t;
}
}
return null;
}
@Nullable
public List<Token> tokenize(final String expr) {
this.reset();
this.m_expr = expr;
int start;
int end;
for (start = 0, end = 0; (start = expr.indexOf("<trans>", start)) >= 0; end = (start = end + ScriptConstants.END_TRANS_LENGTH)) {
if (start > end) {
this.m_tokenVector.add(new Token(401, expr.substring(end, start), this, false));
}
start += ScriptConstants.BEGIN_TRANS_LENGTH;
end = find_END_TRANS_position(expr, start);
if (end < 0) {
this.m_err = new ErrObject(this.m_expr, (Token)null, "The expression:" + expr + "\nMissing closing tag " + "</trans>", false);
return null;
}
this.m_tokenVector.add(new Token(-1, "<trans>", this, false));
final String str0 = expr.substring(start, end);
this.m_pos = start;
this.m_err = null;
this.m_savedTokenStr = null;
this.m_savedToken = null;
this.m_tk = new KongaStringTokenizer(str0, "+-*/^=&|\"'\\<>!(),;{}[] \n\r\t", true);
Token last = null;
Token tokenOfBracket = null;
boolean isAtStart = true;
boolean hasValidLHS = false;
int withinArgumentList = 0;
final Stack<Bracket> bracket_pair = new Stack<Bracket>();
Token t;
while ((t = this.nextToken(isAtStart, hasValidLHS)) != null) {
if (this.m_err != null) {
return null;
}
if (t.isArgumentList() || withinArgumentList > 0) {
this.m_tokenVector.add(t);
Label_0506: {
switch (withinArgumentList) {
case 0: {
withinArgumentList = 1;
break;
}
case 1: {
switch (t.m_id) {
case 1100:
case 1101: {
continue;
}
case 408: {
this.lde_status_map.put(t.m_str.toUpperCase(), 1);
withinArgumentList = 2;
break Label_0506;
}
case 405: {
this.m_err = new ErrObject(this.m_expr, t, "ArgumentList argument name is invalid, it is already used as source DE.", false);
return null;
}
default: {
this.m_err = new ErrObject(this.m_expr, t, "ArgumentList argument must be a local variable.", false);
return null;
}
}
break;
}
case 2: {
switch (t.m_id) {
case 1100:
case 1101: {
continue;
}
case 200: {
withinArgumentList = 1;
break Label_0506;
}
case 1000: {
withinArgumentList = 0;
break Label_0506;
}
default: {
this.m_err = new ErrObject(this.m_expr, t, "ArgumentList argument must be a local variable.", false);
return null;
}
}
break;
}
}
}
last = t;
}
else {
if (t.m_id == 1012) {
if (last != null && last.isLHSDataElement()) {
t.m_id = 1010;
}
}
else if (t.m_id == 1013 && last != null && last.isLHSDataElement()) {
t.m_id = 1011;
}
if (t.m_id == 1004 && this.isFunctionTakesNodeArgument(last)) {
tokenOfBracket = t;
}
else if (t.m_id == 1005 && last != null && last.m_id == 1004) {
final int i = this.m_tokenVector.size();
this.m_tokenVector.remove(i - 1);
t.m_id = 1006;
t.m_str = "[]";
t.m_pos1 = last.m_pos1;
tokenOfBracket = null;
this.m_tokenVector.add(t);
continue;
}
if (tokenOfBracket != null && tokenOfBracket.m_id == 1004) {
if (t.m_id == 405) {
tokenOfBracket = t;
}
else if (t.m_id == 1005) {
final int i = this.m_tokenVector.size();
this.m_tokenVector.remove(i - 1);
t.m_id = 405;
t.m_str = "[]";
t.m_pos1 = tokenOfBracket.m_pos1;
tokenOfBracket = null;
hasValidLHS = false;
}
}
else if (tokenOfBracket != null && tokenOfBracket.m_id == 405 && t.m_id == 1005) {
final int i = this.m_tokenVector.size();
final Token left = this.m_tokenVector.get(i - 2);
this.m_tokenVector.remove(i - 1);
this.m_tokenVector.remove(i - 2);
t.m_id = 405;
t.m_str = "[" + tokenOfBracket.m_str + "]";
t.m_pos1 = left.m_pos1;
tokenOfBracket = null;
hasValidLHS = false;
}
else {
tokenOfBracket = null;
}
this.m_tokenVector.add(t);
final int id = t.getId();
Label_1386: {
switch (id) {
case -2: {
break Label_1386;
}
case 400: {
this.m_err = new ErrObject(this.m_expr, t, "Unknown token " + t.m_str, false);
return null;
}
case 1004: {
bracket_pair.add(new Bracket(hasValidLHS));
hasValidLHS = false;
isAtStart = false;
break Label_1386;
}
case 1005: {
if (bracket_pair.size() > 0) {
hasValidLHS = bracket_pair.pop().isValidLHS();
isAtStart = false;
break Label_1386;
}
this.m_err = new ErrObject(this.m_expr, t, "The expression:" + expr + "\nextra closing bracket ']'", false);
return null;
}
case 1100:
case 1101: {
continue;
}
case 1000: {
if (last != null && last.m_id == 100) {
this.m_err = new ErrObject(this.m_expr, t, "Nothing between parentheses", true);
break;
}
break;
}
}
hasValidLHS = (id == 406 || id == 320 || id == 408);
isAtStart = (id == 320 || id == 321 || id == 322 || id == 100 || id == 101 || id == 500 || id == 407 || id == 600 || id == 601 || id == 602 || id == 603 || id == 604 || id == 605 || id == 0 || id == 200 || id == 310);
}
final boolean invalidPreOp = last != null && last.isPreOp() && !t.isLHSDataElement();
final Token last_last;
final boolean invalidPostOp = t.isPostOp() && last != null && last.isLHSDataElement() && (last_last = getLastLastToken(this.m_tokenVector)) != null && last_last.isPreOp();
if (invalidPreOp || invalidPostOp) {
this.m_err = new ErrObject(this.m_expr, last, "Invalid argument to operator ++/--.", false);
return null;
}
if (id == -2) {
break;
}
last = t;
}
}
if (this.m_err != null) {
return null;
}
}
if (this.m_err != null) {
return null;
}
if (start < 0 && end < expr.length()) {
this.m_tokenVector.add(new Token(401, expr.substring(end), this, false));
}
return this.m_tokenVector;
}
private void reset() {
this.m_err = null;
this.m_lastId = -1;
this.currentToken = null;
this.m_savedToken = null;
this.m_tokenVector.clear();
this.lde_status_map.clear();
}
public List<Token> getRpnTokenVector(final String expr) {
if (this.tokenize(expr) == null) {
return null;
}
this.m_stackVector.clear();
return this.rpnOrdering(this.m_tokenVector, this.m_stackVector, false);
}
@Nullable
public List<Token> rpnOrder() {
this.m_stackVector.clear();
return this.rpnOrdering(this.m_tokenVector, this.m_stackVector, false);
}
@Nullable
private List<Token> rpnOrdering(final List<Token> tokenList, final List<Token> rpnTokenList, final boolean initialized) {
rpnTokenList.clear();
for (int i = 0, n = tokenList.size(); i < n; ++i) {
Token t = tokenList.get(i);
if (initialized || t.m_id == -1) {
if (t.m_id == -1) {
++i;
}
this.m_err = null;
final Stack<Token> opStack = new Stack<Token>();
this.m_nData = 0;
this.m_levelObj = new Level(null, 0, this.m_nData);
final Stack<Level> nCommaStack = new Stack<Level>();
this.m_lastId = -1;
boolean no_tokens = true;
boolean currentTokenIsOperand = false;
boolean currentTokenIsOperator = false;
boolean lastTokenIsOperand = false;
boolean lastTokenIsOperator = false;
Token lastOperandToken = null;
Token lastOperatorToken = null;
while (i < n) {
t = tokenList.get(i);
final int id = t.getId();
if (id == -2) {
break;
}
if (id == 400) {
return null;
}
if (id != 1100) {
if (id != 1101) {
currentTokenIsOperand = false;
currentTokenIsOperator = false;
final int priority = t.getPriority();
no_tokens = false;
final String name;
boolean found_set_operator;
int k;
int check;
switch (id) {
case 408:
Label_0702: {
name = t.m_str.toUpperCase();
if (this.isLocalVariableDefined(name) || this.isArgument(name)) {
break Label_0702;
}
found_set_operator = false;
k = i + 1;
while (k < n) {
check = tokenList.get(k).m_id;
if (check != 1100 && check != 1101) {
if (check == 320) {
this.lde_status_map.put(name, 1);
found_set_operator = true;
break;
}
break;
}
else {
++k;
}
}
if (!found_set_operator) {
this.m_err = new ErrObject(this.m_expr, t, "Local variable " + name + " hasn't been initialized", true);
return null;
}
break Label_0702;
}
case 401:
case 402:
case 403:
case 404:
case 405:
case 406:
case 1006: {
currentTokenIsOperand = true;
if (lastTokenIsOperand && lastOperandToken != null) {
this.m_err = new ErrObject(this.m_expr, t, "Missing operator between two operands: " + lastOperandToken.m_str + " and " + t.m_str, true);
return null;
}
rpnTokenList.add(t);
this.m_nData += t.numberOfDataToBeAdd();
if (opStack.size() > 0) {
final Token op = opStack.peek();
switch (op.m_id) {
case 1012:
case 1013: {
if (id != 406 && id != 408) {
this.m_err = new ErrObject(this.m_expr, t, "The operand for " + op.m_str + " operator must be local or global data element", true);
return null;
}
if (id == 408 && !this.isLocalVariableDefined(t.m_str.toUpperCase())) {
this.m_err = new ErrObject(this.m_expr, t, "Local variable " + t.m_str + " hasn't bean initialized", true);
return null;
}
}
case 901:
case 1001: {
rpnTokenList.add(op);
this.m_nData += op.numberOfDataToBeAdd();
opStack.pop();
break;
}
}
}
lastOperandToken = t;
break;
}
case 320:
case 321:
case 322:
case 407:
case 500:
case 600:
case 601:
case 602:
case 603:
case 604:
case 605:
case 700:
case 701:
case 800:
case 801:
case 900: {
if (lastTokenIsOperator) {
if (lastOperatorToken == null) {
throw new RuntimeException("lastOperatorToken should have been set");
}
this.m_err = new ErrObject(this.m_expr, t, "Operator: " + t.m_str + " cannot proceeded by another operator: " + lastOperatorToken.m_str, true);
return null;
}
else {
currentTokenIsOperator = true;
lastOperatorToken = t;
if (this.m_nData <= 0) {
this.m_err = new ErrObject(this.m_expr, t, "Not enough operand for the operation: \"" + t.m_str + "\"", true);
return null;
}
while (opStack.size() > 0) {
final Token op = opStack.peek();
if (op.getPriority() < priority) {
break;
}
final int opid = op.getId();
if (opid == 200) {
continue;
}
rpnTokenList.add(op);
this.m_nData += op.numberOfDataToBeAdd();
if (this.m_nData <= this.m_levelObj.get_nData_expected()) {
this.m_err = new ErrObject(this.m_expr, t, "Not enough operand for the operation: \"" + op.m_str + "\"", true);
return null;
}
opStack.pop();
}
opStack.push(t);
break;
}
break;
}
case 200: {
Token op = null;
while (opStack.size() > 0) {
op = opStack.peek();
if (op.getPriority() <= priority) {
break;
}
rpnTokenList.add(op);
this.m_nData += op.numberOfDataToBeAdd();
if (this.m_nData < this.m_levelObj.get_nData_expected()) {
this.m_err = new ErrObject(this.m_expr, t, "Not enough operands for the operation: \"" + op.m_str + "\"", true);
return null;
}
opStack.pop();
}
final String msg = this.m_levelObj.checkSemicolon(t, rpnTokenList);
if (msg.length() > 0) {
this.m_err = new ErrObject(this.m_expr, t, msg, true);
return null;
}
this.m_levelObj.add_comma();
rpnTokenList.add(t);
break;
}
case 310: {
while (opStack.size() > 0) {
final Token op = opStack.peek();
if (!op.isSetOp() && op.getPriority() <= priority) {
break;
}
rpnTokenList.add(op);
final int nData_required = 1 - op.numberOfDataToBeAdd();
if (this.m_nData < nData_required) {
this.m_err = new ErrObject(this.m_expr, t, "Not enough operand for the operation: \"" + t.m_str + "\"", true);
return null;
}
if (op.m_id == 320) {
int j;
for (j = 0; j < i && tokenList.get(j) != op; ++j) {}
--j;
while (j >= 0) {
final Token check2 = tokenList.get(j);
final int check_id = check2.m_id;
if (check_id != 1100 && check_id != 1101) {
if (check_id == 408) {
final String variable_name = check2.m_str.toUpperCase();
final Integer status = this.lde_status_map.get(variable_name);
if (status != null && status == 0) {
this.lde_status_map.put(variable_name, 1);
}
break;
}
break;
}
else {
--j;
}
}
}
this.m_nData += op.numberOfDataToBeAdd();
opStack.pop();
}
rpnTokenList.add(t);
--this.m_nData;
break;
}
case 0:
case 101: {
if (!lastTokenIsOperand) {
rpnTokenList.add(new Token(1002, "[", null, false));
}
if (lastOperandToken == null) {
throw new RuntimeException("lastOperandToken should have been set");
}
this.m_err = new ErrObject(this.m_expr, t, "Missing operator between two operands: " + lastOperandToken.m_str + " and " + t.m_str, true);
return null;
}
case 100: {
if (!lastTokenIsOperand) {
opStack.push(t);
nCommaStack.push(this.m_levelObj);
this.m_levelObj = new Level(t, 0, this.m_nData);
break;
}
if (lastOperandToken == null) {
throw new RuntimeException("lastOperandToken should have been set");
}
this.m_err = new ErrObject(this.m_expr, t, "Missing operator between two operands: " + lastOperandToken.m_str + " and " + t.m_str, true);
return null;
}
case 1001:
case 1012:
case 1013: {
opStack.push(t);
break;
}
case 1010:
case 1011: {
rpnTokenList.add(t);
break;
}
case 901: {
if (!lastTokenIsOperand) {
opStack.push(t);
break;
}
if (lastOperandToken == null) {
throw new RuntimeException("lastOperandToken should have been defined");
}
this.m_err = new ErrObject(this.m_expr, t, "Operator ! cannot proceeded with an operand: " + lastOperandToken.m_str, true);
return null;
}
case 1003: {
int opid = -1;
Token op = null;
while (opStack.size() > 0) {
op = opStack.peek();
opStack.pop();
opid = op.getId();
if (opid == 0) {
break;
}
if (opid == 200) {
continue;
}
rpnTokenList.add(op);
this.m_nData += op.numberOfDataToBeAdd();
}
final String msg = this.m_levelObj.checkSemicolon(t, rpnTokenList);
if (msg.length() > 0) {
this.m_err = new ErrObject(this.m_expr, t, msg, true);
return null;
}
if (opid == 0) {
rpnTokenList.add(op);
}
if (op != null) {
this.m_nData += op.numberOfDataToBeAdd();
}
if (nCommaStack.size() == 0) {
this.m_err = new ErrObject(this.m_expr, t, "Extra right BRACE '}' in the expression.", true);
return null;
}
this.m_levelObj = nCommaStack.pop();
break;
}
case 1000: {
if (lastTokenIsOperator) {
if (lastOperatorToken == null) {
throw new RuntimeException("lastOperatorToken should have been set");
}
this.m_err = new ErrObject(this.m_expr, t, "Missing operand between " + lastOperatorToken.m_str + " and " + t.m_str, true);
return null;
}
else {
currentTokenIsOperand = true;
lastOperandToken = t;
int opid = -1;
Token op = null;
while (opStack.size() > 0) {
op = opStack.peek();
opStack.pop();
opid = op.getId();
if (opid == 100) {
break;
}
if (opid == 101) {
break;
}
if (opid == 200) {
continue;
}
rpnTokenList.add(op);
this.m_nData += op.numberOfDataToBeAdd();
}
final String msg = this.m_levelObj.checkSemicolon(t, rpnTokenList);
if (msg.length() > 0) {
this.m_err = new ErrObject(this.m_expr, t, msg, true);
return null;
}
if (opid == 101) {
if (op == null) {
throw new RuntimeException("op token should have been set at this point");
}
rpnTokenList.add(op);
final int argc = 1 - op.numberOfDataToBeAdd();
final FCN fcn = this.getFcn(op.m_str);
if (fcn.checkFcnArguments(argc) == FCN.ArgumentCheck.fail) {
this.m_err = new ErrObject(this.m_expr, t, "Wrong number of arguments in " + op.m_str + "): number of arguments = " + argc + ", " + fcn.msg(), true);
return null;
}
}
if (op != null) {
this.m_nData += op.numberOfDataToBeAdd();
}
if (nCommaStack.size() == 0) {
this.m_err = new ErrObject(this.m_expr, t, "Extra right parenthesis ')' in the expression", true);
return null;
}
this.m_levelObj = nCommaStack.pop();
break;
}
break;
}
case 1004: {
final List<Token> subTokenList = Lists.newArrayList();
final List<Token> subRpnTokenList = Lists.newArrayList();
final int i2 = i + 1;
int nBrace = 0;
Token sub_token = null;
int i3;
for (i3 = i2; i3 < n; ++i3) {
sub_token = tokenList.get(i3);
final int sub_token_id = sub_token.m_id;
switch (sub_token_id) {
case 1005: {
--nBrace;
break;
}
case 1004: {
++nBrace;
break;
}
}
if (nBrace < 0) {
break;
}
if (sub_token_id != 1100 && sub_token_id != 1101) {
subTokenList.add(sub_token);
}
}
if (nBrace >= 0) {
this.m_err = new ErrObject(this.m_expr, subTokenList.isEmpty() ? t : sub_token, "The closing bracket is missing.", false);
return null;
}
final List<Token> endArray_list = Lists.newArrayList();
if (this.m_lastId == 1004 && !rpnTokenList.isEmpty()) {
final int size = rpnTokenList.size();
for (int l = size - 1; l > 0; --l) {
final Token t2 = rpnTokenList.get(l);
if (t2.m_id != 1007) {
break;
}
rpnTokenList.remove(l);
endArray_list.add(t2);
}
}
if (subTokenList.isEmpty()) {
t.m_id = 1006;
rpnTokenList.add(t);
++this.m_nData;
}
else {
nCommaStack.push(this.m_levelObj);
final int nData_start = this.m_nData;
if (this.rpnOrdering(subTokenList, subRpnTokenList, true) == null) {
return null;
}
final Token last = subRpnTokenList.isEmpty() ? null : subRpnTokenList.get(subRpnTokenList.size() - 1);
this.m_nData += nData_start;
if (this.m_nData == nData_start && last != null && last.m_id == 310) {
subRpnTokenList.remove(subRpnTokenList.size() - 1);
++this.m_nData;
}
this.m_levelObj = nCommaStack.pop();
if (this.m_nData != nData_start + 1) {
this.m_err = new ErrObject(this.m_expr, (last == null) ? t : last, "Syntax error inside []. The expression could not be evaluate to a single value.", false);
return null;
}
}
rpnTokenList.addAll(subRpnTokenList);
final Token t3 = tokenList.get(i = i3);
if (subRpnTokenList.isEmpty()) {
t3.m_id = 1007;
}
rpnTokenList.add(t3);
rpnTokenList.addAll(endArray_list);
--this.m_nData;
currentTokenIsOperand = true;
lastOperandToken = t;
break;
}
}
this.m_lastId = id;
}
}
++i;
lastTokenIsOperand = currentTokenIsOperand;
lastTokenIsOperator = currentTokenIsOperator;
}
if (this.m_levelObj.get_nComma() != 0) {
this.m_err = new ErrObject(this.m_expr, t, "Extra comma in the expression.", true);
return null;
}
while (opStack.size() > 0) {
final Token op = opStack.peek();
if (op.m_id == 200) {
this.m_err = new ErrObject(this.m_expr, op, "Misplaced comma in the expression.", true);
return null;
}
if (op.m_id == 101) {
this.m_err = new ErrObject(this.m_expr, op, "Function does not have closing parenthesis.", true);
return null;
}
if (op.m_id == 100) {
this.m_err = new ErrObject(this.m_expr, op, "Unbalanced parenthesis.", true);
return null;
}
rpnTokenList.add(op);
this.m_nData += op.numberOfDataToBeAdd();
if (this.m_nData <= 0) {
this.m_err = new ErrObject(this.m_expr, op, "Not enough operand for the operation: \"" + op.m_str + "\"", true);
return null;
}
opStack.pop();
}
if (!no_tokens) {
if (this.m_nData == 0) {
final int size2 = rpnTokenList.size();
if (size2 > 0 && rpnTokenList.get(size2 - 1).m_id == 310) {
rpnTokenList.remove(size2 - 1);
this.m_nData = 1;
if (size2 > 1) {
return rpnTokenList;
}
}
}
if (this.m_nData < 1) {
this.m_err = new ErrObject(this.m_expr, t, "This expression returns no value.", true);
return null;
}
if (this.m_nData > 1) {
this.m_err = new ErrObject(this.m_expr, t, "This expression returns more than one value. Maybe some operators are missing.", true);
return null;
}
}
}
}
return rpnTokenList;
}
private boolean isLocalVariableDefined(final String NAME) {
final Integer status = this.lde_status_map.get(NAME);
return status != null && status > 0;
}
private boolean isArgument(final String name) {
assert name != null;
if (name.length() < 2) {
return false;
}
if (!name.startsWith("_")) {
return false;
}
for (int i = 1, n = name.length(); i < n; ++i) {
final char c = name.charAt(i);
if (!Character.isDigit(c)) {
return false;
}
}
return true;
}
private boolean isLocalVariable(final String str) {
if (str == null || str.isEmpty()) {
return false;
}
char c = str.charAt(0);
if (c != '_' && !Character.isLetter(c)) {
return false;
}
for (int i = 1, n = str.length(); i < n; ++i) {
c = str.charAt(i);
if (c != '_' && !Character.isLetterOrDigit(c)) {
return false;
}
}
return true;
}
public void initTokenizer(final String expr, final int start, final int end) {
this.reset();
this.m_expr = expr;
this.m_pos = start;
final String str0 = expr.substring(start, end);
this.m_tk = new KongaStringTokenizer(str0, "+-*/^=&|\"'\\<>!(),;{}[] \n\r\t", true);
}
public void setLastId(final int id) {
this.m_lastId = id;
}
public Token nextToken(final boolean isAtStart, final boolean hasValidLHS) {
if (this.m_savedToken != null) {
this.currentToken = this.m_savedToken;
this.m_savedToken = null;
return this.currentToken;
}
int id = 400;
String str;
if (this.m_savedTokenStr != null) {
str = this.m_savedTokenStr;
this.m_savedTokenStr = null;
}
else {
if (!this.m_tk.hasMoreTokens()) {
return this.currentToken = new Token(-2, "</trans>", this, false);
}
str = this.m_tk.nextToken();
this.m_pos += str.length();
}
final char c;
switch (c = str.charAt(0)) {
case '+':
case '-': {
if (this.checkNextToken(str).equals(str)) {
str += str;
id = ((c == '+') ? 1012 : 1013);
break;
}
if (this.checkNextToken("=").equals("=")) {
str += "=";
id = ((c == '+') ? 321 : 322);
if (!hasValidLHS) {
this.m_err = new ErrObject(this.m_expr, this.currentToken, "The left hand side of the assignment operator '+=' or '-=' must be a local or global data element, such as x=... or $x=... error occured ", false);
return null;
}
break;
}
else {
if (isAtStart) {
final Token t = this.nextToken(false, false);
switch (t.m_id) {
case 402:
case 403: {
if (c == '+') {
str = t.m_str;
}
else {
str += t.m_str;
}
id = t.m_id;
break;
}
default: {
this.m_savedToken = t;
if (c == '+') {
id = 700;
break;
}
if (c == '-') {
id = 1001;
break;
}
break;
}
}
break;
}
id = ((c == '+') ? 700 : 701);
break;
}
break;
}
case '*': {
if (this.checkNextToken("/").equals("/")) {
str += '/';
break;
}
id = 800;
break;
}
case '/': {
if (this.checkNextToken("/").equals("/")) {
final StringBuilder sb = new StringBuilder(str);
sb.append('/');
this.m_savedToken = null;
while (this.m_tk.hasMoreTokens()) {
final String tmp = this.m_tk.nextToken();
this.m_pos += tmp.length();
sb.append(tmp);
if (tmp.equals("\n")) {
break;
}
}
str = sb.toString();
id = 1101;
break;
}
if (this.checkNextToken("*").equals("*")) {
final StringBuilder sb = new StringBuilder(str);
sb.append('*');
this.m_savedToken = null;
while (this.m_tk.hasMoreTokens()) {
final String tmp = this.m_tk.nextToken();
this.m_pos += tmp.length();
sb.append(tmp);
final int len = sb.length();
if (sb.charAt(len - 2) == '*' && sb.charAt(len - 1) == '/') {
break;
}
}
str = sb.toString();
id = 1101;
break;
}
id = 801;
break;
}
case '^': {
id = 900;
break;
}
case '&': {
str += this.checkNextToken("&");
id = 500;
break;
}
case '|': {
str += this.checkNextToken("|");
id = 407;
break;
}
case '\"':
case '\'': {
boolean bEscaped = false;
final StringBuilder sb2 = new StringBuilder(str);
while (this.m_tk.hasMoreTokens()) {
final String tmp2 = this.m_tk.nextToken();
this.m_pos += tmp2.length();
sb2.append(tmp2);
final char c2 = tmp2.charAt(0);
if (c2 == c && !bEscaped) {
id = 401;
break;
}
if (c2 != '\\' && !bEscaped) {
continue;
}
bEscaped = !bEscaped;
}
str = sb2.toString();
if (id == 400) {
this.m_err = new ErrObject(this.m_expr, this.currentToken, "missing closing quote.", false);
}
break;
}
case '=': {
str += this.checkNextToken("=");
id = (str.equals("==") ? 601 : 320);
if (id == 320 && !hasValidLHS) {
this.m_err = new ErrObject(this.m_expr, this.currentToken, "The left hand side of the assignment operator '=' must be a local or global data element, such as x=... or $x=... error occured ", false);
return null;
}
break;
}
case '!': {
str += this.checkNextToken("=");
id = (str.equals("!=") ? 600 : 901);
break;
}
case '<': {
str += this.checkNextToken("=");
id = (str.equals("<=") ? 605 : 604);
break;
}
case '>': {
str += this.checkNextToken("=");
id = (str.equals(">=") ? 603 : 602);
break;
}
case '(': {
id = 100;
break;
}
case ')': {
id = 1000;
break;
}
case '$': {
id = 406;
break;
}
case '\t':
case '\n':
case '\r':
case ' ': {
id = 1100;
break;
}
case ',': {
id = 200;
break;
}
case ';': {
id = 310;
break;
}
case '{': {
id = 0;
break;
}
case '}': {
id = 1003;
break;
}
case '[': {
id = 1004;
break;
}
case ']': {
id = 1005;
break;
}
default: {
String tmp3 = this.checkNextToken("(");
str += tmp3;
if (tmp3.equals("(")) {
if (this.getFcn(str) != null) {
id = 101;
break;
}
this.m_err = new ErrObject(this.m_expr, this.currentToken, "unrecognized token: " + str + " ", false);
id = 400;
break;
}
else {
if (isNum(str)) {
id = 402;
break;
}
if (isFloat(str)) {
id = 403;
break;
}
if (isTerminatedWithE(str)) {
tmp3 = this.checkNextToken("+");
if (tmp3.equals("")) {
tmp3 = this.checkNextToken("-");
}
str += tmp3;
if (tmp3.equals("")) {
break;
}
if (!this.m_tk.hasMoreTokens()) {
break;
}
tmp3 = this.m_tk.nextToken();
this.m_pos += tmp3.length();
str += tmp3;
if (!isNum(tmp3)) {
this.m_err = new ErrObject(this.m_expr, this.currentToken, "expecting scientific notation ", false);
id = 400;
break;
}
id = 403;
break;
}
else {
if (str.equalsIgnoreCase("TRUE") || str.equalsIgnoreCase("FALSE")) {
id = 404;
break;
}
if (str.contains("$$")) {
this.m_err = new ErrObject(this.m_expr, this.currentToken, "Duplicate '$': " + str, false);
id = 400;
break;
}
if (this.isSourceDE(str)) {
id = 405;
break;
}
if (this.isLocalVariable(str)) {
id = 408;
break;
}
break;
}
}
break;
}
}
return this.currentToken = new Token(id, str, this, isAtStart);
}
private boolean isSourceDE(final String name) {
if (this.sourceDataElementContainer.isFlat()) {
final String var_name = ScriptUtils.stripInstanceTag(name);
return this.sourceDataElementContainer.contains(var_name);
}
return name.indexOf(36) > 0 || name.indexOf(46) >= 0;
}
public static List<String> getDEVector(final List<Token> stackVector) {
return getDEVector(stackVector, 405);
}
public static List<String> getDEVector(final List<Token> stackVector, final int deKind) {
final List<String> deVector = Lists.newArrayList();
for (final Token t : stackVector) {
if (t.getId() == deKind) {
boolean found = false;
for (final String s : deVector) {
found = t.m_str.equals(s);
if (found) {
break;
}
}
if (found) {
continue;
}
deVector.add(t.m_str);
}
}
return deVector;
}
private String checkNextToken(final String testToken) {
String tmp;
if (this.m_savedTokenStr != null) {
tmp = this.m_savedTokenStr;
this.m_savedTokenStr = null;
}
else {
if (!this.m_tk.hasMoreTokens()) {
return "";
}
tmp = this.m_tk.nextToken();
this.m_pos += tmp.length();
}
if (tmp.equals(testToken)) {
return tmp;
}
this.m_savedTokenStr = tmp;
return "";
}
private FCN getFcn(final String str) {
return this.m_fcnList.get(str.toUpperCase());
}
private static boolean isNum(final String str) {
for (int n = str.length(), i = 0; i < n; ++i) {
if (!Character.isDigit(str.charAt(i))) {
return false;
}
}
return true;
}
private static boolean isFloat(final String str) {
final int n = str.length();
boolean isNumbered = false;
boolean isDotted = false;
boolean isELabeled = false;
boolean isENumbered = false;
for (int i = 0; i < n; ++i) {
final char c;
if (!Character.isDigit(c = str.charAt(i))) {
if (c == '.') {
if (isDotted || isELabeled) {
return false;
}
isDotted = true;
}
else {
if (c != 'E' && c != 'e' && c != 'd' && c != 'D') {
return false;
}
if (isELabeled) {
return false;
}
isELabeled = true;
}
}
else if (isELabeled) {
isENumbered = true;
}
else {
isNumbered = true;
}
}
return isNumbered && (!isELabeled || isENumbered);
}
private static boolean isTerminatedWithE(final String str) {
final int n = str.length();
if (n == 1) {
return false;
}
boolean isDotted = false;
for (int i = 0; i < n; ++i) {
final char c;
if (!Character.isDigit(c = str.charAt(i))) {
if (c == '.') {
if (isDotted) {
return false;
}
isDotted = true;
}
else {
if (c != 'E' && c != 'e' && c != 'd' && c != 'D') {
return false;
}
if (i < n - 1) {
return false;
}
}
}
}
return true;
}
public boolean hasCResolve() {
for (final Token token : this.m_tokenVector) {
if (token.m_id == 101 && this.hasCResolve(FunctionRegistry.getRegistry().getFunctionByName(token.m_str))) {
return true;
}
}
return false;
}
private boolean hasCResolve(final Function fcn) {
return fcn != null && fcn.getFormat() != null && fcn.getFormat().indexOf(35) >= 0;
}
private int getNumberOfTags(final Stack<InstanceTagging> fcnStack) {
int n = 0;
for (final InstanceTagging item : fcnStack) {
if (item.tagShouldBeAdded()) {
++n;
}
if (item.bReturnArray) {
--n;
}
}
return n;
}
public String addInstanceTag(final String expr, final String fixedGenerator) throws Exception {
this.m_expr = expr;
final int n = this.m_tokenVector.size();
final Stack<InstanceTagging> fcnStack = new Stack<InstanceTagging>();
for (int i = 0; i < n; ++i) {
final Token token = this.m_tokenVector.get(i);
switch (token.m_id) {
case 101: {
this.addInstanceTagToFunction(expr, fcnStack, token);
break;
}
case 405: {
this.addInstanceTagToDataElement(fixedGenerator, fcnStack, token);
break;
}
case 200: {
if (!fcnStack.isEmpty()) {
fcnStack.peek().nextArg();
break;
}
break;
}
case 100: {
fcnStack.push(new InstanceTagging());
break;
}
case 1000: {
fcnStack.pop();
break;
}
}
}
return this.m_expr;
}
private void addInstanceTagToDataElement(final String fixedGenerator, final Stack<InstanceTagging> fcnStack, final Token token) throws Exception {
final int numberOfInstanceFcns = this.getNumberOfTags(fcnStack);
if (numberOfInstanceFcns > 0) {
token.addInstanceTags(numberOfInstanceFcns);
}
else if (fixedGenerator != null) {
token.addInstanceTags(fixedGenerator);
}
}
private void addInstanceTagToFunction(final String expr, final Stack<InstanceTagging> fcnStack, final Token token) throws Exception {
final Function fcn = FunctionRegistry.getRegistry().getFunctionByName(token.m_str);
if (fcn == null) {
throw new Exception("The expression contains undefined functions " + expr);
}
fcnStack.push(new InstanceTagging(fcn));
}
public String rmInstanceTag(final String expr) {
this.m_expr = expr;
for (final Token token : this.m_stackVector) {
if (token.m_id == 405) {
if (token.m_str.indexOf(35) < 0) {
continue;
}
token.rmInstanceTag();
}
}
return this.m_expr;
}
public static String switchCaseForDbSourceDataElement(final String expr, final Case toCase) throws Exception {
final Transform tr = new Transform();
final List<Token> list = tr.tokenize(expr);
if (list == null) {
throw new Exception("Invalid script: " + expr);
}
for (final Token token : list) {
if (token.m_id == 405) {
token.m_str = toCase.convert(token.m_str);
}
}
final StringBuilder sb = new StringBuilder();
for (final Token token2 : list) {
sb.append(token2.m_str);
}
return sb.toString();
}
private static String changeOwner(String path, final String newOwner) throws Exception {
assert newOwner != null;
final boolean isNode = path.startsWith("[") && path.endsWith("]");
if (isNode) {
path = path.substring(1, path.length() - 1);
}
final StringTokenizer tk = PathSeparator.tokenize(path, true);
final List<String> list = new ArrayList<String>();
while (tk.hasMoreTokens()) {
list.add(tk.nextToken());
}
final int n = list.size() - (isNode ? 0 : 1);
final StringBuilder sb = new StringBuilder();
if (isNode) {
sb.append('[');
}
for (int i = 0; i < n; ++i) {
String str = list.get(i);
final char c = str.charAt(0);
if (PathSeparator.isSeparator(c)) {
sb.append(c);
}
else {
final int pos = str.indexOf(58);
if (pos >= 0) {
str = str.substring(pos + 1);
}
if (!newOwner.isEmpty()) {
str = newOwner + ":" + str;
}
sb.append(str);
}
}
if (n < list.size()) {
sb.append(list.get(n));
}
if (isNode) {
sb.append(']');
}
return sb.toString();
}
public static String switchCaseOwnerForDbDataElement(final String mapping_line, final Case toCase, final String owner_source, final String owner_target) throws Exception {
if (mapping_line == null) {
return null;
}
final int pos = mapping_line.indexOf(9);
if (pos < 0) {
throw new Exception("Can't find separater character in the input string.");
}
String target = mapping_line.substring(0, pos);
if (owner_target != null) {
target = toCase.convert(target);
target = changeOwner(target, owner_target);
}
String expr = mapping_line.substring(pos + 1);
if (owner_source != null) {
final boolean isNode = target.startsWith("[") && target.endsWith("]");
if (isNode) {
expr = toCase.convert(expr);
expr = changeOwner(expr, owner_source);
}
else {
final Transform tr = new Transform();
final List<Token> list = tr.tokenize(expr);
if (list == null) {
throw new Exception("Invalid script: " + expr);
}
for (final Token token : list) {
if (token.m_id == 405) {
token.m_str = toCase.convert(token.m_str);
token.m_str = changeOwner(token.m_str, owner_source);
}
}
final StringBuilder sb = new StringBuilder();
for (final Token token2 : list) {
sb.append(token2.m_str);
}
expr = sb.toString();
}
}
return target + "\t" + expr;
}
enum Token_status
{
lhs_token,
set_token,
rhs_token,
start_token;
}
static class Bracket
{
boolean m_bValidLHS;
Bracket(final boolean bValidLHS) {
this.m_bValidLHS = bValidLHS;
}
boolean isValidLHS() {
return this.m_bValidLHS;
}
}
public class Token
{
public String m_str;
public int m_id;
int m_pos1;
Token_status m_status;
public Token(final int id, final String strToken) {
this.m_pos1 = -1;
this.m_id = id;
this.m_str = strToken;
}
public Token(final int id, final String strToken, final Transform transform, final boolean start) {
this.m_pos1 = -1;
this.m_id = id;
this.m_str = strToken;
if (this.m_id != 1002) {
this.m_pos1 = Transform.this.m_expr.lastIndexOf(strToken, Transform.this.m_pos);
}
if (start) {
this.m_status = Token_status.start_token;
}
if (this.m_id == 320) {
for (int i = Transform.this.m_tokenVector.size() - 1; i >= 0; --i) {
final Token t = Transform.this.m_tokenVector.get(i);
if (t.m_status == Token_status.start_token) {
break;
}
if (t.m_id == 1005) {
t.m_status = Token_status.lhs_token;
}
}
}
}
public String source_location() {
if (this.m_str == null) {
return "";
}
final StringTokenizer tk = new StringTokenizer(Transform.this.m_expr.substring(0, this.m_pos1 + this.m_str.length()), "\n", true);
int line = 1;
String token = null;
while (tk.hasMoreTokens()) {
token = tk.nextToken();
if (token.charAt(0) == '\n') {
++line;
}
}
if (line == 1) {
return token;
}
return "at line " + line + "\n" + token + "\n";
}
public int getId() {
return this.m_id;
}
public int getPriority() {
return this.m_id / 100;
}
public String getStr() {
return this.m_str;
}
public int numberOfDataToBeAdd() {
switch (this.m_id) {
case 401:
case 402:
case 403:
case 404:
case 405:
case 406:
case 408:
case 1006: {
return 1;
}
case 320:
case 321:
case 322: {
return -1;
}
case 407:
case 500:
case 600:
case 601:
case 602:
case 603:
case 604:
case 605:
case 700:
case 701:
case 800:
case 801:
case 900: {
return -1;
}
case 901:
case 1001:
case 1010:
case 1011:
case 1012:
case 1013: {
return 0;
}
case 0: {
return 1 - (Transform.this.m_levelObj.get_nComma() + ((Transform.this.m_lastId != 101 && Transform.this.m_lastId != 0) ? 1 : 0));
}
case 101: {
final FCN fcn = Transform.this.getFcn(this.m_str);
if (!fcn.isFixedArgumentNumber()) {
return 1 - (Transform.this.m_levelObj.get_nComma() + ((Transform.this.m_lastId != 101) ? 1 : 0));
}
return 1 - fcn.getMinimumNumberOfArguments();
}
default: {
return 0;
}
}
}
private void insertChar(final int pos, final String s) {
Transform.this.m_expr = Transform.this.m_expr.substring(0, pos) + s + Transform.this.m_expr.substring(pos);
final int s_len = s.length();
this.m_str = Transform.this.m_expr.substring(this.m_pos1, this.m_pos1 + this.m_str.length() + s_len);
for (final Token token : Transform.this.m_stackVector) {
if (token.m_pos1 > pos) {
final Token token2 = token;
token2.m_pos1 += s_len;
}
}
}
public boolean isLHSDataElement() {
return this.m_id == 406 || this.m_id == 408;
}
boolean isPreOp() {
return this.m_id == 1013 || this.m_id == 1012;
}
boolean isPostOp() {
return this.m_id == 1011 || this.m_id == 1010;
}
boolean isNoOp() {
return this.m_id == 1100 || this.m_id == 1101;
}
boolean isSetOp() {
return this.m_id == 320 || this.m_id == 321 || this.m_id == 322;
}
boolean isArgumentList() {
return this.m_id == 101 && this.m_str.equalsIgnoreCase("ArgumentList(");
}
private void removeChar(final int pos) {
Transform.this.m_expr = Transform.this.m_expr.substring(0, pos) + Transform.this.m_expr.substring(pos + 1);
if (this.m_str.length() > 0) {
this.m_str = Transform.this.m_expr.substring(this.m_pos1, this.m_pos1 + this.m_str.length() - 1);
}
}
private void addInstanceTags(final int n0) throws Exception {
if (this.m_id != 405 || n0 == 0) {
return;
}
int n = n0;
final int pos1 = this.m_pos1;
final int pos2 = this.m_pos1 + this.m_str.length();
for (int pos3 = Transform.this.m_expr.lastIndexOf(".", pos2); pos3 > pos1; pos3 = Transform.this.m_expr.lastIndexOf(".", pos3 - 1)) {
for (int k = pos3 - 1; k >= pos1; --k) {
char c = Transform.this.m_expr.charAt(k);
switch (c) {
case '$':
case '.': {
if (n > 0) {
this.insertChar(pos3, "#");
k = -1;
--n;
break;
}
break;
}
case '#': {
if (k == pos3 - 1) {
if (n > 0) {
--n;
}
else {
this.removeChar(k);
}
k = -1;
break;
}
break;
}
default: {
if (k == pos3 - 1 && Character.isDigit(c)) {
--k;
while (k >= pos1) {
c = Transform.this.m_expr.charAt(k);
if (!Character.isDigit(c)) {
if (c != '#' && (c != '_' || k <= 0 || Transform.this.m_expr.charAt(k - 1) != '#')) {
break;
}
k = -1;
}
--k;
}
break;
}
break;
}
}
}
}
if (n > 0) {
throw new Exception("Add Instance Tag failed. The data element does not have enough layers of loop node to be resolved by instance functions. number of instance functions in the chain:" + n0 + ", the DE name:" + this.m_str);
}
}
private void addInstanceTags(final String fixedGenerator) {
if (this.m_id != 405 || fixedGenerator == null || fixedGenerator.isEmpty()) {
return;
}
int pos1 = this.m_pos1;
int pos2 = this.m_pos1;
for (int pos3 = this.m_pos1 + this.m_str.length(); pos2 < pos3; ++pos2) {
pos1 = pos2;
pos2 = Transform.this.m_expr.indexOf(".", pos1);
if (pos2 < 0) {
break;
}
if (!fixedGenerator.startsWith(Transform.this.m_expr.substring(this.m_pos1, pos2))) {
break;
}
}
while (pos2 >= 0 && pos2 < this.m_pos1 + this.m_str.length()) {
final int pos3 = Transform.this.m_expr.indexOf(35, pos1);
if (pos3 < 0 || pos3 > pos2) {
this.insertChar(pos2, "#");
++pos2;
}
pos2 = Transform.this.m_expr.indexOf(".", pos2 + 1);
}
}
private void rmInstanceTag() {
final int pos1 = this.m_pos1;
final int pos2 = this.m_pos1 + this.m_str.length();
if (Transform.this.m_expr.substring(pos1, pos2).equals(this.m_str)) {
this.m_str = ScriptUtils.stripInstanceTag(this.m_str);
Transform.this.m_expr = Transform.this.m_expr.substring(0, pos1) + this.m_str + Transform.this.m_expr.substring(pos2);
final int d = this.m_str.length() - (pos2 - pos1);
for (final Token token : Transform.this.m_stackVector) {
if (token.m_pos1 > pos1) {
final Token token2 = token;
token2.m_pos1 += d;
}
}
}
}
@Override
public String toString() {
if (this.m_id == 1101) {
return "t_Comment";
}
if (this.m_id == 1100) {
return "t_Space";
}
if (this.m_id == 1006) {
return "start";
}
if (this.m_id == 1007) {
return "t_end";
}
if (this.m_id == 1002) {
return "t_Marker";
}
if (this.m_id == 200) {
return "t_Comma";
}
if (this.m_id == 405) {
return this.m_str + "(DE)";
}
if (this.m_id == 408) {
return this.m_str + "(Var)";
}
if (this.m_id == 406) {
return this.m_str + "(GVar)";
}
if (this.m_status == Token_status.lhs_token) {
return this.m_str + "(LHS)";
}
return this.m_str;
}
}
private static class InstanceTagging
{
private String argTypeChars;
private int currentArgPos;
protected boolean bReturnArray;
public InstanceTagging() {
this.argTypeChars = "";
}
public InstanceTagging(final Function fcn) {
this.argTypeChars = "";
if (fcn == null) {
return;
}
final String sFormat = fcn.getFormat();
final int k = sFormat.lastIndexOf(59);
if (k > 0 && k + 1 < sFormat.length()) {
this.argTypeChars = sFormat.substring(k + 1);
}
this.bReturnArray = fcn.isReturnArray();
}
public void nextArg() {
++this.currentArgPos;
}
public boolean tagShouldBeAdded() {
return this.currentArgPos < this.argTypeChars.length() && this.argTypeChars.charAt(this.currentArgPos) == '#';
}
}
public static final class ErrObject
{
private final int lineNumber;
private final String line;
private final String err_msg;
private final Token token;
private final boolean beforeOrAfter;
private ErrObject(final String expr, final Token token, final String err_msg, final boolean beforeOrAfter) {
this.token = token;
this.err_msg = err_msg;
this.beforeOrAfter = beforeOrAfter;
if (token != null) {
final StringTokenizer tk = new StringTokenizer(expr.substring(0, token.m_pos1 + token.m_str.length()), "\n", true);
int nline = 1;
String t = null;
while (tk.hasMoreTokens()) {
t = tk.nextToken();
if (t.charAt(0) == '\n') {
++nline;
}
}
this.line = t;
this.lineNumber = nline;
}
else {
this.line = "";
this.lineNumber = 1;
}
}
public int getLineNumber() {
return this.lineNumber;
}
public String getErrorMessage() {
return this.err_msg;
}
public String getErrorLine() {
return this.line;
}
public Token getErrorToken() {
return this.token;
}
public boolean occursBeforeToken() {
return this.beforeOrAfter;
}
public String getFormattedErrorMessage() {
String msg = this.getErrorMessage();
if (this.getLineNumber() >= 0) {
msg = "Error on line " + this.getLineNumber() + ": " + msg;
}
return msg;
}
@Override
public String toString() {
return this.getFormattedErrorMessage();
}
}
private class Level
{
private int comma_n;
private final int data_stack_size_when_start;
private final Token token;
Level(final Token token, final int comma_n, final int data_stack_size) {
assert token.m_id == 0;
this.token = token;
this.comma_n = comma_n;
this.data_stack_size_when_start = data_stack_size;
}
void add_comma() {
++this.comma_n;
}
int get_nComma() {
return this.comma_n;
}
int get_nData_expected() {
return this.data_stack_size_when_start + this.comma_n;
}
String checkSemicolon(final Token close, final List<Token> rpnTokenList) {
final int end_id = close.m_id;
assert end_id == 1003;
if (this.token == null) {
return "Syntax error, misplaced operator \"" + close.getStr() + "\".";
}
final int start_id = this.token.m_id;
if (end_id == 200) {
if (start_id != 101 && start_id != 0) {
return "Syntax error, misplaced Comma operator.";
}
}
else if (end_id == 1003 != (start_id == 0)) {
return "Syntax error, misplaced brace operator.";
}
if (end_id == 1000 && start_id == 101) {
final int argc = Transform.this.m_levelObj.get_nComma() + ((Transform.this.m_lastId != 101) ? 1 : 0);
final int argc2 = 1 - this.token.numberOfDataToBeAdd();
if (argc != argc2) {
return "Wrong number of arguments for the function " + this.token.getStr();
}
if (Transform.this.m_lastId == 101) {
return "";
}
}
final int nRpnTokens = rpnTokenList.size();
final Token last_token = (nRpnTokens > 0) ? rpnTokenList.get(nRpnTokens - 1) : null;
int nData_expected = this.get_nData_expected();
if (last_token != null && last_token.m_id != 1002) {
++nData_expected;
}
if (Transform.this.m_nData == nData_expected) {
return "";
}
if (Transform.this.m_nData != nData_expected - 1) {
return "The expression for the argument created more than one data element.";
}
if (last_token != null && last_token.m_id == 310) {
rpnTokenList.remove(nRpnTokens - 1);
Transform.this.m_nData++;
return "";
}
return "No data created for the argument";
}
@Override
public String toString() {
return "NComma=" + this.comma_n + ", data_stack_size_when_start=" + this.data_stack_size_when_start;
}
}
}This library is likely responsible for syntax highlighting.
jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\Tokens.java
package org.jitterbit.integration.client.ui.script.partition;
import org.jitterbit.util.text.partition.WordToken;
import org.jitterbit.util.text.partition.RegExToken;
import org.jitterbit.util.text.partition.BooleanConstant;
import org.jitterbit.util.text.partition.StringLiteralToken;
import org.jitterbit.util.text.partition.BlockCommentToken;
import org.jitterbit.util.text.partition.SingleLineCommentToken;
import org.jitterbit.util.text.partition.OneOfToken;
import org.jitterbit.util.text.partition.Token;
public final class Tokens
{
public static final Token NO_CODE;
public static final Token BEGIN_TRANS;
public static final Token END_TRANS;
public static final Token SINGLE_LINE_COMMENT;
public static final Token BLOCK_COMMENT;
public static final Token STRING;
public static final Token RAISE_ERROR;
public static final Token FUNCTION_JS;
public static final Token FUNCTION_JB;
public static final Token OPERATOR;
public static final Token BOOLEAN;
public static final Token GROUPING;
public static final Token SOURCE_PATH;
public static final Token GLOBAL_DATA_ELEMENT;
public static final NonGlobalVariableToken PROJECT_VARIABLE;
public static final NonGlobalVariableToken LOCAL_VARIABLE;
public static final Token NUMERIC_CONSTANT;
public static final Token WORD_TOKEN;
private Tokens() {
}
static {
NO_CODE = (Token)new NoCodeToken();
BEGIN_TRANS = (Token)new OneOfToken(new String[] { "<trans>" }, false);
END_TRANS = (Token)new OneOfToken(new String[] { "</trans>" }, false);
SINGLE_LINE_COMMENT = (Token)new SingleLineCommentToken();
BLOCK_COMMENT = (Token)new BlockCommentToken();
STRING = (Token)new StringLiteralToken();
RAISE_ERROR = (Token)new RaiseErrorToken();
FUNCTION_JS = (Token)new FunctionToken(true);
FUNCTION_JB = (Token)new FunctionToken(false);
OPERATOR = (Token)new OperatorToken();
BOOLEAN = (Token)new BooleanConstant();
GROUPING = (Token)new GroupingToken();
SOURCE_PATH = (Token)new SourcePathToken();
GLOBAL_DATA_ELEMENT = (Token)new GlobalDataElementToken();
PROJECT_VARIABLE = new NonGlobalVariableToken();
LOCAL_VARIABLE = new NonGlobalVariableToken();
NUMERIC_CONSTANT = (Token)new RegExToken("[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?\\b");
WORD_TOKEN = (Token)new WordToken();
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\Token.java
package org.jitterbit.util.text.partition;
public interface Token
{
String lexeme();
int offset();
boolean matches(final String p0, final int p1);
boolean isWhitespace();
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\NonGlobalVariableToken.java
package org.jitterbit.integration.client.ui.script.partition;
import org.jitterbit.util.text.partition.Token;
public final class NonGlobalVariableToken implements Token
{
private int offset;
private String lexeme;
public void replace(final Token globalDe) {
this.offset = globalDe.offset();
this.lexeme = globalDe.lexeme();
}
public String lexeme() {
return this.lexeme;
}
public int offset() {
return this.offset;
}
public boolean matches(final String input, final int offset) {
throw new UnsupportedOperationException();
}
public boolean isWhitespace() {
return false;
}
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\NoCodeToken.java
package org.jitterbit.integration.client.ui.script.partition;
import org.jitterbit.util.text.partition.Token;
final class NoCodeToken implements Token
{
private String lexeme;
private int offset;
public String lexeme() {
return this.lexeme;
}
public boolean matches(final String input, final int offset) {
final int end = input.toLowerCase().indexOf("<trans>", offset);
if (end == -1 || end > offset) {
this.offset = offset;
this.lexeme = ((end >= 0) ? input.substring(offset, end) : input.substring(offset));
return true;
}
this.offset = -1;
this.lexeme = null;
return false;
}
public int offset() {
return this.offset;
}
public boolean isWhitespace() {
return false;
}
@Override
public String toString() {
return this.lexeme + " @" + this.offset;
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\OneOfToken.java
package org.jitterbit.util.text.partition;
import com.google.common.collect.UnmodifiableIterator;
import java.util.Collection;
import com.google.common.collect.ImmutableList;
public class OneOfToken implements Token
{
private final ImmutableList<String> values;
private final boolean caseSensitive;
private final boolean wholeWord;
private String lexeme;
private int offset;
public OneOfToken(final String[] values, final boolean caseSensitive) {
this(values, caseSensitive, false);
}
public OneOfToken(final String[] values, final boolean caseSensitive, final boolean wholeWord) {
this((Collection<String>)ImmutableList.copyOf((Object[])values), caseSensitive, wholeWord);
}
public OneOfToken(final Collection<String> values, final boolean caseSensitive) {
this(values, caseSensitive, false);
}
public OneOfToken(final Collection<String> values, final boolean caseSensitive, final boolean wholeWord) {
this.offset = -1;
this.values = (ImmutableList<String>)ImmutableList.copyOf((Collection)values);
this.caseSensitive = caseSensitive;
this.wholeWord = wholeWord;
}
@Override
public String lexeme() {
return this.lexeme;
}
@Override
public final int offset() {
return this.offset;
}
@Override
public final boolean isWhitespace() {
return false;
}
@Override
public final boolean matches(final String input, final int offset) {
for (final String value : this.values) {
if (input.length() - offset < value.length()) {
continue;
}
final String part = input.substring(offset, offset + value.length());
final boolean matches = this.caseSensitive ? part.equals(value) : part.equalsIgnoreCase(value);
if (!matches) {
continue;
}
if (this.wholeWord && !this.checkWordBoundary(input, offset, value)) {
continue;
}
this.lexeme = value;
this.offset = offset;
return true;
}
this.lexeme = null;
this.offset = -1;
return false;
}
private boolean checkWordBoundary(final String input, final int offset, final String value) {
final int end = offset + value.length();
if (end == input.length()) {
return true;
}
final char c = input.charAt(end);
return !Character.isJavaIdentifierStart(c) && !Character.isJavaIdentifierPart(c);
}
@Override
public final String toString() {
return this.lexeme() + " @" + this.offset();
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\SingleLineCommentToken.java
package org.jitterbit.util.text.partition;
public class SingleLineCommentToken extends StartEndBoundaryToken
{
public SingleLineCommentToken() {
super("//", "\n");
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\StartEndBoundaryToken.java
package org.jitterbit.util.text.partition;
public class StartEndBoundaryToken implements Token
{
private final String startDelim;
private final int startLength;
private final String endDelim;
private final int endLength;
private String lexeme;
private int offset;
public StartEndBoundaryToken(final String start, final String end) {
this.startDelim = start;
this.startLength = start.length();
this.endDelim = end;
this.endLength = end.length();
}
@Override
public String lexeme() {
return this.lexeme;
}
@Override
public int offset() {
return this.offset;
}
@Override
public boolean isWhitespace() {
return false;
}
@Override
public boolean matches(final String input, final int offset) {
if (input.length() - offset >= this.startLength && input.substring(offset, offset + this.startLength).equals(this.startDelim)) {
this.lexeme = this.extractLexeme(input, offset + this.startLength);
this.offset = offset;
return true;
}
this.lexeme = null;
this.offset = -1;
return false;
}
private String extractLexeme(final String input, final int start) {
final int end = input.indexOf(this.endDelim, start);
if (end == -1) {
return input.substring(start - this.startLength);
}
return input.substring(start - this.startLength, end + this.endLength);
}
@Override
public String toString() {
return this.lexeme() + " @" + this.offset();
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\BlockCommentToken.java
package org.jitterbit.util.text.partition;
public class BlockCommentToken extends StartEndBoundaryToken
{
public BlockCommentToken() {
super("/*", "*/");
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\StringLiteralToken.java
package org.jitterbit.util.text.partition;
public class StringLiteralToken implements Token
{
private String lexeme;
private int offset;
@Override
public String lexeme() {
return this.lexeme;
}
@Override
public int offset() {
return this.offset;
}
@Override
public boolean isWhitespace() {
return false;
}
@Override
public boolean matches(final String input, final int offset) {
final char c = input.charAt(offset);
if (c == '\"' || c == '\'') {
this.lexeme = this.extractLexeme(input, offset + 1, c);
this.offset = offset;
return true;
}
return false;
}
private String extractLexeme(final String input, final int start, final char qualifier) {
boolean escaped = false;
for (int n = start, len = input.length(); n < len; ++n) {
final char c = input.charAt(n);
if (c == qualifier) {
if (!escaped) {
return input.substring(start - 1, n + 1);
}
escaped = false;
}
else {
escaped = (c == '\\' && !escaped);
}
}
return input.substring(start - 1);
}
@Override
public String toString() {
return this.lexeme + " @" + this.offset;
}
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\RaiseErrorToken.java
package org.jitterbit.integration.client.ui.script.partition;
import org.apache.commons.lang3.StringUtils;
import org.jitterbit.util.text.partition.Token;
import org.jitterbit.util.text.partition.OneOfToken;
final class RaiseErrorToken extends OneOfToken implements Token
{
public RaiseErrorToken() {
super(new String[] { "RaiseError(" }, false);
}
public String lexeme() {
return StringUtils.removeEnd(super.lexeme(), "(");
}
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\FunctionToken.java
package org.jitterbit.integration.client.ui.script.partition;
import java.util.stream.Stream;
import org.jitterbit.integration.data.script.FunctionCategory;
import org.jitterbit.integration.data.script.Function;
import java.util.stream.Collector;
import java.util.stream.Collectors;
import java.util.List;
import org.jitterbit.integration.data.script.FunctionRegistry;
import org.apache.commons.lang3.StringUtils;
import org.jitterbit.util.text.partition.Token;
import org.jitterbit.util.text.partition.OneOfToken;
final class FunctionToken extends OneOfToken implements Token
{
private static final String[] TOKENS_JB;
private static final String[] TOKENS_JS;
public FunctionToken(final boolean isJavaScript) {
super(isJavaScript ? FunctionToken.TOKENS_JS : FunctionToken.TOKENS_JB, false);
}
public String lexeme() {
return StringUtils.removeEnd(super.lexeme(), "(");
}
private static String[] createTokens(final boolean isJavaScript) {
final FunctionRegistry functions = FunctionRegistry.getRegistry(isJavaScript);
final List<String> tokens = functions.getCategories().stream().flatMap(c -> c.getFunctions().stream()).map(f -> f.getName() + (isJavaScript ? "" : "(")).collect((Collector<? super Object, ?, List<String>>)Collectors.toList());
return tokens.toArray(new String[tokens.size()]);
}
static {
TOKENS_JB = createTokens(false);
TOKENS_JS = createTokens(true);
}
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\OperatorToken.java
package org.jitterbit.integration.client.ui.script.partition;
import org.jitterbit.util.text.partition.Token;
import org.jitterbit.util.text.partition.OneOfToken;
final class OperatorToken extends OneOfToken implements Token
{
private static final String[] TOKENS;
public OperatorToken() {
super(OperatorToken.TOKENS, true);
}
static {
TOKENS = new String[] { "<=", ">=", "==", "!=", ">", "<", "&", "|", "^", "+", "-", "*", "/", ";" };
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\BooleanConstant.java
package org.jitterbit.util.text.partition;
public class BooleanConstant extends OneOfToken implements Token
{
public BooleanConstant() {
super(new String[] { "true", "false" }, false, true);
}
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\GroupingToken.java
package org.jitterbit.integration.client.ui.script.partition;
import org.jitterbit.util.text.partition.Token;
import org.jitterbit.util.text.partition.OneOfToken;
final class GroupingToken extends OneOfToken implements Token
{
public GroupingToken() {
super(new String[] { "(", ")" }, true);
}
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\SourcePathToken.java
package org.jitterbit.integration.client.ui.script.partition;
import org.jitterbit.integration.structure.mapping.PathSeparator;
import org.jitterbit.util.text.partition.Token;
final class SourcePathToken implements Token
{
private String lexeme;
private int offset;
public String lexeme() {
return this.lexeme;
}
public boolean isWhitespace() {
return false;
}
public boolean matches(final String input, final int offset) {
if (Character.isJavaIdentifierStart(input.charAt(offset))) {
final int end = this.getEndPosition(input, offset + 1);
this.lexeme = input.substring(offset, end);
this.offset = offset;
return true;
}
this.lexeme = null;
this.offset = -1;
return false;
}
public int offset() {
return this.offset;
}
private int getEndPosition(final String input, final int start) {
for (int n = start, len = input.length(); n < len; ++n) {
final char c = input.charAt(n);
if (!Character.isJavaIdentifierPart(c) && !PathSeparator.isSeparator(c)) {
return n;
}
}
return input.length();
}
@Override
public String toString() {
return this.lexeme + " @" + this.offset;
}
}jitterbit-studio-core-ui-1.0.0-SNAPSHOT\org\jitterbit\integration\client\ui\script\partition\GlobalDataElementToken.java
package org.jitterbit.integration.client.ui.script.partition;
import org.jitterbit.integration.dataelement.DeSyntax;
import org.jitterbit.util.text.partition.Token;
public final class GlobalDataElementToken implements Token
{
private String lexeme;
private int offset;
public GlobalDataElementToken() {
this.offset = -1;
}
public boolean isWhitespace() {
return false;
}
public String lexeme() {
return this.lexeme;
}
public boolean matches(final String input, final int offset) {
if (DeSyntax.SCRIPT.isStartMarker(input.charAt(offset))) {
final int end = this.getEndOffset(input, offset);
if (end - offset > 1) {
this.offset = offset;
this.lexeme = input.substring(offset, end);
return true;
}
}
this.offset = -1;
this.lexeme = null;
return false;
}
public int offset() {
return this.offset;
}
private int getEndOffset(final String input, final int start) {
int index = start + 1;
for (int len = input.length(); index < len; ++index) {
final char c = input.charAt(index);
if (!isValid(c)) {
break;
}
}
return index;
}
private static boolean isValid(final char c) {
switch (c) {
case '!':
case '\"':
case '&':
case '\'':
case '(':
case ')':
case '*':
case '+':
case ',':
case '-':
case '/':
case ';':
case '<':
case '=':
case '>':
case '[':
case ']':
case '^':
case '{':
case '|':
case '}': {
return false;
}
default: {
return !Character.isWhitespace(c);
}
}
}
@Override
public String toString() {
return this.lexeme + " @" + this.offset;
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\RegExToken.java
package org.jitterbit.util.text.partition;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class RegExToken implements Token
{
private final Pattern pattern;
private Matcher matcher;
private String lexeme;
private int offset;
public RegExToken(final String pattern) {
this.pattern = Pattern.compile(pattern);
this.offset = -1;
}
@Override
public String lexeme() {
return this.lexeme;
}
@Override
public boolean isWhitespace() {
return false;
}
@Override
public boolean matches(final String input, final int offset) {
final String part = input.substring(offset, input.length());
this.matcher = this.pattern.matcher(part);
if (this.matcher.find() && this.matcher.start() == 0) {
this.offset = offset;
this.lexeme = part.substring(0, this.matcher.end());
return true;
}
this.offset = -1;
this.lexeme = null;
return false;
}
@Override
public int offset() {
return this.offset;
}
@Override
public String toString() {
return this.lexeme + " @" + this.offset;
}
}jitterbit-utils-1.0.0-SNAPSHOT\org\jitterbit\util\text\partition\WordToken.java
package org.jitterbit.util.text.partition;
import java.text.BreakIterator;
public class WordToken implements Token
{
private String lexeme;
private int offset;
@Override
public String lexeme() {
return this.lexeme;
}
@Override
public int offset() {
return this.offset;
}
@Override
public boolean isWhitespace() {
return false;
}
@Override
public boolean matches(final String input, final int offset) {
final BreakIterator it = BreakIterator.getWordInstance();
final String part = input.substring(offset, input.length());
it.setText(part);
final int start = it.first();
final int end = it.next();
if (end == -1) {
this.offset = -1;
this.lexeme = null;
return false;
}
this.offset = offset;
this.lexeme = part.substring(start, end);
return true;
}
@Override
public String toString() {
return this.lexeme + " @" + this.offset;
}
}