Deep match amb child memoization

src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc

@@ -50,5 +50,5 @@ int main(list[str] args) {
     return 0;
 }
 
-int rascalSmokeTest() = main(["source-loc=|std:///|", "max-amb-depth=2", "max-files=3", "max-file-size=500", "sample-window=3", "random-seed=1"]);
+int rascalSmokeTest() = main(["source-loc=|std:///|", "max-amb-depth=2", "max-files=3", "max-file-size=500", "sample-window=3", "random-seed=1", "count-nodes=true"]);
 int rascalStandardTest() = main(["source-loc=|std:///|", "max-files=1000", "max-file-size=5120"]);

src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorTreeSemanticsTests.rsc

@@ -281,7 +281,7 @@ test bool testConcreteMatchWithErrors() {
 }
 
 @description{
-This function a test tree that has plenty of oppoertunities to memo amb children:
+This function creates a test tree that has plenty of oppoertunities to memo amb children:
  ❖
  ├─ Amb = AmbWord  () 
  │  ├─ ❖
@@ -318,7 +318,7 @@ This function a test tree that has plenty of oppoertunities to memo amb children
 }
 private Amb ambTestTree() = parse(#Amb, "^X$", allowRecovery=true, allowAmbiguity=true);
 
-test bool testDeepMatchAmbMemo() {
+test bool testNodeDeepMatchAmbMemo() {
     Amb ambTree = ambTestTree();
 
     // Count the number of errors that is actually found by a deep match
@@ -328,6 +328,24 @@ test bool testDeepMatchAmbMemo() {
     return count == 3;
 }
 
+test bool testConcreteDeepMatchAmbMemo() {
+    Amb ambTree = ambTestTree();
+
+    // Count the number of errors that is actually found by a deep match
+    int count = (0 | it + 1 | /(AmbWord)`<AmbWord w>` := ambTree, isParseError(w));
+
+    // There will only be 3 matches if deep matches are memoized, 6 if they are not.
+    return count == 3;
+}
+
+test bool testAmbMatchAmbMemo() {
+    Amb ambTree = ambTestTree();
+
+    int count = (0 | it + 1 | /a:amb(alts) := ambTree);
+
+    return count == 2;
+}
+
 test bool testVisitAmbMemo() {
     Amb ambTree = ambTestTree();
 

src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc

@@ -48,6 +48,7 @@ public data RecoveryTestConfig = recoveryTestConfig(
     int fromFile = 0,
     int sampleWindow = 1,
     bool countNodes = false,
+    bool countTreeMatches = true,
     bool verifyResult = false,
     loc statFile = |unknown:///|
 );
@@ -112,6 +113,7 @@ private TestMeasurement testRecovery(RecoveryTestConfig config, &T (value input,
     int nodeCountUnique = -1;
     int disambNodeCount = -1;
     int disambNodeCountUnique = -1;
+    int matchTreeCount = -1;
 
     TestMeasurement measurement = successfulParse();
     startTime = realTime();
@@ -135,6 +137,15 @@ private TestMeasurement testRecovery(RecoveryTestConfig config, &T (value input,
                 list[Tree] errors = findAllParseErrors(disambTree);
                 errorCount = size(errors);
                 errorSize = (0 | it + size(getErrorText(err)) | err <- errors);
+
+                if (config.countTreeMatches) {
+                    int matchStartTime = realTime();
+                    matchTreeCount = (0 | it + 1 | /Tree t <- tree);
+                    int matchDuration = realTime() - matchStartTime;
+                    if (matchDuration > 1000) {
+                        println("Tree match count calculation took too long: <matchDuration> ms, <matchTreeCount> nodes matched");
+                    }
+                }
             }
 
             measurement = recovered(source=source, duration=duration, errorCount=errorCount, errorSize=errorSize);
@@ -158,8 +169,9 @@ private TestMeasurement testRecovery(RecoveryTestConfig config, &T (value input,
         if (config.countNodes) {
             int nodeRatio = percent(nodeCount, referenceNodeCount);
             int unodeRatio = percent(nodeCountUnique, referenceNodeCountUnique);
+            int matchTreeCountRatio = matchTreeCount / nodeCountUnique;
 
-            appendToFile(config.statFile, "<source>,<size(input)>,<result>,<duration>,<durationRatio>,<nodeRatio>,<unodeRatio>,<disambDuration>,<errorCount>,<errorSize>,<nodeCount>,<nodeCountUnique>,<disambNodeCount>,<disambNodeCountUnique>\n");
+            appendToFile(config.statFile, "<source>,<size(input)>,<result>,<duration>,<durationRatio>,<nodeRatio>,<unodeRatio>,<disambDuration>,<errorCount>,<errorSize>,<nodeCount>,<nodeCountUnique>,<disambNodeCount>,<disambNodeCountUnique>,<matchTreeCount>,<matchTreeCountRatio>\n");
         } else {
             appendToFile(config.statFile, "<source>,<size(input)>,<result>,<duration>,<durationRatio>\n");
         }
@@ -577,7 +589,11 @@ void batchRecoveryTest(RecoveryTestConfig config) {
 
     if (config.statFile != |unknown:///|) {
         if (config.countNodes) {
-            writeFile(config.statFile, "source,size,result,duration,durationRatio,nodeRatio,unodeRatio,disambiguationDuration,errorCount,errorSize,nodes,unodes,disambNodes,udisambNodes\n");
+            if (config.countTreeMatches) {
+                writeFile(config.statFile, "source,size,result,duration,durationRatio,nodeRatio,unodeRatio,disambiguationDuration,errorCount,errorSize,nodes,unodes,disambNodes,udisambNodes,matchTreeCount,matchTreeCountRatio\n");
+            } else {
+                writeFile(config.statFile, "source,size,result,duration,durationRatio,nodeRatio,unodeRatio,disambiguationDuration,errorCount,errorSize,nodes,unodes,disambNodes,udisambNodes\n");
+            }
         } else {
             writeFile(config.statFile, "source,size,result,duration,durationRatio\n");
         }

src/org/rascalmpl/values/iterators/DescendantReader.java

@@ -16,6 +16,7 @@
 import java.util.Iterator;
 import java.util.Stack;
 
+import io.usethesource.capsule.Set;
 import io.usethesource.vallang.IConstructor;
 import io.usethesource.vallang.IList;
 import io.usethesource.vallang.IMap;
@@ -35,6 +36,10 @@
 public class DescendantReader implements Iterator<IValue> {
 
 	Stack<Object> spine = new Stack<Object>();
+	IValue nextValue;
+
+
+	private Set.Transient<ITree> visitedAmbs = Set.Transient.of();
 
 	private boolean debug = false;
 
@@ -49,40 +54,76 @@ public class DescendantReader implements Iterator<IValue> {
 		push(val);
 	}
 
+	private void findNext() {
+		do {
+			nextValue = null;
+
+			if (spine.size() == 0) {
+				return;
+			}
+
+			Object next = spine.peek();
+			if (next instanceof Iterator) {
+				Iterator<?> iter = (Iterator<?>) next;
+				if (!iter.hasNext()) {
+					spine.pop();
+					continue;
+				}
+
+				push((IValue) iter.next());
+				continue;
+			}
+
+			nextValue = (IValue) spine.pop();
+
+		} while (nextValue == null);
+	}
+
 	public boolean hasNext() {
-		while((spine.size() > 0) 
-				&& (spine.peek() instanceof Iterator && !((Iterator<?>) spine.peek()).hasNext())){
-			spine.pop();
-		}		
-		return spine.size() > 0;
+		if (nextValue == null) {
+			findNext();
+		}
+		return nextValue != null;
 	}
 
 	public IValue next() {
-		if (spine.peek() instanceof Iterator) {
-			Iterator<?> iter = (Iterator<?>) spine.peek();
-
-			if (!iter.hasNext()) {
-				spine.pop();
-				return next();
-			}
-			push((IValue) iter.next());
-			return next();
-		}
-		return (IValue) spine.pop();
+		IValue next = nextValue;
+		findNext();
+		return next;
 	}
 
 	private void push(IValue v, Iterator<IValue> children){
 		spine.push(v);
 		spine.push(children);
 	}
-
+
+	private void pushAmb(ITree amb) {
+		if (!visitedAmbs.contains(amb)) {
+			visitedAmbs.__insert(amb);
+			spine.push(amb);
+			for (IValue alt : TreeAdapter.getAlternatives(amb)) {
+				push(alt);
+			}
+		}
+	}
+
 	private void push(IValue v){
 		Type type = v.getType();
 		if (type.isNode() || type.isConstructor() || type.isAbstractData()) {
-			if (interpretTree && type.isSubtypeOf(RascalValueFactory.Tree)) {
-				pushConcreteSyntaxNode((ITree) v);
-				return;
+			if (type.isSubtypeOf(RascalValueFactory.Tree)) {
+				ITree tree = (ITree) v;
+
+				if (TreeAdapter.isAmb(tree)) {
+					pushAmb(tree);
+					return;
+				}
+
+				if (interpretTree) {
+					pushConcreteSyntaxNode((ITree) tree);
+					return;
+				}
 			}
+
 			INode node = (INode) v;
 			push(v, node.getChildren().iterator());
 			if (node.mayHaveKeywordParameters()) {
@@ -105,7 +146,7 @@ else if(type.isTuple()) {
 			spine.push(v);
 		}
 	}
-	
+
 	private void pushKeywordParameters(IWithKeywordParameters<? extends INode> node) {
 		for (String name : node.getParameterNames()) {
 			push(node.getParameter(name));
@@ -121,10 +162,7 @@ private void pushConcreteSyntaxNode(ITree tree){
 		}
 
 		if (TreeAdapter.isAmb(tree)) {
-			// only recurse
-			for (IValue alt : TreeAdapter.getAlternatives(tree)) {
-				pushConcreteSyntaxNode((ITree) alt);
-			}
+			pushAmb(tree);
 			return;
 		}