Merge pull request #49 from rob2001/KDTree

K-D Tree Build Improvements

Author: pushkar

src/func/inst/KDTree.java

@@ -15,13 +15,13 @@
  * @version 1.0
  */
 public class KDTree implements Serializable {
-    
+
     /**
      * Random number generator
      */
     private static final Random random = new Random();
 
-    /** 
+    /**
      * The head node of the kd tree
      */
     private KDTreeNode head;
@@ -30,7 +30,7 @@ public class KDTree implements Serializable {
      * The dimensionality of the tree (k)
      */
     private int dimensions;
-    
+
     /**
      * The distance measure to use
      */
@@ -51,7 +51,7 @@ public KDTree(DataSet keys, DistanceMeasure distance) {
         }
         head = buildTree(nodes, 0, nodes.length);
     }
-    
+
     /**
      * Build a kd tree from the given parallel arrays
      * of keys and data
@@ -63,12 +63,23 @@ public KDTree(DataSet keys) {
         this(keys, new EuclideanDistance());
     }
 
+    /**
+     * Builds a tree from a list of nodes
+     * @param nodes the list of nodes
+     * @param start the starting index
+     * @param end the ending index
+     * @return the head node of the build tree
+     */
+    private KDTreeNode buildTree(KDTreeNode[] nodes, int start, int end) {
+        return buildTree(nodes, start, end, 0);
+    }
+
     /**
      * Build a tree from a list of nodes
      * @param nodes the list of nodes
      * @return the head node of the built tree
      */
-    private KDTreeNode buildTree(KDTreeNode[] nodes, int start, int end) {
+    private KDTreeNode buildTree(KDTreeNode[] nodes, int start, int end, int depth) {
         if (start >= end) {
             // if we're done return null
             return null;
@@ -77,16 +88,16 @@ private KDTreeNode buildTree(KDTreeNode[] nodes, int start, int end) {
             return nodes[start];
         }
         // choose splitter
-        int splitterIndex = chooseSplitterRandom(nodes, start, end);
+        int splitterIndex = chooseApproxBestSplitter(nodes, start, end,depth);
         KDTreeNode splitter = nodes[splitterIndex];
-        //  patition based on splitter
+        //  partition based on splitter
         splitterIndex = partition(nodes, start, end, splitterIndex);
         // recursively build tree
-        splitter.setLeft(buildTree(nodes, start, splitterIndex));
-        splitter.setRight(buildTree(nodes, splitterIndex + 1, end));
+        splitter.setLeft(buildTree(nodes, start, splitterIndex, depth+1));
+        splitter.setRight(buildTree(nodes, splitterIndex + 1, end, depth+1));
         return splitter;
     }
-    
+
     /**
      * Partition an array based on a splitter
      * @param comparables the array
@@ -95,7 +106,7 @@ private KDTreeNode buildTree(KDTreeNode[] nodes, int start, int end) {
      * @param splitterIndex the splitter's index
      * @return the new splitter index
      */
-    private int partition(Comparable[] comparables, int start, int end, 
+    private int partition(Comparable[] comparables, int start, int end,
             int splitterIndex) {
         swap(comparables, splitterIndex, end - 1);
         splitterIndex = end - 1;
@@ -110,7 +121,7 @@ private int partition(Comparable[] comparables, int start, int end,
         swap(comparables, splitterIndex, i + 1);
         return i + 1;
     }
-    
+
     /**
      * Swap two elements in an array
      * @param objects the array
@@ -122,7 +133,7 @@ private void swap(Object[] objects, int i, int j) {
         objects[i] = objects[j];
         objects[j] = temp;
     }
-    
+
     /**
      * Choose a random splitter
      * @param nodes the nodes to choose from
@@ -136,7 +147,81 @@ private int chooseSplitterRandom(KDTreeNode[] nodes, int start, int end) {
       nodes[splitter].setDimension(dimension);
       return splitter;
     }
-    
+
+
+    /**
+    * Use quickSelect and medianOfMedians to select a item
+    * guarenteed to be in the middle 50% of the data. Along the chosen dimension
+    * This function perpares the data to be processed by the quickSelect function
+    * It selects dimensions in order, so that each level of the final KDTree
+    * splits on a different dimension
+    * @param nodes the nodes to choose from
+    * @param start the starting index
+    * @param end the ending index
+    * @param depth the tree depth that the splitter will be placed at
+    * @return the index of the splitter
+    */
+    private int chooseApproxBestSplitter(KDTreeNode[] nodes, int start, int end, int depth){
+        int dimension = depth % dimensions;
+        for (int k = start; k < end; k++) nodes[k].setDimension(dimension);
+        return medianOfMedians(nodes,start,end);
+    }
+    /**
+    * This function implements quickSelect on the passed in KDTreeNode Array
+    * quickSelect returns the n's element of a list, defined by a predefined
+    * ordering of elements (implemented in KDTreeNode)
+    *
+    * @param n the element, by number, we want to return
+    * @param nodes the list of KDTreeNodes to search through
+    * @param start the lower bound of the search, inclusive
+    * @param end the upper bound of the search, exclusive
+    * @return the index of element n
+    */
+    private int quickSelect(int n, KDTreeNode[] nodes, int start, int end){
+        while (start != end){
+            // if (n < start || n >=end){
+            //     System.out.println("ERROR");
+            //     return start;
+            // }
+            int pivot = medianOfMedians(nodes,start,end);
+            pivot = partition(nodes,start,end,pivot);
+            if (n == pivot) return n;
+            else if (n > pivot) start = pivot+1;
+            else end = pivot;
+        }
+        return start;
+    }
+
+    /**
+    * This function implements medianOfMedians on the passed in KDTreeNode Array
+    * this fn returns an acceptable splitter, guarenteed to be greater than 25%
+    * of the data and less than 25% of the data.
+    *
+    * @param nodes the list of KDTreeNodes to search through
+    * @param start the lower bound of the search, inclusive
+    * @param end the upper bound of the search, exclusive
+    * @return the index of the splitter
+    */
+    private int medianOfMedians(KDTreeNode[] nodes, int start, int end){
+        int MEDIAN_SIZE = 5;
+        int partitions;
+        int length = end-start;
+        if (length < 10) {
+            Arrays.sort(nodes,start,end);
+            return start + length/2;
+        }
+        partitions = length/MEDIAN_SIZE;
+        if (length % MEDIAN_SIZE != 0) partitions++;
+        for (int i = 0; i < partitions; i++){
+            int pstart = start + (i*MEDIAN_SIZE);
+            int pend = Math.min(pstart+MEDIAN_SIZE, end);
+            int pmiddle = pstart + (pend-pstart)/2;
+            Arrays.sort(nodes,pstart,pend);
+            swap(nodes,start+i,pmiddle);
+        }
+        return quickSelect(start+(partitions/2),nodes,start,start+partitions);
+
+    }
 
     /**
      * Choose a splitter from a list of nodes
@@ -173,7 +258,7 @@ private int chooseSplitterSmart(KDTreeNode[] nodes, int start, int end) {
         double median = (max[widestDimension] - min[widestDimension]) / 2;
         // find the best splitter
         double bestDifference = Double.POSITIVE_INFINITY;
-        int splitterIndex = -1; 
+        int splitterIndex = -1;
         for (int i = start; i < end; i++) {
             KDTreeNode node = nodes[i];
             if (Math.abs(node.getInstance().getContinuous(widestDimension) - median)
@@ -198,7 +283,7 @@ public Instance[] knn(Instance target, int k) {
         knn(head, target, new HyperRectangle(dimensions), results);
         return results.getNearest();
     }
-    
+
     /**
      * Perform a nearest neighbor search
      * @param target the target
@@ -207,7 +292,7 @@ public Instance[] knn(Instance target, int k) {
     public Instance[] nn(Instance target) {
         NearestNeighborQueue results = new NearestNeighborQueue();
         knn(head, target, new HyperRectangle(dimensions), results);
-        return results.getNearest();         
+        return results.getNearest();
     }
 
     /**
@@ -219,9 +304,9 @@ public Instance[] nn(Instance target) {
     public Instance[] range(Instance target, double range) {
         NearestNeighborQueue results = new NearestNeighborQueue(range);
         knn(head, target, new HyperRectangle(dimensions), results);
-        return results.getNearest();         
+        return results.getNearest();
     }
-    
+
     /**
      * Perform a k nearest neighbor range search
      * @param target the target
@@ -232,17 +317,17 @@ public Instance[] range(Instance target, double range) {
     public Instance[] knnrange(Instance target, int k, double range) {
         NearestNeighborQueue results = new NearestNeighborQueue(k, range);
         knn(head, target, new HyperRectangle(dimensions), results);
-        return results.getNearest();         
+        return results.getNearest();
     }
-    
+
     /**
      * Perform a nearest neighbor search
      * @param node the node to search on
      * @param target the target
      * @param hr the hyper rectangle
      * @param results the current results
      */
-    private void knn(KDTreeNode node, Instance target, HyperRectangle hr,  
+    private void knn(KDTreeNode node, Instance target, HyperRectangle hr,
             NearestNeighborQueue results) {
         if (node == null) {
             return;
@@ -262,11 +347,12 @@ private void knn(KDTreeNode node, Instance target, HyperRectangle hr,
         }
         knn(nearNode, target, nearHR, results);
         if (distanceMeasure.value(
-                farHR.pointNearestTo(target), target) 
+                farHR.pointNearestTo(target), target)
                 <= results.getMaxDistance()) {
-            results.add(node.getInstance(), 
+            results.add(node.getInstance(),
                 distanceMeasure.value(node.getInstance(), target));
             knn(farNode, target, farHR, results);
         }
     }
+
 }

src/func/test/KNNClassifierAbaloneTest.java

@@ -0,0 +1,85 @@
+package func.test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.util.Scanner;
+
+import func.inst.KDTree;
+import opt.test.AbaloneTest;
+import shared.DataSet;
+import shared.Instance;
+
+/**
+ * KNN test built to test the speed of splitter selectors in func.inst.KDTree
+ * @author Robert Smith [email protected]
+ * @version 1.0
+ */
+public class KNNClassifierAbaloneTest {
+
+    public static void main(String[] args){
+        Instance[] instances = initializeAbaloneInstances();
+        Instance[] training = new Instance[3000];
+        Instance[] testing = new Instance[1177];
+        for (int i = 0; i < training.length; i++){
+            training[i] = instances[i];
+        }
+        for (int i = 0; i < testing.length; i++){
+            testing[i] = instances[training.length+i];
+        }
+        long buildTime = System.nanoTime();
+        KDTree tree = new KDTree(new DataSet(training));
+        buildTime = System.nanoTime() - buildTime;
+        System.out.println("BuildTime = " + buildTime);
+
+        for (int k = 1; k < 10; k++){
+            long testTime = 0;
+            for (int i = 0; i < testing.length; i++){
+                long searchTime = System.nanoTime();
+                tree.knn(testing[i], k);
+                searchTime = System.nanoTime() - searchTime;
+                testTime += searchTime;
+            }
+            testTime /= testing.length;
+            System.out.println("K = " + k + ", average search time = " + testTime);
+        }
+
+
+    }
+
+    private static Instance[] initializeAbaloneInstances() {
+
+        double[][][] attributes = new double[4177][][];
+
+        try {
+            BufferedReader br = new BufferedReader(new FileReader(new File("src/opt/test/abalone.txt")));
+
+            for(int i = 0; i < attributes.length; i++) {
+                Scanner scan = new Scanner(br.readLine());
+                scan.useDelimiter(",");
+
+                attributes[i] = new double[2][];
+                attributes[i][0] = new double[7]; // 7 attributes
+                attributes[i][1] = new double[1];
+
+                for(int j = 0; j < 7; j++)
+                    attributes[i][0][j] = Double.parseDouble(scan.next());
+
+                attributes[i][1][0] = Double.parseDouble(scan.next());
+            }
+        }
+        catch(Exception e) {
+            e.printStackTrace();
+        }
+
+        Instance[] instances = new Instance[attributes.length];
+
+        for(int i = 0; i < instances.length; i++) {
+            instances[i] = new Instance(attributes[i][0]);
+            // classifications range from 0 to 30; split into 0 - 14 and 15 - 30
+            instances[i].setLabel(new Instance(attributes[i][1][0] < 15 ? 0 : 1));
+        }
+
+        return instances;
+    }
+}