fixed missing value handling in csv data type detection

Author: lwhite1

src/main/java/com/github/lwhite1/tablesaw/columns/BooleanColumn.java

@@ -28,6 +28,15 @@ public static BooleanColumn create(String name) {
     return new BooleanColumn(name);
   }
 
+  public static BooleanColumn create(String name, int size, RoaringBitmap values) {
+    BooleanColumn booleanColumn = new BooleanColumn(name, size);
+    IntIterator intIterator = values.getIntIterator();
+    while(intIterator.hasNext()){
+      booleanColumn.set(intIterator.next(), true);
+    }
+    return booleanColumn;
+  }
+
   public static BooleanColumn create(String name, int rowSize) {
     return new BooleanColumn(name, rowSize);
   }

src/main/java/com/github/lwhite1/tablesaw/columns/FloatColumn.java

@@ -268,7 +268,7 @@ public void addCell(String object) {
    */
   public static float convert(String stringValue) {
     if (Strings.isNullOrEmpty(stringValue) || TypeUtils.MISSING_INDICATORS.contains(stringValue)) {
-      return Float.NaN;
+      return MISSING_VALUE;
     }
     Matcher matcher = COMMA_PATTERN.matcher(stringValue);
     return Float.parseFloat(matcher.replaceAll(""));

src/main/java/com/github/lwhite1/tablesaw/io/CsvReader.java

@@ -9,6 +9,7 @@
 import com.github.lwhite1.tablesaw.api.Table;
 import com.github.lwhite1.tablesaw.columns.Column;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import org.apache.commons.lang3.StringUtils;
 
@@ -457,7 +458,11 @@ private static ColumnType detectType(List<String> valuesList) {
 
     CopyOnWriteArrayList<ColumnType> typeCandidates = new CopyOnWriteArrayList<>(typeArray);
 
+
     for (String s : valuesList) {
+      if (Strings.isNullOrEmpty(s) || TypeUtils.MISSING_INDICATORS.contains(s)) {
+        continue;
+      }
       if (typeCandidates.contains(LOCAL_DATE_TIME)) {
         if (!isLocalDateTime.test(s)) {
           typeCandidates.remove(LOCAL_DATE_TIME);
@@ -516,7 +521,7 @@ private static ColumnType selectType(List<ColumnType> typeCandidates) {
   }
 
   private static java.util.function.Predicate<String> isBoolean = s ->
-      TypeUtils.TRUE_STRINGS.contains(s) || TypeUtils.FALSE_STRINGS.contains(s);
+      TypeUtils.TRUE_STRINGS_FOR_DETECTION.contains(s) || TypeUtils.FALSE_STRINGS_FOR_DETECTION.contains(s);
 
   private static Predicate<String> isLong = new Predicate<String>() {
 

src/main/java/com/github/lwhite1/tablesaw/io/TypeUtils.java

@@ -39,10 +39,17 @@ private TypeUtils() {
   public static final List<String> TRUE_STRINGS =
       Arrays.asList("T", "t", "Y", "y", "TRUE", "true", "1");
 
+  public static final List<String> TRUE_STRINGS_FOR_DETECTION =
+      Arrays.asList("T", "t", "Y", "y", "TRUE", "true");
+
   // These Strings will convert to true booleans
   public static final List<String> FALSE_STRINGS =
       Arrays.asList("F", "f", "N", "n", "FALSE", "false", "0");
 
+  // These Strings will convert to true booleans
+  public static final List<String> FALSE_STRINGS_FOR_DETECTION =
+      Arrays.asList("F", "f", "N", "n", "FALSE", "false");
+
   // Formats that we accept in parsing dates from strings
   // TODO: Add more types, especially dates with month names spelled-out fully.
   private static final DateTimeFormatter dtf1 = DateTimeFormatter.ofPattern("yyyyMMdd");

src/test/java/com/github/lwhite1/tablesaw/integration/AirlineDelays2.java

@@ -1,42 +1,57 @@
 package com.github.lwhite1.tablesaw.integration;
 
-import com.github.lwhite1.tablesaw.api.Table;
 import com.github.lwhite1.tablesaw.api.ColumnType;
-import com.github.lwhite1.tablesaw.store.StorageManager;
+import com.github.lwhite1.tablesaw.api.Table;
+import com.github.lwhite1.tablesaw.columns.BooleanColumn;
+import com.github.lwhite1.tablesaw.io.CsvReader;
 import com.google.common.base.Stopwatch;
 
 import java.util.concurrent.TimeUnit;
 
+import static com.github.lwhite1.tablesaw.api.QueryHelper.column;
+import static java.lang.System.out;
+
 /**
  *
  */
 public class AirlineDelays2 {
 
-  private static Table flights2015;
+  private static Table flt2007;
 
   public static void main(String[] args) throws Exception {
 
     new AirlineDelays2();
     Stopwatch stopwatch = Stopwatch.createStarted();
-    Table sorted = flights2015.sortAscendingOn("ORIGIN", "UNIQUE_CARRIER");
+    Table sorted = flt2007.sortAscendingOn("ORIGIN", "UniqueCarrier");
     System.out.println("Sorting " + stopwatch.elapsed(TimeUnit.SECONDS));
     System.out.println(sorted.first(1000).print());
     System.exit(0);
   }
 
   private AirlineDelays2() throws Exception {
     Stopwatch stopwatch = Stopwatch.createStarted();
-    System.out.println("loading");
+    out.println("loading");
+    out.println(CsvReader.printColumnTypes("/Users/larrywhite/Downloads/flight delays/2007.csv", true, ','));
+    flt2007 = Table.create("/Users/larrywhite/Downloads/flight delays/2007.csv");
 
-    flights2015 = StorageManager.readTable("bigdata/3f07b9bf-053f-4f9b-9dff-9d354835b276");
+    out.println(flt2007.first(5).print());
 
-    System.out.println(String.format("loaded %d records in %d seconds",
-        flights2015.rowCount(),
+    out.println(String.format("loaded %d records in %d seconds",
+        flt2007.rowCount(),
         (int) stopwatch.elapsed(TimeUnit.SECONDS)));
 
-    out(flights2015.shape());
-    out(flights2015.columnNames().toString());
-    out(flights2015.first(10).print());
+    out(flt2007.shape());
+
+    Table ord = flt2007.selectWhere(column("Origin").isEqualTo("ORD"));
+
+    BooleanColumn delayed = new BooleanColumn("Delayed?", ord.floatColumn("DepDelay").isGreaterThan(15), ord.rowCount());
+    ord.addColumn(delayed);
+
+    out("total flights: " + ord.rowCount());
+    out("delayed flights: " + delayed.countTrue());
+
+    out(flt2007.columnNames().toString());
+    out(flt2007.first(10).print());
   }
 
   private static void out(Object obj) {