[PIO-199] Spark 2.4 support for Scala 2.11 (#503)

* Spark 2.4 support for Scala 2.11
* Update Spark 2.2 series in .travis.yml to 2.2.3

Author: shimamoto

.travis.yml

@@ -73,7 +73,7 @@ env:
     - BUILD_TYPE=Integration
       METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
       PIO_SCALA_VERSION=2.11.12
-      PIO_SPARK_VERSION=2.2.2
+      PIO_SPARK_VERSION=2.2.3
       PIO_HADOOP_VERSION=2.6.5
     - BUILD_TYPE=Integration
       METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
@@ -94,13 +94,18 @@ env:
     - BUILD_TYPE=Integration
       METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
       PIO_SCALA_VERSION=2.11.12
-      PIO_SPARK_VERSION=2.2.2
+      PIO_SPARK_VERSION=2.2.3
       PIO_HADOOP_VERSION=2.7.7
     - BUILD_TYPE=Integration
       METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
       PIO_SCALA_VERSION=2.11.12
       PIO_SPARK_VERSION=2.3.2
       PIO_HADOOP_VERSION=2.7.7
+    - BUILD_TYPE=Integration
+      METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+      PIO_SCALA_VERSION=2.11.12
+      PIO_SPARK_VERSION=2.4.0
+      PIO_HADOOP_VERSION=2.7.7
 
     - BUILD_TYPE=LicenseCheck
 

build.sbt

@@ -16,25 +16,6 @@
  */
 import PIOBuild._
 
-lazy val scalaSparkDepsVersion = Map(
-  "2.11" -> Map(
-    "2.0" -> Map(
-      "akka" -> "2.5.16",
-      "hadoop" -> "2.7.7",
-      "json4s" -> "3.2.11"),
-    "2.1" -> Map(
-      "akka" -> "2.5.17",
-      "hadoop" -> "2.7.7",
-      "json4s" -> "3.2.11"),
-    "2.2" -> Map(
-      "akka" -> "2.5.17",
-      "hadoop" -> "2.7.7",
-      "json4s" -> "3.2.11"),
-    "2.3" -> Map(
-      "akka" -> "2.5.17",
-      "hadoop" -> "2.7.7",
-      "json4s" -> "3.2.11")))
-
 name := "apache-predictionio-parent"
 
 version in ThisBuild := "0.14.0-SNAPSHOT"
@@ -60,23 +41,22 @@ sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.3")
 
 sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value)
 
-akkaVersion in ThisBuild := sys.props.getOrElse(
-  "akka.version",
-  scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("akka"))
+hadoopVersion in ThisBuild := sys.props.getOrElse("hadoop.version", "2.7.7")
+
+akkaVersion in ThisBuild := sys.props.getOrElse("akka.version", "2.5.17")
 
 lazy val es = sys.props.getOrElse("elasticsearch.version", "5.6.9")
 
 elasticsearchVersion in ThisBuild := es
 
-lazy val hbase = sys.props.getOrElse("hbase.version", "1.2.6")
-
-hbaseVersion in ThisBuild := hbase
+hbaseVersion in ThisBuild := sys.props.getOrElse("hbase.version", "1.2.6")
 
-json4sVersion in ThisBuild := scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("json4s")
-
-hadoopVersion in ThisBuild := sys.props.getOrElse(
-  "hadoop.version",
-  scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("hadoop"))
+json4sVersion in ThisBuild := {
+  sparkBinaryVersion.value match {
+    case "2.0" | "2.1" | "2.2" | "2.3" => "3.2.11"
+    case "2.4" => "3.5.3"
+  }
+}
 
 val conf = file("conf")
 

data/build.sbt

@@ -20,6 +20,7 @@ import PIOBuild._
 name := "apache-predictionio-data"
 
 libraryDependencies ++= Seq(
+  "org.scala-lang"          % "scala-reflect"  % scalaVersion.value,
   "com.github.nscala-time" %% "nscala-time"    % "2.6.0",
   "com.google.guava"        % "guava"          % "14.0.1",
   "com.typesafe.akka"      %% "akka-http-testkit" % "10.1.5" % "test",

docs/manual/source/install/install-sourcecode.html.md.erb

@@ -27,8 +27,8 @@ replace `/home/abc` with your own home directory wherever you see it.
 You can use pre-built binary distribution for Apache PredictionIO® if you are
 building against
 
-* Scala 2.11.8
-* Spark 2.1.1
+* Scala 2.11.12
+* Spark 2.1.3
 * Hadoop 2.7.7
 * Elasticsearch 5.6.9
 
@@ -100,17 +100,14 @@ and complete information.
 ### Building
 
 Run the following at the directory where you downloaded the source code to build
-Apache PredictionIO®. By default, the build will be against
-
-* Scala 2.11.8
-* Spark 2.1.1
-* Hadoop 2.7.7
-* Elasticsearch 5.6.9
+Apache PredictionIO®.
+As an example, if you want to build PredictionIO to support Scala 2.11.12,
+Spark 2.4.0, and Elasticsearch 6.4.2, you can do
 
 ```
 $ tar zxvf apache-predictionio-<%= data.versions.pio %>.tar.gz
 $ cd apache-predictionio-<%= data.versions.pio %>
-$ ./make-distribution.sh
+$ ./make-distribution.sh -Dscala.version=2.11.12 -Dspark.version=2.4.0 -Delasticsearch.version=6.4.2
 ```
 
 You should see something like the following when it finishes building
@@ -136,17 +133,10 @@ Starting from version 0.11.0, PredictionIO can be built against different
 versions of dependencies. As of writing, one could build PredictionIO against
 these different dependencies:
 
-* 2.11.x
-* 2.0.x, 2.1.x, 2.2.x, 2.3.x
+* Scala 2.11.x
+* Spark 2.0.x, 2.1.x, 2.2.x, 2.3.x, 2.4.x
 * Hadoop 2.6.x, 2.7.x
-* Elasticsearch 1.7.x(deprecated), 5.x
-
-As an example, if you want to build PredictionIO to support Scala 2.11.8,
-Spark 2.1.0, and Elasticsearch 5.3.0, you can do
-
-```
-$ ./make-distribution.sh -Dscala.version=2.11.8 -Dspark.version=2.1.0 -Delasticsearch.version=5.3.0
-```
+* Elasticsearch 1.7.x(deprecated), 5.6.x, 6.x
 
 ## Installing Dependencies
 

storage/elasticsearch/build.sbt

@@ -19,15 +19,13 @@ import PIOBuild._
 
 name := "apache-predictionio-data-elasticsearch"
 
-elasticsearchSparkArtifact := (if (majorVersion(sparkVersion.value) == 2) "elasticsearch-spark-20" else "elasticsearch-spark-13")
-
 elasticsearchVersion := (if (majorVersion(elasticsearchVersion.value) < 5) "5.6.9" else elasticsearchVersion.value)
 
 libraryDependencies ++= Seq(
   "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
   "org.apache.spark"        %% "spark-core"               % sparkVersion.value % "provided",
-  "org.elasticsearch.client" % "elasticsearch-rest-client"                     % elasticsearchVersion.value,
-  "org.elasticsearch"       %% elasticsearchSparkArtifact.value % elasticsearchVersion.value
+  "org.elasticsearch.client" % "elasticsearch-rest-client" % elasticsearchVersion.value,
+  "org.elasticsearch"       %% "elasticsearch-spark-20" % elasticsearchVersion.value
     exclude("org.apache.spark", "*"),
   "org.elasticsearch"        % "elasticsearch-hadoop-mr"  % elasticsearchVersion.value,
   "org.specs2"              %% "specs2"                   % "2.3.13" % "test")