Skip to content

Commit 9008b8c

Browse files
mccheahmccheah
committed
Use an nginx server for remote jars tests.
1 parent e24ecda commit 9008b8c

File tree

9 files changed

+191
-6
lines changed

9 files changed

+191
-6
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
.idea/
22
spark/
3+
spark
34
integration-test/target/
45
*.class
56
*.log

‎e2e/runner.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,28 +90,39 @@ git checkout -B $BRANCH origin/$BRANCH
9090
TAG=$(git rev-parse HEAD | cut -c -6)
9191
echo "Spark distribution built at SHA $TAG"
9292

93+
FILE_SERVER_IMAGE="$IMAGE_REPO/spark-examples-file-server:$TAG"
94+
FILE_SERVER_BUILD_DIR="$TEST_ROOT/integration-test/docker-file-server"
95+
rm -rf $FILE_SERVER_BUILD_DIR/jars
96+
mkdir -p $FILE_SERVER_BUILD_DIR/jars
97+
cp $SPARK_REPO_ROOT/dist/examples/jars/spark-examples*.jar $FILE_SERVER_BUILD_DIR/jars/.
9398
cd $SPARK_REPO_ROOT/dist
9499

95100
if [[ $DEPLOY_MODE == cloud ]] ;
96101
then
102+
docker build -t $FILE_SERVER_IMAGE "$TEST_ROOT/integration-test/docker-file-server"
97103
./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $TAG build
98104
if [[ $IMAGE_REPO == gcr.io* ]] ;
99105
then
100106
gcloud docker -- push $IMAGE_REPO/spark-driver:$TAG && \
101107
gcloud docker -- push $IMAGE_REPO/spark-executor:$TAG && \
102108
gcloud docker -- push $IMAGE_REPO/spark-init:$TAG
109+
gcloud docker -- push $FILE_SERVER_IMAGE
103110
else
104111
./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $TAG push
112+
docker push $FILE_SERVER_IMAGE
105113
fi
106114
else
107115
# -m option for minikube.
116+
eval $(minikube docker-env)
117+
docker build -t $FILE_SERVER_IMAGE "$TEST_ROOT/integration-test/docker-file-server"
108118
./sbin/build-push-docker-images.sh -m -r $IMAGE_REPO -t $TAG build
109119
fi
110120

111121
cd $TEST_ROOT/integration-test
112122
$SPARK_REPO_ROOT/build/mvn clean -Ddownload.plugin.skip=true integration-test \
113123
-Dspark-distro-tgz=$SPARK_REPO_ROOT/*.tgz \
114124
-DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://$MASTER \
125+
-Dspark.docker.test.fileServerImage=$FILE_SERVER_IMAGE \
115126
-Dspark.docker.test.driverImage=$IMAGE_REPO/spark-driver:$TAG \
116127
-Dspark.docker.test.executorImage=$IMAGE_REPO/spark-executor:$TAG \
117128
-Dspark.docker.test.initContainerImage=$IMAGE_REPO/spark-init:$TAG" || :

‎integration-test/docker-file-server/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
jars

‎integration-test/docker-file-server/Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
FROM nginx:alpine
2+
3+
COPY jars /opt/spark/jars
4+
COPY nginx.conf /etc/nginx/nginx.conf

‎integration-test/docker-file-server/nginx.conf

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
user nginx;
2+
worker_processes 1;
3+
4+
error_log /var/log/nginx/error.log warn;
5+
pid /var/run/nginx.pid;
6+
7+
events {
8+
worker_connections 1024;
9+
}
10+
11+
http {
12+
server {
13+
root /opt/spark/jars;
14+
location /ping {
15+
return 200 'pong';
16+
add_header Content-Type text/plain;
17+
}
18+
}
19+
include /etc/nginx/mime.types;
20+
default_type application/octet-stream;
21+
22+
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
23+
'$status $body_bytes_sent "$http_referer" '
24+
'"$http_user_agent" "$http_x_forwarded_for"';
25+
26+
access_log /var/log/nginx/access.log main;
27+
28+
sendfile on;
29+
#tcp_nopush on;
30+
31+
keepalive_timeout 65;
32+
33+
#gzip on;
34+
}

‎integration-test/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@
135135
<executable>/bin/sh</executable>
136136
<arguments>
137137
<argument>-c</argument>
138-
<argument>rm -rf spark-distro; mkdir spark-distro-tmp; cd spark-distro-tmp; tar xfz ${spark-distro-tgz}; mv * ../spark-distro; cd ..; rm -rf spark-distro-tmp</argument>
138+
<argument>rm -rf spark-distro; mkdir spark-distro-tmp; cd spark-distro-tmp; tar xfz ${spark-distro-tgz}; mv * ../spark-distro; cd ..; rm -rf spark-distro-tmp; rm -rf docker-file-server/jars; mkdir -p docker-file-server/jars; cp spark-distro/examples/jars/spark-examples*.jar docker-file-server/jars/.</argument>
139139
</arguments>
140140
</configuration>
141141
</execution>

‎integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@
1717
package org.apache.spark.deploy.k8s.integrationtest
1818

1919
import java.io.File
20+
import java.net.URI
2021
import java.nio.file.Paths
2122
import java.util.UUID
2223
import java.util.regex.Pattern
2324

2425
import scala.collection.JavaConverters._
25-
2626
import com.google.common.io.PatternFilenameFilter
2727
import io.fabric8.kubernetes.api.model.{Container, Pod}
2828
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSuite}
@@ -39,6 +39,7 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
3939
private val APP_LOCATOR_LABEL = UUID.randomUUID().toString.replaceAll("-", "")
4040
private var kubernetesTestComponents: KubernetesTestComponents = _
4141
private var sparkAppConf: SparkAppConf = _
42+
private var remoteExamplesJarUri: URI = _
4243

4344
private val driverImage = System.getProperty(
4445
"spark.docker.test.driverImage",
@@ -50,7 +51,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
5051
"spark.docker.test.initContainerImage",
5152
"spark-init:latest")
5253

53-
5454
override def beforeAll(): Unit = {
5555
testBackend.initialize()
5656
kubernetesTestComponents = new KubernetesTestComponents(testBackend.getKubernetesClient)
@@ -67,6 +67,8 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
6767
.set("spark.kubernetes.driver.label.spark-app-locator", APP_LOCATOR_LABEL)
6868
.set("spark.kubernetes.executor.label.spark-app-locator", APP_LOCATOR_LABEL)
6969
kubernetesTestComponents.createNamespace()
70+
remoteExamplesJarUri = SparkExamplesFileServerRunner
71+
.launchServerAndGetUriForExamplesJar(kubernetesTestComponents)
7072
}
7173

7274
after {
@@ -97,6 +99,11 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
9799
runSparkPiAndVerifyCompletion(appArgs = Array("5"))
98100
}
99101

102+
test("Run SparkPi using the remote example jar.") {
103+
sparkAppConf.set("spark.kubernetes.initContainer.image", initContainerImage)
104+
runSparkPiAndVerifyCompletion(appResource = remoteExamplesJarUri.toString)
105+
}
106+
100107
test("Run SparkPi with custom driver pod name, labels, annotations, and environment variables.") {
101108
sparkAppConf
102109
.set("spark.kubernetes.driver.pod.name", "spark-integration-spark-pi")
@@ -163,8 +170,8 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
163170

164171
createTestSecret()
165172

166-
runSparkPageRankAndVerifyCompletion(
167-
appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE),
173+
runSparkPiAndVerifyCompletion(
174+
appResource = remoteExamplesJarUri.toString,
168175
driverPodChecker = (driverPod: Pod) => {
169176
doBasicDriverPodCheck(driverPod)
170177
checkTestSecret(driverPod, withInitContainer = true)
@@ -188,7 +195,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
188195
driverPodChecker,
189196
executorPodChecker)
190197
}
191-
192198
private def runSparkPageRankAndVerifyCompletion(
193199
appResource: String = CONTAINER_LOCAL_SPARK_DISTRO_EXAMPLES_JAR,
194200
driverPodChecker: Pod => Unit = doBasicDriverPodCheck,

‎integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkExamplesFileServerRunner.scala

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.spark.deploy.k8s.integrationtest
18+
19+
import java.net.{URI, URL}
20+
import java.nio.file.Paths
21+
import java.util.UUID
22+
23+
import io.fabric8.kubernetes.api.model.{Endpoints, Pod, Service}
24+
import org.apache.http.client.utils.URIBuilder
25+
26+
private[spark] object SparkExamplesFileServerRunner {
27+
28+
private val fileServerImage = System.getProperty(
29+
"spark.docker.test.fileServerImage", "spark-examples-file-server:latest")
30+
private val fileServerExampleJarsDir = Paths.get("docker-file-server", "jars")
31+
require(
32+
fileServerExampleJarsDir
33+
.toFile
34+
.listFiles()
35+
.exists(file => file.getName.startsWith("spark-examples")),
36+
s"No spark-examples jar found in $fileServerExampleJarsDir.")
37+
require(
38+
fileServerExampleJarsDir
39+
.toFile
40+
.listFiles()
41+
.count(file => file.getName.startsWith("spark-examples")) == 1,
42+
s"Multiple spark-examples jars found in $fileServerExampleJarsDir.")
43+
private val fileServerExampleJar = Paths.get("docker-file-server", "jars")
44+
.toFile
45+
.listFiles()
46+
.filter(file => file.getName.startsWith("spark-examples"))(0)
47+
.getName
48+
private val fileServerPodLocatorLabelKey = "fileServerLocator"
49+
private val fileServerPodLocatorLabelValue = UUID.randomUUID().toString.replaceAll("-", "")
50+
private val fileServerName = "spark-examples-file-server"
51+
52+
def launchServerAndGetUriForExamplesJar(
53+
kubernetesTestComponents: KubernetesTestComponents): URI = {
54+
val podReadinessWatcher = new SparkReadinessWatcher[Pod]
55+
Utils.tryWithResource(
56+
kubernetesTestComponents
57+
.kubernetesClient
58+
.pods()
59+
.withName(fileServerName)
60+
.watch(podReadinessWatcher)) { _ =>
61+
kubernetesTestComponents.kubernetesClient.pods().createNew()
62+
.withNewMetadata()
63+
.withName(fileServerName)
64+
.addToLabels(fileServerPodLocatorLabelKey, fileServerPodLocatorLabelValue)
65+
.endMetadata()
66+
.withNewSpec()
67+
.addNewContainer()
68+
.withName("main")
69+
.withImage(fileServerImage)
70+
.withImagePullPolicy("Never")
71+
.withNewReadinessProbe()
72+
.withNewHttpGet()
73+
.withNewPort(80)
74+
.withPath("/ping")
75+
.endHttpGet()
76+
.endReadinessProbe()
77+
.endContainer()
78+
.endSpec()
79+
.done()
80+
podReadinessWatcher.waitUntilReady()
81+
}
82+
val endpointsReadinessWatcher = new SparkReadinessWatcher[Endpoints]
83+
Utils.tryWithResource(
84+
kubernetesTestComponents
85+
.kubernetesClient
86+
.endpoints()
87+
.withName(fileServerName)
88+
.watch(endpointsReadinessWatcher)) { _ =>
89+
kubernetesTestComponents.kubernetesClient.services().createNew()
90+
.withNewMetadata()
91+
.withName(fileServerName)
92+
.endMetadata()
93+
.withNewSpec()
94+
.addToSelector(fileServerPodLocatorLabelKey, fileServerPodLocatorLabelValue)
95+
.addNewPort()
96+
.withName("file-server-port")
97+
.withNewTargetPort(80)
98+
.withPort(80)
99+
.endPort()
100+
.withType("NodePort")
101+
.endSpec()
102+
.done()
103+
endpointsReadinessWatcher.waitUntilReady()
104+
}
105+
val resolvedNodePort = kubernetesTestComponents
106+
.kubernetesClient
107+
.services()
108+
.withName(fileServerName)
109+
.get()
110+
.getSpec
111+
.getPorts
112+
.get(0)
113+
.getNodePort
114+
val masterHostname = URI.create(kubernetesTestComponents.clientConfig.getMasterUrl).getHost
115+
new URIBuilder()
116+
.setHost(masterHostname)
117+
.setPort(resolvedNodePort)
118+
.setScheme("http")
119+
.setPath(s"/$fileServerExampleJar")
120+
.build()
121+
}
122+
}

‎integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/SparkDockerImageBuilder.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ private[spark] class SparkDockerImageBuilder
4343
private val dockerHost = dockerEnv.getOrElse("DOCKER_HOST",
4444
throw new IllegalStateException("DOCKER_HOST env not found."))
4545

46+
private val FILE_SERVER_BUILD_PATH = Paths.get("docker-file-server")
47+
4648
private val originalDockerUri = URI.create(dockerHost)
4749
private val httpsDockerUri = new URIBuilder()
4850
.setHost(originalDockerUri.getHost)
@@ -68,6 +70,10 @@ private[spark] class SparkDockerImageBuilder
6870
buildImage("spark-driver", DRIVER_DOCKER_FILE)
6971
buildImage("spark-executor", EXECUTOR_DOCKER_FILE)
7072
buildImage("spark-init", INIT_CONTAINER_DOCKER_FILE)
73+
dockerClient.build(
74+
FILE_SERVER_BUILD_PATH,
75+
"spark-examples-file-server",
76+
new LoggingBuildHandler())
7177
}
7278

7379
private def buildImage(

0 commit comments

Comments
 (0)