Skip to content

Commit 0d3f5cb

Browse files
robert3005Robert Kruszewski
robert3005
authored and
Robert Kruszewski
committed
Update build docker images (apache-spark-on-k8s#458)
1 parent 77c70df commit 0d3f5cb

File tree

7 files changed

+25
-29
lines changed

7 files changed

+25
-29
lines changed

‎.circleci/config.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: 2
22

33
defaults: &defaults
44
docker:
5-
- image: palantirtechnologies/circle-spark-base
5+
- image: palantirtechnologies/circle-spark-base:0.1.0
66
resource_class: xlarge
77
environment: &defaults-environment
88
TERM: dumb
@@ -128,7 +128,7 @@ jobs:
128128
<<: *defaults
129129
# Some part of the maven setup fails if there's no R, so we need to use the R image here
130130
docker:
131-
- image: palantirtechnologies/circle-spark-r
131+
- image: palantirtechnologies/circle-spark-r:0.1.0
132132
steps:
133133
# Saves us from recompiling every time...
134134
- restore_cache:
@@ -300,7 +300,7 @@ jobs:
300300
# depends on build-sbt, but we only need the assembly jars
301301
<<: *defaults
302302
docker:
303-
- image: palantirtechnologies/circle-spark-python
303+
- image: palantirtechnologies/circle-spark-python:0.1.0
304304
parallelism: 2
305305
steps:
306306
- *checkout-code
@@ -325,7 +325,7 @@ jobs:
325325
# depends on build-sbt, but we only need the assembly jars
326326
<<: *defaults
327327
docker:
328-
- image: palantirtechnologies/circle-spark-r
328+
- image: palantirtechnologies/circle-spark-r:0.1.0
329329
steps:
330330
- *checkout-code
331331
- attach_workspace:
@@ -438,7 +438,7 @@ jobs:
438438
<<: *defaults
439439
# Some part of the maven setup fails if there's no R, so we need to use the R image here
440440
docker:
441-
- image: palantirtechnologies/circle-spark-r
441+
- image: palantirtechnologies/circle-spark-r:0.1.0
442442
steps:
443443
- *checkout-code
444444
- restore_cache:
@@ -458,7 +458,7 @@ jobs:
458458
deploy-gradle:
459459
<<: *defaults
460460
docker:
461-
- image: palantirtechnologies/circle-spark-r
461+
- image: palantirtechnologies/circle-spark-r:0.1.0
462462
steps:
463463
- *checkout-code
464464
- *restore-gradle-wrapper-cache
@@ -470,7 +470,7 @@ jobs:
470470
<<: *defaults
471471
# Some part of the maven setup fails if there's no R, so we need to use the R image here
472472
docker:
473-
- image: palantirtechnologies/circle-spark-r
473+
- image: palantirtechnologies/circle-spark-r:0.1.0
474474
steps:
475475
# This cache contains the whole project after version was set and mvn package was called
476476
# Restoring first (and instead of checkout) as mvn versions:set mutates real source code...

‎dev/docker-images/Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717

1818
.PHONY: all publish base python r
1919

20-
BASE_IMAGE_NAME = palantirtechnologies/circle-spark-base
21-
PYTHON_IMAGE_NAME = palantirtechnologies/circle-spark-python
22-
R_IMAGE_NAME = palantirtechnologies/circle-spark-r
20+
BASE_IMAGE_NAME = palantirtechnologies/circle-spark-base:0.1.0
21+
PYTHON_IMAGE_NAME = palantirtechnologies/circle-spark-python:0.1.0
22+
R_IMAGE_NAME = palantirtechnologies/circle-spark-r:0.1.0
2323

2424
all: base python r
2525

‎dev/docker-images/base/Dockerfile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# limitations under the License.
1616
#
1717

18-
FROM buildpack-deps:xenial
18+
FROM buildpack-deps:cosmic
1919

2020
# make Apt non-interactive
2121
RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \
@@ -107,10 +107,11 @@ WORKDIR $CIRCLE_HOME
107107
# Install miniconda, we are using it to test conda support and a bunch of tests expect CONDA_BIN to be set
108108
ENV CONDA_ROOT=$CIRCLE_HOME/miniconda
109109
ENV CONDA_BIN=$CIRCLE_HOME/miniconda/bin/conda
110-
ENV MINICONDA2_VERSION=4.3.31
110+
ENV MINICONDA2_VERSION=4.5.11
111111
RUN curl -sO https://repo.continuum.io/miniconda/Miniconda2-${MINICONDA2_VERSION}-Linux-x86_64.sh \
112112
&& bash Miniconda2-${MINICONDA2_VERSION}-Linux-x86_64.sh -b -p ${CONDA_ROOT} \
113113
&& $CONDA_BIN clean --all \
114+
&& sudo mkdir -m 777 /home/.conda \
114115
&& rm -f Miniconda2-${MINICONDA2_VERSION}-Linux-x86_64.sh
115116

116117
# END IMAGE CUSTOMIZATIONS

‎dev/docker-images/python/Dockerfile

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,14 @@ FROM palantirtechnologies/circle-spark-base
2121
ENV PATH="$CIRCLE_HOME/.pyenv/bin:$PATH"
2222
RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \
2323
&& cat >>.bashrc <<<'eval "$($HOME/.pyenv/bin/pyenv init -)"' \
24-
&& cat >>.bashrc <<<'eval "$($HOME/.pyenv/bin/pyenv virtualenv-init -)"' \
25-
&& pyenv doctor
24+
&& cat >>.bashrc <<<'eval "$($HOME/.pyenv/bin/pyenv virtualenv-init -)"'
2625

2726
# Must install numpy 1.11 or else a bunch of tests break due to different output formatting on e.g. nparray
2827
# A version I've tested earlier that I know it breaks with is 1.14.1
2928
RUN mkdir -p $(pyenv root)/versions \
3029
&& ln -s $CONDA_ROOT $(pyenv root)/versions/our-miniconda \
31-
&& $CONDA_BIN create -y -n python2 -c anaconda -c conda-forge python==2.7.11 numpy=1.11.2 pyarrow==0.8.0 pandas nomkl \
32-
&& $CONDA_BIN create -y -n python3 -c anaconda -c conda-forge python=3.6 numpy=1.11.2 pyarrow==0.8.0 pandas nomkl \
30+
&& $CONDA_BIN create -y -n python2 -c anaconda -c conda-forge python==2.7.15 numpy=1.14.0 pyarrow==0.8.0 pandas nomkl \
31+
&& $CONDA_BIN create -y -n python3 -c anaconda -c conda-forge python=3.6 numpy=1.14.0 pyarrow==0.8.0 pandas nomkl \
3332
&& $CONDA_BIN clean --all
3433

3534
RUN pyenv global our-miniconda/envs/python2 our-miniconda/envs/python3 \

‎dev/docker-images/r/Dockerfile

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,8 @@ FROM palantirtechnologies/circle-spark-base
2020
USER root
2121

2222
### Install R
23-
RUN echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list \
24-
&& apt-get update \
25-
&& gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 \
26-
&& gpg -a --export E084DAB9 | sudo apt-key add - \
27-
&& apt-get update \
28-
&& apt-get --assume-yes install r-base r-base-dev qpdf \
23+
RUN apt-get update \
24+
&& apt-get install r-base r-base-dev qpdf \
2925
&& rm -rf /var/lib/apt/lists/* \
3026
&& chmod 777 /usr/local/lib/R/site-library \
3127
&& /usr/lib/R/bin/R -e "install.packages(c('devtools'), repos='http://cran.us.r-project.org', lib='/usr/local/lib/R/site-library'); devtools::install_github('r-lib/[email protected]', lib='/usr/local/lib/R/site-library'); install.packages(c('knitr', 'rmarkdown', 'e1071', 'survival', 'roxygen2', 'lintr'), repos='http://cran.us.r-project.org', lib='/usr/local/lib/R/site-library')"

‎python/pyspark/tests.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2279,20 +2279,20 @@ def test_conda(self):
22792279
|from pyspark import SparkContext
22802280
|
22812281
|sc = SparkContext()
2282-
|sc.addCondaPackages('numpy=1.11.1')
2282+
|sc.addCondaPackages('numpy=1.14.0')
22832283
|
22842284
|# Ensure numpy is accessible on the driver
22852285
|import numpy
22862286
|arr = [1, 2, 3]
22872287
|def mul2(x):
22882288
| # Also ensure numpy accessible from executor
2289-
| assert numpy.version.version == "1.11.1"
2289+
| assert numpy.version.version == "1.14.0"
22902290
| return x * 2
22912291
|print(sc.parallelize(arr).map(mul2).collect())
22922292
""")
22932293
props = self.createTempFile("properties", """
22942294
|spark.conda.binaryPath {}
2295-
|spark.conda.channelUrls https://repo.continuum.io/pkgs/free
2295+
|spark.conda.channelUrls https://repo.continuum.io/pkgs/main
22962296
|spark.conda.bootstrapPackages python=3.5
22972297
""".format(os.environ["CONDA_BIN"]))
22982298
env = dict(os.environ)

‎resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,14 +84,14 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
8484
| exit(-1)
8585
| sc = SparkContext(conf=SparkConf())
8686
|
87-
| sc.addCondaPackages('numpy=1.11.1')
87+
| sc.addCondaPackages('numpy=1.14.0')
8888
| import numpy
8989
|
9090
| status = open(sys.argv[1],'w')
9191
|
9292
| # Addict exists only in external-conda-forge, not anaconda
9393
| sc.addCondaChannel("https://conda.anaconda.org/conda-forge")
94-
| sc.addCondaPackages('addict=1.0.0')
94+
| sc.addCondaPackages('addict=2.2.0')
9595
|
9696
| def numpy_multiply(x):
9797
| # Ensure package from non-base channel is installed
@@ -376,8 +376,8 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
376376

377377
val extraConf: Map[String, String] = Map(
378378
"spark.conda.binaryPath" -> sys.env("CONDA_BIN"),
379-
"spark.conda.channelUrls" -> "https://repo.continuum.io/pkgs/free",
380-
"spark.conda.bootstrapPackages" -> "python=3.5"
379+
"spark.conda.channelUrls" -> "https://repo.continuum.io/pkgs/main",
380+
"spark.conda.bootstrapPackages" -> "python=3.6"
381381
)
382382

383383
val moduleDir =

0 commit comments

Comments
 (0)