SparkLE/docker/hibench-base/Dockerfile

FROM ubuntu:18.04

RUN apt -y update && apt-get -y update

RUN apt -y install git bc 

RUN git clone https://github.com/Intel-bigdata/HiBench.git

#RUN apt-get -y install maven openjdk-8-jre-headless scala openjdk-8-jdk vim wget python2.7 software-properties-common
RUN apt-get -y install maven scala vim wget python2.7 software-properties-common
RUN wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add -
RUN add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/
RUN apt update
RUN apt install adoptopenjdk-8-hotspot
ENV JAVA_HOME=/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64


WORKDIR /HiBench
RUN mvn -Psparkbench -Dspark=3.0 -Dscala=2.12 -Dhadoop=2.7 clean package
ENV HIBENCH_HOME=/HiBench

WORKDIR /usr/bin
RUN ln -s python2.7 python2
RUN ln -s python2.7 python

WORKDIR /usr/local

RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.4/hadoop-2.7.4.tar.gz
RUN tar xzvf hadoop-2.7.4.tar.gz
RUN mv hadoop-2.7.4 hadoop
ENV HADOOP_HOME=/usr/local/hadoop
ENV HADOOP_INSTALL=/usr/local/hadoop
ENV HADOOP_PREFIX /usr/local/hadoop
ENV HADOOP_COMMON_HOME /usr/local/hadoop
ENV HADOOP_HDFS_HOME /usr/local/hadoop
ENV HADOOP_MAPRED_HOME /usr/local/hadoop
ENV HADOOP_YARN_HOME /usr/local/hadoop
ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
ENV HADOOP_EXAMPLES_JAR=/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar
#ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
ENV PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin
RUN rm ${HADOOP_CONF_DIR}/core-site.xml


WORKDIR /
RUN wget https://ftp.cc.uoc.gr/mirrors/apache/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz
RUN tar xvf spark-3.0.1-bin-hadoop2.7.tgz
RUN mv spark-3.0.1-bin-hadoop2.7 spark
ENV SPARK_HOME=/spark
RUN mkdir ~/.kube

COPY spark-exec.template.yaml /template.yaml

WORKDIR /HiBench/