cdap-spark-core

pkg:maven/co.cask.cdap/cdap-spark-core@5.1.2

Used in: 3 components

Overview
Overview

Versions
Versions

Dependents
Dependents

Dependencies
Dependencies

Overview

Description

Data Application Platform for Hadoop

Snippets

<dependency>
    <groupId>co.cask.cdap</groupId>
    <artifactId>cdap-spark-core</artifactId>
    <version>5.1.2</version>
</dependency>

Maven POM File

<?xml version="1.0" encoding="UTF-8"?>
<!--
  Copyright © 2016 Cask Data, Inc.

  Licensed under the Apache License, Version 2.0 (the "License"); you may not
  use this file except in compliance with the License. You may obtain a copy of
  the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  License for the specific language governing permissions and limitations under
  the License.
  -->

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <parent>
    <groupId>co.cask.cdap</groupId>
    <artifactId>cdap</artifactId>
    <version>5.1.2</version>
  </parent>

  <artifactId>cdap-spark-core</artifactId>
  <name>CDAP Spark1 Core Scala 2.10</name>
  <packaging>jar</packaging>

  <dependencies>
    <!-- CAUTION. When adding new dependencies, pay attention to the scope and the copy-dependencies section -->
    <dependency>
      <groupId>co.cask.cdap</groupId>
      <artifactId>cdap-api</artifactId>
      <version>${project.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>co.cask.cdap</groupId>
      <artifactId>cdap-api-spark</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>co.cask.cdap</groupId>
      <artifactId>cdap-app-fabric</artifactId>
      <version>${project.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>co.cask.cdap</groupId>
      <artifactId>cdap-spark-python</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.twill</groupId>
      <artifactId>twill-api</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.ow2.asm</groupId>
      <artifactId>asm-all</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>co.cask.http</groupId>
      <artifactId>netty-http</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.10</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.10</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.10</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-mllib_2.10</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-repl_2.10</artifactId>
    </dependency>
    <dependency>
      <groupId>javax.servlet</groupId>
      <artifactId>javax.servlet-api</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-core</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.tephra</groupId>
      <artifactId>tephra-core</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
      <!-- Only TransactionCodec is used. Exclude unrelated dependencies -->
      <exclusions>
        <exclusion>
          <groupId>com.google.inject</groupId>
          <artifactId>guice</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.google.inject.extensions</groupId>
          <artifactId>guice-assistedinject</artifactId>
        </exclusion>
        <exclusion>
          <groupId>ch.qos.logback</groupId>
          <artifactId>logback-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>it.unimi.dsi</groupId>
          <artifactId>fastutil</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.twill</groupId>
          <artifactId>twill-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.twill</groupId>
          <artifactId>twill-discovery-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.twill</groupId>
          <artifactId>twill-zookeeper</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-server</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>ch.qos.logback</groupId>
      <artifactId>logback-classic</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>it.unimi.dsi</groupId>
      <artifactId>fastutil</artifactId>
      <version>6.5.6</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-minicluster</artifactId>
    </dependency>
  </dependencies>

  <build>
    <resources>
      <resource>
        <directory>${project.basedir}/src/main/resources</directory>
      </resource>
      <resource>
        <directory>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/resources</directory>
      </resource>
    </resources>
    <testResources>
      <testResource>
        <directory>${project.basedir}/src/test/resources</directory>
      </testResource>
      <testResource>
        <directory>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test/resources</directory>
      </testResource>
    </testResources>

    <plugins>
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
        <version>3.3.1</version>
      </plugin>
      <!-- copy the source from cdap-spark-core-base in order to compile everything as scala 2.10 -->
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-resources-plugin</artifactId>
        <version>2.6</version>
        <executions>
          <execution>
            <id>copy-base</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>copy-resources</goal>
            </goals>
            <configuration>
              <outputDirectory>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main</outputDirectory>
              <resources>
                <resource>
                  <directory>${project.parent.basedir}/cdap-spark-core-base/src/main</directory>
                </resource>
              </resources>
            </configuration>
          </execution>
          <execution>
            <id>copy-base-test</id>
            <phase>generate-test-sources</phase>
            <goals>
              <goal>copy-resources</goal>
            </goals>
            <configuration>
              <outputDirectory>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test</outputDirectory>
              <resources>
                <resource>
                  <directory>${project.parent.basedir}/cdap-spark-core-base/src/test</directory>
                </resource>
              </resources>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>build-helper-maven-plugin</artifactId>
        <version>3.0.0</version>
        <executions>
          <execution>
            <id>add-source</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>add-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/java</source>
                <source>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/scala</source>
              </sources>
            </configuration>
          </execution>
          <execution>
            <id>add-test-source</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>add-test-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test/java</source>
                <source>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test/scala</source>
              </sources>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>

  <profiles>
    <profile>
      <!-- This profile is just for Spark development in IDE, don't use it in normal Maven build -->
      <id>spark1-dev</id>
      <properties>
        <spark.base.dir>${project.basedir}/../cdap-spark-core-base</spark.base.dir>
      </properties>
      <build>
        <plugins>
          <!-- Disable the copying of files from base -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-resources-plugin</artifactId>
            <version>2.6</version>
            <executions>
              <execution>
                <id>copy-base</id>
                <phase>none</phase>
              </execution>
              <execution>
                <id>copy-base-test</id>
                <phase>none</phase>
              </execution>
            </executions>
          </plugin>

          <!-- Symlink the source from base -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-antrun-plugin</artifactId>
            <version>1.7</version>
            <configuration>
              <skip>true</skip>
            </configuration>
            <executions>
              <execution>
                <id>mkdir-symlink-dir</id>
                <phase>generate-sources</phase>
                <goals>
                  <goal>run</goal>
                </goals>
                <configuration combine.self="override">
                  <skip>false</skip>
                  <target>
                    <mkdir dir="${project.build.directory}/generated-sources/cdap-spark-core-base/src"/>
                    <mkdir dir="${project.build.directory}/generated-test-sources/cdap-spark-core-base/src"/>
                    <symlink link="${project.build.directory}/generated-sources/cdap-spark-core-base/src/main" resource="${spark.base.dir}/src/main" overwrite="true" />
                    <symlink link="${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test" resource="${spark.base.dir}/src/test" overwrite="true" />
                  </target>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>

    <profile>
      <id>dist</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-jar-plugin</artifactId>
            <version>2.4</version>
            <executions>
              <execution>
                <id>jar</id>
                <phase>prepare-package</phase>
                <configuration combine.self="override">
                  <outputDirectory>${project.build.directory}/libexec</outputDirectory>
                  <finalName>${project.groupId}.${project.build.finalName}</finalName>
                </configuration>
                <goals>
                  <goal>jar</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <!-- Copy cdap-api-spark -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-dependency-plugin</artifactId>
            <version>2.8</version>
            <executions>
              <execution>
                <id>copy-dependencies</id>
                <phase>prepare-package</phase>
                <goals>
                  <goal>copy-dependencies</goal>
                </goals>
                <configuration combine.self="override">
                  <outputDirectory>${project.build.directory}/libexec</outputDirectory>
                  <overWriteReleases>false</overWriteReleases>
                  <overWriteSnapshots>false</overWriteSnapshots>
                  <overWriteIfNewer>true</overWriteIfNewer>
                  <includeArtifactIds>cdap-api-spark,cdap-spark-python</includeArtifactIds>
                  <prependGroupId>true</prependGroupId>
                  <silent>true</silent>
                  <includeScope>runtime</includeScope>
                </configuration>
              </execution>
            </executions>
          </plugin>
          <!-- Build the Spark assembly jar -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>2.4.3</version>
            <executions>
              <execution>
                <phase>prepare-package</phase>
                <goals>
                  <goal>shade</goal>
                </goals>
                <configuration>
                  <artifactSet>
                    <excludes>
                      <exclude>co.cask.cdap:*</exclude>
                    </excludes>
                  </artifactSet>
                  <createDependencyReducedPom>false</createDependencyReducedPom>
                  <filters>
                    <filter>
                      <!-- Excludes all files related to JAR signature -->
                      <artifact>*:*</artifact>
                      <excludes>
                        <exclude>META-INF/*.SF</exclude>
                        <exclude>META-INF/*.DSA</exclude>
                        <exclude>META-INF/*.RSA</exclude>
                      </excludes>
                    </filter>
                  </filters>
                  <outputFile>${project.build.directory}/libexec/${project.groupId}.spark-assembly-${spark1.version}.jar</outputFile>
                  <transformers>
                    <!-- Needed to merge data source service loaders, otherwise 'libsvm' format doesn't work -->
                    <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                      <resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
                    </transformer>
                    <!-- Needed to merge akka config files -->
                    <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                      <resource>reference.conf</resource>
                    </transformer>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                      <manifestEntries>
                        <Main-Class>akka.Main</Main-Class>
                      </manifestEntries>
                    </transformer>
                  </transformers>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

</project>

Metadata

Age:7 years ago

Licenses:

The Apache Software License, Version 2.0

Organization

co.cask.cdap

External Resources

Project URL

Source Control

Known Contributors

Cask Data

OSS Index

View