Description

<dependency>
    <groupId>io.cdap.cdap</groupId>
    <artifactId>cdap-spark-core3_2.12</artifactId>
    <version>6.11.0</version>
</dependency>

<?xml version="1.0" encoding="UTF-8"?>
<!--
  Copyright © 2017-2021 Cask Data, Inc.

  Licensed under the Apache License, Version 2.0 (the "License"); you may not
  use this file except in compliance with the License. You may obtain a copy of
  the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  License for the specific language governing permissions and limitations under
  the License.
  -->

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <parent>
    <groupId>io.cdap.cdap</groupId>
    <artifactId>cdap</artifactId>
    <version>6.11.0</version>
  </parent>

  <artifactId>cdap-spark-core3_2.12</artifactId>
  <name>CDAP Spark3 Core Scala 2.12</name>
  <packaging>jar</packaging>

  <properties>
    <sonar.sources>
      ${project.basedir}/src/main/java,
      ${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/java,
      ${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/scala,
    </sonar.sources>
    <sonar.tests>
      ${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test/java,
    </sonar.tests>
  </properties>

  <dependencies>
    <!-- CAUTION. When adding new dependencies, pay attention to the scope and the copy-dependencies section -->
    <dependency>
      <groupId>io.cdap.cdap</groupId>
      <artifactId>cdap-api</artifactId>
      <version>${project.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>io.cdap.cdap</groupId>
      <artifactId>cdap-api-spark3_2.12</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>io.cdap.cdap</groupId>
      <artifactId>cdap-app-fabric</artifactId>
      <version>${project.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>io.cdap.cdap</groupId>
      <artifactId>cdap-spark-python</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>io.cdap.twill</groupId>
      <artifactId>twill-api</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.ow2.asm</groupId>
      <artifactId>asm-commons</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>io.cdap.http</groupId>
      <artifactId>netty-http</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.12</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.12</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.12</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-mllib_2.12</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-repl_2.12</artifactId>
    </dependency>
    <dependency>
      <groupId>javax.servlet</groupId>
      <artifactId>javax.servlet-api</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-core</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>ch.qos.logback</groupId>
      <artifactId>logback-classic</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>it.unimi.dsi</groupId>
      <artifactId>fastutil</artifactId>
      <version>6.5.6</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.tephra</groupId>
      <artifactId>tephra-core</artifactId>
      <!-- Currently comes from cdap-app-fabric -->
      <scope>provided</scope>
      <!-- Only TransactionCodec is used. Exclude unrelated dependencies -->
      <exclusions>
        <exclusion>
          <groupId>com.google.inject</groupId>
          <artifactId>guice</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.google.inject.extensions</groupId>
          <artifactId>guice-assistedinject</artifactId>
        </exclusion>
        <exclusion>
          <groupId>ch.qos.logback</groupId>
          <artifactId>logback-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>it.unimi.dsi</groupId>
          <artifactId>fastutil</artifactId>
        </exclusion>
        <exclusion>
          <groupId>io.cdap.twill</groupId>
          <artifactId>twill-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>io.cdap.twill</groupId>
          <artifactId>twill-discovery-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>io.cdap.twill</groupId>
          <artifactId>twill-zookeeper</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-server</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-minicluster</artifactId>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.mockito</groupId>
      <artifactId>mockito-core</artifactId>
    </dependency>
  </dependencies>

  <build>
    <resources>
      <resource>
        <directory>${project.basedir}/src/main/resources</directory>
      </resource>
      <resource>
        <directory>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/resources</directory>
      </resource>
    </resources>
    <testResources>
      <testResource>
        <directory>${project.basedir}/src/test/resources</directory>
      </testResource>
      <testResource>
        <directory>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test/resources</directory>
      </testResource>
    </testResources>

    <plugins>
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
        <version>3.3.1</version>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-resources-plugin</artifactId>
        <version>2.6</version>
        <executions>
          <execution>
            <id>copy-base</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>copy-resources</goal>
            </goals>
            <configuration>
              <outputDirectory>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main</outputDirectory>
              <resources>
                <resource>
                  <directory>${project.parent.basedir}/cdap-spark-core-base/src/main</directory>
                </resource>
              </resources>
            </configuration>
          </execution>
          <execution>
            <id>copy-base-test</id>
            <phase>generate-test-sources</phase>
            <goals>
              <goal>copy-resources</goal>
            </goals>
            <configuration>
              <outputDirectory>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test</outputDirectory>
              <resources>
                <resource>
                  <directory>${project.parent.basedir}/cdap-spark-core-base/src/test</directory>
                </resource>
              </resources>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>build-helper-maven-plugin</artifactId>
        <version>3.0.0</version>
        <executions>
          <execution>
            <id>add-source</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>add-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/java</source>
                <source>${project.build.directory}/generated-sources/cdap-spark-core-base/src/main/scala</source>
              </sources>
            </configuration>
          </execution>
          <execution>
            <id>add-test-source</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>add-test-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test/java</source>
                <source>${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test/scala</source>
              </sources>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>

  <profiles>
    <profile>
      <!-- This profile is just for Spark development in IDE, don't use it in normal Maven build -->
      <id>spark-dev</id>
      <properties>
        <spark.base.dir>${project.basedir}/../cdap-spark-core-base</spark.base.dir>
      </properties>
      <build>
        <plugins>
          <!-- Disable the copying of files from base -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-resources-plugin</artifactId>
            <version>2.6</version>
            <executions>
              <execution>
                <id>copy-base</id>
                <phase>none</phase>
              </execution>
              <execution>
                <id>copy-base-test</id>
                <phase>none</phase>
              </execution>
            </executions>
          </plugin>

          <!-- Symlink the source from base -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-antrun-plugin</artifactId>
            <version>1.7</version>
            <configuration>
              <skip>true</skip>
            </configuration>
            <executions>
              <execution>
                <id>mkdir-symlink-dir</id>
                <phase>generate-sources</phase>
                <goals>
                  <goal>run</goal>
                </goals>
                <configuration combine.self="override">
                  <skip>false</skip>
                  <target>
                    <mkdir dir="${project.build.directory}/generated-sources/cdap-spark-core-base/src"/>
                    <mkdir dir="${project.build.directory}/generated-test-sources/cdap-spark-core-base/src"/>
                    <symlink link="${project.build.directory}/generated-sources/cdap-spark-core-base/src/main" resource="${spark.base.dir}/src/main" overwrite="true" />
                    <symlink link="${project.build.directory}/generated-test-sources/cdap-spark-core-base/src/test" resource="${spark.base.dir}/src/test" overwrite="true" />
                  </target>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>

    <profile>
      <id>dist</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-jar-plugin</artifactId>
            <version>2.4</version>
            <executions>
              <execution>
                <id>jar</id>
                <phase>prepare-package</phase>
                <configuration combine.self="override">
                  <outputDirectory>${project.build.directory}/libexec</outputDirectory>
                  <finalName>${project.groupId}.${project.build.finalName}</finalName>
                </configuration>
                <goals>
                  <goal>jar</goal>
                </goals>
              </execution>
              <execution>
                <id>jar-k8s</id>
                <phase>prepare-package</phase>
                <configuration combine.self="override">
                  <outputDirectory>${project.build.directory}/k8s</outputDirectory>
                  <finalName>${project.groupId}.${project.build.finalName}</finalName>
                </configuration>
                <goals>
                  <goal>jar</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <!-- Copy cdap-api-spark -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-dependency-plugin</artifactId>
            <version>2.8</version>
            <executions>
              <execution>
                <id>copy-dependencies</id>
                <phase>prepare-package</phase>
                <goals>
                  <goal>copy-dependencies</goal>
                </goals>
                <configuration combine.self="override">
                  <outputDirectory>${project.build.directory}/libexec</outputDirectory>
                  <overWriteReleases>false</overWriteReleases>
                  <overWriteSnapshots>false</overWriteSnapshots>
                  <overWriteIfNewer>true</overWriteIfNewer>
                  <includeArtifactIds>cdap-api-spark3_2.12,cdap-spark-python</includeArtifactIds>
                  <prependGroupId>true</prependGroupId>
                  <silent>true</silent>
                  <includeScope>runtime</includeScope>
                </configuration>
              </execution>
              <!--
                copy dependencies needed to run cdap-spark-core,
                excluding spark, hadoop, and some other classes that should come from
                the Spark classpath.
                -->
              <execution>
                <id>copy-dependencies-k8s</id>
                <phase>prepare-package</phase>
                <goals>
                  <goal>copy-dependencies</goal>
                </goals>
                <configuration combine.self="override">
                  <outputDirectory>${project.build.directory}/k8s/lib</outputDirectory>
                  <overWriteReleases>false</overWriteReleases>
                  <overWriteSnapshots>false</overWriteSnapshots>
                  <overWriteIfNewer>true</overWriteIfNewer>
                  <prependGroupId>true</prependGroupId>
                  <silent>true</silent>
                  <includeScope>compile</includeScope>
                  <excludeGroupIds>
                    org.apache.commons,
                    org.apache.spark,
                    org.spark-project.spark,
                    org.apache.hadoop,
                    org.apache.parquet,
                    org.apache.avro,
                    org.scala-lang,
                    org.scala-lang.modules,
                    org.scalanlp,
                    com.fasterxml.jackson.module,
                    com.fasterxml.jackson.core,
                    org.codehaus.jackson
                  </excludeGroupIds>
                </configuration>
              </execution>
            </executions>
          </plugin>
          <!-- Build the Spark assembly jar -->
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.2.1</version>
            <executions>
              <execution>
                <phase>prepare-package</phase>
                <goals>
                  <goal>shade</goal>
                </goals>
                <configuration>
                  <artifactSet>
                    <excludes>
                      <exclude>io.cdap.cdap:*</exclude>
                    </excludes>
                  </artifactSet>
                  <createDependencyReducedPom>false</createDependencyReducedPom>
                  <filters>
                    <filter>
                      <!-- Excludes all files related to JAR signature -->
                      <artifact>*:*</artifact>
                      <excludes>
                        <exclude>META-INF/*.SF</exclude>
                        <exclude>META-INF/*.DSA</exclude>
                        <exclude>META-INF/*.RSA</exclude>
                      </excludes>
                    </filter>
                  </filters>
                  <outputFile>${project.build.directory}/libexec/${project.groupId}.spark-assembly-${spark3.version}.jar</outputFile>
                  <transformers>
                    <!-- Needed to merge data source service loaders, otherwise 'libsvm' format doesn't work -->
                    <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                      <resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
                    </transformer>
                    <!-- Needed to merge akka config files -->
                    <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                      <resource>reference.conf</resource>
                    </transformer>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                      <manifestEntries>
                        <Main-Class>akka.Main</Main-Class>
                      </manifestEntries>
                    </transformer>
                  </transformers>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

</project>