spark-network-yarn_2.13

pkg:maven/org.apache.spark/spark-network-yarn_2.13@4.0.0

Used in: 1 component

Overview
Overview

Versions
Versions

Dependents
Dependents

Dependencies
Dependencies

Overview

Description

The Apache Software Foundation provides support for the Apache community of open-source software projects. The Apache projects are characterized by a collaborative, consensus based development process, an open and pragmatic software license, and a desire to create high quality software that leads the way in its field. We consider ourselves not simply a group of projects sharing a server, but rather a community of developers and users.

Snippets

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-network-yarn_2.13</artifactId>
    <version>4.0.0</version>
</dependency>

Maven POM File

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <parent>
    <artifactId>spark-parent_2.13</artifactId>
    <groupId>org.apache.spark</groupId>
    <version>4.0.0</version>
    <relativePath>../../pom.xml</relativePath>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>spark-network-yarn_2.13</artifactId>
  <name>Spark Project YARN Shuffle Service</name>
  <url>https://spark.apache.org/</url>
  <build>
    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
    <plugins>
      <plugin>
        <artifactId>maven-shade-plugin</artifactId>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>
              <transformers>
                <transformer />
              </transformers>
            </configuration>
          </execution>
        </executions>
        <configuration>
          <shadedArtifactAttached>false</shadedArtifactAttached>
          <outputFile>${shuffle.jar}</outputFile>
          <artifactSet>
            <includes>
              <include>*:*</include>
            </includes>
            <excludes>
              <exclude>org.scala-lang:scala-library</exclude>
            </excludes>
          </artifactSet>
          <filters>
            <filter>
              <artifact>*:*</artifact>
              <excludes>
                <exclude>META-INF/*.SF</exclude>
                <exclude>META-INF/*.DSA</exclude>
                <exclude>META-INF/*.RSA</exclude>
                <exclude>META-INF/LICENSE</exclude>
              </excludes>
            </filter>
          </filters>
          <relocations>
            <relocation>
              <pattern>com.fasterxml.jackson</pattern>
              <shadedPattern>${spark.shade.packageName}.com.fasterxml.jackson</shadedPattern>
              <includes>
                <include>com.fasterxml.jackson.**</include>
              </includes>
            </relocation>
            <relocation>
              <pattern>io.netty</pattern>
              <shadedPattern>${spark.shade.packageName}.io.netty</shadedPattern>
              <includes>
                <include>io.netty.**</include>
              </includes>
            </relocation>
          </relocations>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>build-helper-maven-plugin</artifactId>
        <executions>
          <execution>
            <id>regex-property</id>
            <goals>
              <goal>regex-property</goal>
            </goals>
            <configuration>
              <name>spark.shade.native.packageName</name>
              <value>${spark.shade.packageName}</value>
              <regex>\.</regex>
              <replacement>_</replacement>
              <failIfNoMatch>true</failIfNoMatch>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <artifactId>maven-antrun-plugin</artifactId>
        <executions>
          <execution>
            <id>unpack</id>
            <phase>package</phase>
            <goals>
              <goal>run</goal>
            </goals>
            <configuration>
              <target>
                <echo />
                <unzip />
                <move />
                <move />
                <move />
                <move />
                <move />
                <move />
                <move />
                <move />
                <move />
                <jar />
              </target>
            </configuration>
          </execution>
          <execution>
            <phase>verify</phase>
            <goals>
              <goal>run</goal>
            </goals>
            <configuration>
              <target>
                <macrodef>
                  <attribute />
                  <sequential>
                    <fail>
                      <condition>
                        <not>
                          <resourceexists>
                            <zipentry />
                          </resourceexists>
                        </not>
                      </condition>
                    </fail>
                  </sequential>
                </macrodef>
                <echo>Verifying dependency shading</echo>
                <shaded />
                <shaded />
                <shaded />
              </target>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>
  <dependencies>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-tags_2.13</artifactId>
      <version>4.0.0</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-tags_2.13</artifactId>
      <version>4.0.0</version>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client-api</artifactId>
      <version>3.4.1</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client-runtime</artifactId>
      <version>3.4.1</version>
      <scope>provided</scope>
      <exclusions>
        <exclusion>
          <artifactId>commons-logging</artifactId>
          <groupId>commons-logging</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <version>33.4.0-jre</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
      <version>2.0.16</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_2.13</artifactId>
      <version>3.2.19</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-featurespec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-flatspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-freespec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-funsuite_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-funspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-propspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-refspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-wordspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-diagrams_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-matchers-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-shouldmatchers_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-mustmatchers_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scala-reflect</artifactId>
          <groupId>org.scala-lang</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>scalacheck-1-18_2.13</artifactId>
      <version>3.2.19.0</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>scalacheck_2.13</artifactId>
          <groupId>org.scalacheck</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>mockito-5-12_2.13</artifactId>
      <version>3.2.19.0</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>mockito-core</artifactId>
          <groupId>org.mockito</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>selenium-4-21_2.13</artifactId>
      <version>3.2.19.0</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>htmlunit-driver</artifactId>
          <groupId>org.seleniumhq.selenium</groupId>
        </exclusion>
        <exclusion>
          <artifactId>selenium-java</artifactId>
          <groupId>org.seleniumhq.selenium</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.junit.jupiter</groupId>
      <artifactId>junit-jupiter</artifactId>
      <version>5.11.4</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>junit-jupiter-api</artifactId>
          <groupId>org.junit.jupiter</groupId>
        </exclusion>
        <exclusion>
          <artifactId>junit-jupiter-params</artifactId>
          <groupId>org.junit.jupiter</groupId>
        </exclusion>
        <exclusion>
          <artifactId>junit-jupiter-engine</artifactId>
          <groupId>org.junit.jupiter</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>com.github.sbt.junit</groupId>
      <artifactId>jupiter-interface</artifactId>
      <version>0.13.3</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>junit-platform-launcher</artifactId>
          <groupId>org.junit.platform</groupId>
        </exclusion>
        <exclusion>
          <artifactId>test-interface</artifactId>
          <groupId>org.scala-sbt</groupId>
        </exclusion>
        <exclusion>
          <artifactId>junit-jupiter-engine</artifactId>
          <groupId>org.junit.jupiter</groupId>
        </exclusion>
      </exclusions>
    </dependency>
  </dependencies>
  <properties>
    <sbt.project.name>network-yarn</sbt.project.name>
    <shade>org/sparkproject/</shade>
    <hadoop.deps.scope>provided</hadoop.deps.scope>
    <shuffle.jar>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</shuffle.jar>
  </properties>
</project>

Metadata

Age:6 months ago

Licenses:

Apache-2.0

Organization

org.apache.spark

External Resources

Project URL

Issue Tracker URL

Source Control

Known Contributors

No contributors found

OSS Index

View