flink-stream-lander

pkg:maven/com.github.lalilou/flink-stream-lander@0.0.4

Used in: 0 components

Overview
Overview

Versions
Versions

Dependents
Dependents

Dependencies
Dependencies

Overview

Description

A base project for landing streams to datalake with next-to-no-coding required.

<dependency>
    <groupId>com.github.lalilou</groupId>
    <artifactId>flink-stream-lander</artifactId>
    <version>0.0.4</version>
</dependency>

<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at
  http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>com.github.lalilou</groupId>
	<artifactId>flink-stream-lander</artifactId>
	<version>0.0.4</version>
	<packaging>jar</packaging>

	<name>l3l-flink-lander</name>
	<description>A base project for landing streams to datalake with next-to-no-coding required.</description>
	<url>http://www.l3l.com.au</url>

	<licenses>
		<license>
			<name>The Apache License, Version 2.0</name>
			<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
		</license>
	</licenses>

	<developers>
		<developer>
			<name>L3L</name>
			<email>lamlinloup@gmail.com</email>
			<organization>L3L</organization>
			<organizationUrl>http://www.l3l.com.au</organizationUrl>
		</developer>
	</developers>

	<scm>
		<connection>scm:git:git@github.com/lalilou/flink-stream-lander.git</connection>
		<developerConnection>scm:git:git@github.com:lalilou/flink-stream-lander.git</developerConnection>
		<url>git@github.com:lalilou/flink-stream-lander.git</url>
		<tag>flink-stream-lander-0.0.2</tag>
	</scm>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<flink.version>1.10.1</flink.version>
		<scala.binary.version>2.11</scala.binary.version>
		<scala.version>2.11.12</scala.version>
		<hadoop.version>2.8.5</hadoop.version>
		<kda.version>1.1.0</kda.version>
		<lib.scope>provided</lib.scope>
		<aws.sdk.version>1.11.297</aws.sdk.version>
		<aws.sdk2.version>2.11.12</aws.sdk2.version>
		<internal.url>https://oss.sonatype.org/service/local/staging/deploy/maven2</internal.url>
		<snapshot.url>https://oss.sonatype.org/content/repositories/snapshots</snapshot.url>
	</properties>

	<dependencies>
		<!-- Apache Flink dependencies -->
		<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-scala_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
			<scope>${lib.scope}</scope>
			<exclusions>
				<exclusion>
					<groupId>log4j</groupId>
					<artifactId>*</artifactId>
				</exclusion>
				<exclusion>
					<groupId>org.slf4j</groupId>
					<artifactId>slf4j-log4j12</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
			<scope>${lib.scope}</scope>
			<exclusions>
				<exclusion>
					<groupId>log4j</groupId>
					<artifactId>*</artifactId>
				</exclusion>
				<exclusion>
					<groupId>org.slf4j</groupId>
					<artifactId>slf4j-log4j12</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-statebackend-rocksdb_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
		</dependency>

		<!-- Connectors -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-json</artifactId>
			<version>${flink.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-avro</artifactId>
			<version>${flink.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-connector-kinesis_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
		</dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch6_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

		<!-- AWS SDK -->
		<dependency>
			<groupId>software.amazon.awssdk</groupId>
			<artifactId>secretsmanager</artifactId>
			<version>${aws.sdk2.version}</version>
		</dependency>

		<!-- Library for running as a Kinesis Analytics job -->
		<dependency>
			<groupId>com.amazonaws</groupId>
			<artifactId>aws-kinesisanalytics-runtime</artifactId>
			<version>${kda.version}</version>
		</dependency>
		<dependency>
			<groupId>com.amazonaws</groupId>
			<artifactId>aws-java-sdk-logs</artifactId>
			<version>${aws.sdk.version}</version>
		</dependency>

		<!-- For parsing JSON files -->
		<dependency>
			<groupId>org.json4s</groupId>
			<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
			<version>3.6.7</version>
		</dependency>
		<!-- For parsing the config files -->
		<dependency>
			<groupId>com.github.pureconfig</groupId>
			<artifactId>pureconfig_${scala.binary.version}</artifactId>
			<version>0.12.3</version>
		</dependency>
		<!-- For Avro deserialization -->
		<dependency>
			<groupId>org.apache.avro</groupId>
			<artifactId>avro</artifactId>
			<version>1.10.0</version>
		</dependency>
		<!-- For Parquet Sink -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-parquet_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
		</dependency>

		<!-- To resolve conflict issue between kinesis connector and kda -->
		<dependency>
			<groupId>com.fasterxml.jackson.core</groupId>
			<artifactId>jackson-core</artifactId>
			<version>2.10.0</version>
		</dependency>
		<dependency>
			<groupId>com.fasterxml.jackson.dataformat</groupId>
			<artifactId>jackson-dataformat-cbor</artifactId>
			<version>2.10.0</version>
		</dependency>

		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-connector-filesystem_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
		</dependency>

		<!-- Config reading library -->
		<dependency>
			<groupId>com.typesafe</groupId>
			<artifactId>config</artifactId>
			<version>1.4.0</version>
		</dependency>

		<!-- End of project's additional dependencies -->

		<!-- Scala Library, provided by Flink as well. -->
		<dependency>
			<groupId>org.scala-lang</groupId>
			<artifactId>scala-library</artifactId>
			<version>${scala.version}</version>
			<scope>${lib.scope}</scope>
		</dependency>

		<!-- Add logging framework, to produce console output when running in the IDE. -->
		<!-- These dependencies are excluded from the application JAR by default. -->
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-log4j12</artifactId>
			<version>1.7.7</version>
			<scope>runtime</scope>
		</dependency>
		<dependency>
			<groupId>log4j</groupId>
			<artifactId>log4j</artifactId>
			<version>1.2.17</version>
			<scope>runtime</scope>
		</dependency>

		<!-- Dependencies for Maven test phase -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-metrics-dropwizard</artifactId>
			<version>${flink.version}</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>com.newrelic.telemetry</groupId>
			<artifactId>dropwizard-metrics-newrelic</artifactId>
			<version>0.4.0</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>com.newrelic.telemetry</groupId>
			<artifactId>telemetry-http-okhttp</artifactId>
			<version>0.4.0</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>io.dropwizard.metrics</groupId>
			<artifactId>metrics-core</artifactId>
			<version>4.1.5</version>
			<scope>test</scope>
		</dependency>

	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-source-plugin</artifactId>
				<version>3.2.1</version>
				<executions>
					<execution>
						<id>attach-sources</id>
						<goals>
							<goal>jar-no-fork</goal>
						</goals>
						<configuration>
							<excludes>
								<exclude>**/ConverJsonToAvro.*</exclude>
								<exclude>**/*Factory.*</exclude>
								<exclude>com/l3l/flink/landing/sink/**</exclude>
							</excludes>
						</configuration>
					</execution>
				</executions>
			</plugin>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-javadoc-plugin</artifactId>
				<version>3.2.0</version>
				<executions>
					<execution>
						<id>attach-javadocs</id>
						<goals>
							<goal>jar</goal>
						</goals>
					</execution>
				</executions>
			</plugin>

			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-gpg-plugin</artifactId>
				<version>1.5</version>
				<executions>
					<execution>
						<id>sign-artifacts</id>
						<phase>verify</phase>
						<goals>
							<goal>sign</goal>
						</goals>
					</execution>
				</executions>
			</plugin>

			<plugin>
				<groupId>org.sonatype.plugins</groupId>
				<artifactId>nexus-staging-maven-plugin</artifactId>
				<version>1.6.7</version>
				<extensions>true</extensions>
				<configuration>
					<serverId>ossrh</serverId>
					<nexusUrl>https://oss.sonatype.org/</nexusUrl>
					<autoReleaseAfterClose>true</autoReleaseAfterClose>
				</configuration>
			</plugin>
			<!-- Java Compiler -->
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.8.0</version>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>

			<!-- Scala Compiler -->
			<plugin>
				<groupId>net.alchim31.maven</groupId>
				<artifactId>scala-maven-plugin</artifactId>
				<version>4.2.0</version>
				<executions>
					<execution>
						<goals>
							<goal>compile</goal>
							<goal>testCompile</goal>
						</goals>
					</execution>
					<execution>
						<id>attach-javadocs</id>
						<goals>
							<goal>doc-jar</goal>
						</goals>
					</execution>
				</executions>
				<configuration>
					<jvmArgs>
						<jvmArg>-Xms256m</jvmArg>
						<jvmArg>-Xmx1024m</jvmArg>
						<jvmArg>-Xss8m</jvmArg>
					</jvmArgs>
					<args>
						<arg>-deprecation</arg>
						<arg>-target:jvm-1.8</arg>
					</args>
				</configuration>
			</plugin>

			<!-- Eclipse Scala Integration -->
			<!--			<plugin>-->
			<!--				<groupId>org.apache.maven.plugins</groupId>-->
			<!--				<artifactId>maven-eclipse-plugin</artifactId>-->
			<!--				<version>2.8</version>-->
			<!--				<configuration>-->
			<!--					<downloadSources>true</downloadSources>-->
			<!--					<projectnatures>-->
			<!--						<projectnature>org.scala-ide.sdt.core.scalanature</projectnature>-->
			<!--						<projectnature>org.eclipse.jdt.core.javanature</projectnature>-->
			<!--					</projectnatures>-->
			<!--					<buildcommands>-->
			<!--						<buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>-->
			<!--					</buildcommands>-->
			<!--					<classpathContainers>-->
			<!--						<classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>-->
			<!--						<classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>-->
			<!--					</classpathContainers>-->
			<!--					<excludes>-->
			<!--						<exclude>org.scala-lang:scala-library</exclude>-->
			<!--						<exclude>org.scala-lang:scala-compiler</exclude>-->
			<!--					</excludes>-->
			<!--					<sourceIncludes>-->
			<!--						<sourceInclude>**/*.scala</sourceInclude>-->
			<!--						<sourceInclude>**/*.java</sourceInclude>-->
			<!--					</sourceIncludes>-->
			<!--				</configuration>-->
			<!--			</plugin>-->
			<plugin>
				<groupId>org.codehaus.mojo</groupId>
				<artifactId>build-helper-maven-plugin</artifactId>
				<version>1.7</version>
				<executions>
					<!-- Add src/main/scala to eclipse build path -->
					<execution>
						<id>add-source</id>
						<phase>generate-sources</phase>
						<goals>
							<goal>add-source</goal>
						</goals>
						<configuration>
							<sources>
								<source>src/main/java</source>
								<source>src/main/scala</source>
							</sources>
						</configuration>
					</execution>
					<!-- Add src/test/scala to eclipse build path -->
					<execution>
						<id>add-test-source</id>
						<phase>generate-test-sources</phase>
						<goals>
							<goal>add-test-source</goal>
						</goals>
						<configuration>
							<sources>
								<source>src/test/scala</source>
							</sources>
						</configuration>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>

	<distributionManagement>
		<snapshotRepository>
			<id>ossrh</id>
			<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
		</snapshotRepository>
		<repository>
			<id>ossrh</id>
			<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
		</repository>
	</distributionManagement>

	<!-- This profile helps to make things run out of the box in IntelliJ -->
	<!-- Its adds Flink's core classes to the runtime class path. -->
	<!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
	<profiles>
		<profile>
			<id>add-dependencies-for-IDEA</id>

			<properties>
				<lib.scope>compile</lib.scope>
			</properties>

			<activation>
				<property>
					<name>idea.version</name>
				</property>
			</activation>

			<dependencies>
				<dependency>
					<groupId>org.apache.flink</groupId>
					<artifactId>flink-scala_${scala.binary.version}</artifactId>
					<version>${flink.version}</version>
					<scope>compile</scope>
				</dependency>
				<dependency>
					<groupId>org.apache.flink</groupId>
					<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
					<version>${flink.version}</version>
					<scope>compile</scope>
				</dependency>
				<dependency>
					<groupId>org.scala-lang</groupId>
					<artifactId>scala-library</artifactId>
					<version>${scala.version}</version>
					<scope>compile</scope>
				</dependency>
				<dependency>
					<groupId>com.amazonaws</groupId>
					<artifactId>aws-kinesisanalytics-runtime</artifactId>
					<version>${kda.version}</version>
				</dependency>
			</dependencies>
		</profile>
	</profiles>
</project>