From 8e20814c9ce6cc671431fbc0fb279a34a148f415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Wed, 18 Jun 2025 14:12:17 +0200 Subject: [PATCH 01/27] * flink bump to 1.20.1 * jdk bump to 17 * scala 2.12 support added (2.13 removed from project name) * building flink images with patched scala --- .github/workflows/ci.yml | 9 +- .github/workflows/publish.yml | 21 ++++ .sbtopts | 1 + Dockerfile | 8 ++ build-images.sh | 37 ++++++ build.sbt | 108 ++++++++++++------ project/plugins.sbt | 2 +- .../types/FlinkScalaKryoInstantiator.scala | 2 +- .../JavaWrapperScala2_13Serializer.scala | 0 .../FlinkScalaKryoInstantiatorSpec.scala | 0 10 files changed, 147 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 .sbtopts create mode 100644 Dockerfile create mode 100755 build-images.sh rename src/main/{scala => scala-2.13}/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala (99%) rename src/main/{scala => scala-2.13}/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala (100%) rename src/test/{scala => scala-2.13}/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c4a65c..203dc5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: olafurpg/setup-scala@v11 + - uses: actions/setup-java@v4 with: - java-version: adopt@1.11 + distribution: temurin + java-version: 17 + cache: sbt + - uses: sbt/setup-sbt@v1 - name: Test - run: sbt test \ No newline at end of file + run: sbt "++test" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..36c4f33 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,21 @@ +name: Publish images +on: + workflow_dispatch: +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + cache: sbt + - uses: sbt/setup-sbt@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push images + run: ./build-images.sh --push diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 0000000..837257c --- /dev/null +++ b/.sbtopts @@ -0,0 +1 @@ +-J--add-opens=java.base/java.util=ALL-UNNAMED diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a324344 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +ARG FLINK_VERSION + +FROM flink:${FLINK_VERSION}-scala_2.12-java17 + +RUN rm $FLINK_HOME/lib/flink-scala*.jar + +ARG FLINK_SCALA_VERSION +COPY --chown=flink:flink flink-scala-assembly-${FLINK_SCALA_VERSION}.jar $FLINK_HOME/lib/ diff --git a/build-images.sh b/build-images.sh new file mode 100755 index 0000000..b9bea31 --- /dev/null +++ b/build-images.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +VERSION=$(sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') +FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') + +if [[ "$1" == "--push" ]]; then + OUTPUT_TYPE="registry" +else + OUTPUT_TYPE="docker" +fi + +sbt "++clean;++assembly" + +IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.12" +echo "Building Docker image with version: $IMAGE_TAG" + +cp target/scala-2.12/flink-scala-assembly-${VERSION}.jar . + +docker buildx build \ + --build-arg FLINK_VERSION=$FLINK_VERSION \ + --build-arg FLINK_SCALA_VERSION=$VERSION \ + --platform linux/amd64,linux/arm64 \ + --tag touk/flink:$IMAGE_TAG \ + --output=type=$OUTPUT_TYPE . + +IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.13" +echo "Building Docker image with version: $IMAGE_TAG" + +cp target/scala-2.13/flink-scala-assembly-${VERSION}.jar . + +docker buildx build \ + --build-arg FLINK_VERSION=$FLINK_VERSION \ + --build-arg FLINK_SCALA_VERSION=$VERSION \ + --platform linux/amd64,linux/arm64 \ + --tag touk/flink:$IMAGE_TAG \ + --output=type=$OUTPUT_TYPE . diff --git a/build.sbt b/build.sbt index c1532d5..a123cba 100644 --- a/build.sbt +++ b/build.sbt @@ -1,29 +1,40 @@ -version := "1.1.2" +import sbtassembly.MergeStrategy -scalaVersion := "2.13.15" +name := "flink-scala" +version := "1.1.3-SNAPSHOT" -name := "flink-scala-2.13" +val scala212 = "2.12.15" +val scala213 = "2.13.15" -lazy val flinkV = "1.16.2" -lazy val scalaTestV = "3.2.17" +scalaVersion := scala212 +crossScalaVersions := List(scala212, scala213) +val flinkV = settingKey[String]("Flink version") // to extract using `show flinkV` +flinkV := "1.20.1" -assembly / artifact := { - val art = (assembly / artifact).value - art.withClassifier(Some("assembly")) -} +lazy val scalaTestV = "3.2.17" -addArtifact(assembly / artifact, assembly) +lazy val assemblySettings = Seq( + assembly / artifact := { + val art = (assembly / artifact).value + art.withClassifier(Some("assembly")) + }, + assembly / assemblyMergeStrategy := { + case PathList(ps@_*) if ps.last == "module-info.class" => MergeStrategy.discard + case x => MergeStrategy.defaultMergeStrategy(x) + }, + addArtifact(assembly / artifact, assembly) +) lazy val publishSettings = Seq( publishMavenStyle := true, publishTo := { - val defaultNexusUrl = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at defaultNexusUrl + "content/repositories/snapshots") - else { - sonatypePublishToBundle.value - } + val defaultNexusUrl = "https://oss.sonatype.org/" + if (isSnapshot.value) + Some("snapshots" at defaultNexusUrl + "content/repositories/snapshots") + else { + sonatypePublishToBundle.value + } }, Test / publishArtifact := false, //We don't put scm information here, it will be added by release plugin and if scm provided here is different than the one from scm @@ -34,34 +45,59 @@ lazy val publishSettings = Seq( scm:git:git@github.com:TouK/flink-scala-2.13.git github.com/TouK/flink-scala-2.13 - - - TouK - TouK - https://touk.pl - - + + + TouK + TouK + https://touk.pl + + }, organization := "pl.touk", ) lazy val root = (project in file(".")) .settings( - name := "flink-scala-2.13", + name := "flink-scala", organization := "pl.touk", licenses := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")), homepage := Some(url("https://github.com/TouK/flink-scala-2.13")), - libraryDependencies ++= { - Seq( - "org.scala-lang" % "scala-compiler" % scalaVersion.value, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - - "org.apache.flink" % "flink-streaming-java" % flinkV % "provided", - "com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"), - "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided", - - "org.scalatest" %% "scalatest" % scalaTestV % "test", - ) - } + libraryDependencies ++= (forScalaVersion(scalaVersion.value) { + case (2, 12) => + Seq( + "org.apache.flink" %% "flink-scala" % flinkV.value excludeAll( + ExclusionRule(organization = "org.apache.flink", name = "flink-core"), + ExclusionRule(organization = "org.apache.flink", name = "flink-java"), + ExclusionRule(organization = "org.apache.flink", name = "flink-shaded-asm-9"), + ExclusionRule(organization = "org.slf4j", name = "slf4j-api"), + ExclusionRule(organization = "com.google.code.findbugs", name = "jsr305"), + ), + "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % Test, + "org.apache.flink" % "flink-java" % flinkV.value % Test, + ) + case (2, 13) => + Seq( + "org.apache.flink" % "flink-streaming-java" % flinkV.value % "provided", + "com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"), + "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided", + ) + } ++ Seq( + "org.scala-lang" % "scala-library" % scalaVersion.value, + "org.scala-lang" % "scala-compiler" % scalaVersion.value, + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "org.scalatest" %% "scalatest" % scalaTestV % Test, + )) ) + .settings(assemblySettings *) .settings(publishSettings) + +def forScalaVersion[T](version: String)(provide: PartialFunction[(Int, Int), T]): T = { + CrossVersion.partialVersion(version) match { + case Some((major, minor)) if provide.isDefinedAt((major.toInt, minor.toInt)) => + provide((major.toInt, minor.toInt)) + case Some(_) => + throw new IllegalArgumentException(s"Scala version $version is not handled") + case None => + throw new IllegalArgumentException(s"Invalid Scala version $version") + } +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 3a6596d..dc39144 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.4") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") diff --git a/src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala b/src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala similarity index 99% rename from src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala rename to src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala index d26b895..e21c6b6 100644 --- a/src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala +++ b/src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala @@ -233,4 +233,4 @@ class ScalaCollectionsRegistrarCompat extends IKryoRegistrar { ) ) } -} \ No newline at end of file +} diff --git a/src/main/scala/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala b/src/main/scala-2.13/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala similarity index 100% rename from src/main/scala/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala rename to src/main/scala-2.13/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala diff --git a/src/test/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala b/src/test/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala similarity index 100% rename from src/test/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala rename to src/test/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala From e3b88e05600c7f7f9ce4d0b6591b3f9dddebb45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Wed, 18 Jun 2025 16:01:33 +0200 Subject: [PATCH 02/27] * flink bump to 1.20.1 * jdk bump to 17 * scala 2.12 support added (2.13 removed from project name) * building flink images with patched scala --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 36c4f33..f185c84 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,4 +18,4 @@ jobs: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push images - run: ./build-images.sh --push + run: ./build-images.sh --push #provoked change From cc9d57e9800ef6fe0986679c7368f6b935bfd4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Wed, 18 Jun 2025 16:16:47 +0200 Subject: [PATCH 03/27] provoked change --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f185c84..36c4f33 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,4 +18,4 @@ jobs: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push images - run: ./build-images.sh --push #provoked change + run: ./build-images.sh --push From 8e0cd7483bdd8e40ae2a023e688dd39033ccc094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:06:17 +0200 Subject: [PATCH 04/27] publish on push to test publish action (should be reverted before merge) --- .github/workflows/publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 36c4f33..838a2e5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,7 @@ name: Publish images on: - workflow_dispatch: +# workflow_dispatch: # todo: requires to be merged to default branch to work + push: jobs: build: runs-on: ubuntu-latest From e3d10f14540abffcc413010792da680a4732ee5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:10:16 +0200 Subject: [PATCH 05/27] additional logging --- build-images.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-images.sh b/build-images.sh index b9bea31..1d45993 100755 --- a/build-images.sh +++ b/build-images.sh @@ -3,6 +3,8 @@ set -e VERSION=$(sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') +echo "FLINK_SCALA_VERSION: ${VERSION}" +echo "FLINK_VERSION: ${FLINK_VERSION}" if [[ "$1" == "--push" ]]; then OUTPUT_TYPE="registry" From 65132f725d9b914c7883a37f64e15085c129dda1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:11:48 +0200 Subject: [PATCH 06/27] additional logging --- build-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-images.sh b/build-images.sh index 1d45993..6f51969 100755 --- a/build-images.sh +++ b/build-images.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e - +sbt -Dsbt.supershell=false "show version" VERSION=$(sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') echo "FLINK_SCALA_VERSION: ${VERSION}" From 135bea5b2c6272323396d0396cf460c490f63b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:13:39 +0200 Subject: [PATCH 07/27] additional logging --- build-images.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-images.sh b/build-images.sh index 6f51969..ee11e88 100755 --- a/build-images.sh +++ b/build-images.sh @@ -1,6 +1,9 @@ #!/bin/bash set -e sbt -Dsbt.supershell=false "show version" +sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " +sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 +sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 | awk '{print $2}' VERSION=$(sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') echo "FLINK_SCALA_VERSION: ${VERSION}" From be170a945b549ef01cd21f1911cc10dadc0cd275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:21:32 +0200 Subject: [PATCH 08/27] additional logging --- build-images.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build-images.sh b/build-images.sh index ee11e88..0fe168b 100755 --- a/build-images.sh +++ b/build-images.sh @@ -1,11 +1,10 @@ #!/bin/bash set -e sbt -Dsbt.supershell=false "show version" -sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " -sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 -sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 | awk '{print $2}' -VERSION=$(sbt -Dsbt.supershell=false "show version" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') -FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | grep -E "^\[info\] " | tail -1 | awk '{print $2}') +sbt -Dsbt.supershell=false "show version" | tail -1 +sbt -Dsbt.supershell=false "show version" | tail -1 | awk '{print $2}' +VERSION=$(sbt -Dsbt.supershell=false "show version" | tail -1 | awk '{print $2}') +FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | tail -1 | awk '{print $2}') echo "FLINK_SCALA_VERSION: ${VERSION}" echo "FLINK_VERSION: ${FLINK_VERSION}" From db2182b3fef2915ac8412cd1aeef3a8fd6d9760f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:23:35 +0200 Subject: [PATCH 09/27] build/publish jdk downgraded --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 838a2e5..41cf214 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17 + java-version: 11 cache: sbt - uses: sbt/setup-sbt@v1 - name: Login to Docker Hub From ee906f496d5d3b10aeed064a5c73d3716a472d93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:26:25 +0200 Subject: [PATCH 10/27] build/publish jdk downgraded --- build-images.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-images.sh b/build-images.sh index 0fe168b..e8aa03f 100755 --- a/build-images.sh +++ b/build-images.sh @@ -1,10 +1,10 @@ #!/bin/bash set -e sbt -Dsbt.supershell=false "show version" -sbt -Dsbt.supershell=false "show version" | tail -1 -sbt -Dsbt.supershell=false "show version" | tail -1 | awk '{print $2}' -VERSION=$(sbt -Dsbt.supershell=false "show version" | tail -1 | awk '{print $2}') -FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | tail -1 | awk '{print $2}') +sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 +sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}' +VERSION=$(sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}') +FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | grep "info" | tail -1 | awk '{print $2}') echo "FLINK_SCALA_VERSION: ${VERSION}" echo "FLINK_VERSION: ${FLINK_VERSION}" From 7f3727338514bccf9cf0d7f51dd64af96fd7712d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:31:33 +0200 Subject: [PATCH 11/27] --no-colors to sbt added --- build-images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-images.sh b/build-images.sh index e8aa03f..7bdeb51 100755 --- a/build-images.sh +++ b/build-images.sh @@ -3,8 +3,8 @@ set -e sbt -Dsbt.supershell=false "show version" sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}' -VERSION=$(sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}') -FLINK_VERSION=$(sbt -Dsbt.supershell=false "show flinkV" | grep "info" | tail -1 | awk '{print $2}') +VERSION=$(sbt --no-colors -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}') +FLINK_VERSION=$(sbt --no-colors -Dsbt.supershell=false "show flinkV" | grep "info" | tail -1 | awk '{print $2}') echo "FLINK_SCALA_VERSION: ${VERSION}" echo "FLINK_VERSION: ${FLINK_VERSION}" From 7708f4b1e31f2646056a4ee6ae52492dd47dbd1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:35:52 +0200 Subject: [PATCH 12/27] setup buildx builder --- .github/workflows/publish.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 41cf214..81c7274 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,6 +13,13 @@ jobs: java-version: 11 cache: sbt - uses: sbt/setup-sbt@v1 + - name: Setup buildx builder + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + config-inline: | + [worker.oci] + max-parallelism = 1 - name: Login to Docker Hub uses: docker/login-action@v1 with: From 0117741bb9ceb0a0f3f898a592c61bf9c5c2278a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 11:57:12 +0200 Subject: [PATCH 13/27] maven artifacts publication + little fixes --- .github/workflows/publish.yml | 4 +++- Dockerfile | 2 +- build-images.sh | 6 ++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 81c7274..25874da 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -name: Publish images +name: Publish on: # workflow_dispatch: # todo: requires to be merged to default branch to work push: @@ -13,6 +13,8 @@ jobs: java-version: 11 cache: sbt - uses: sbt/setup-sbt@v1 + - name: Build and publish maven artifacts + run: sbt "+clean; +publish" - name: Setup buildx builder uses: docker/setup-buildx-action@v3 with: diff --git a/Dockerfile b/Dockerfile index a324344..2f8c202 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG FLINK_VERSION +ARG FLINK_VERSION="invalid" FROM flink:${FLINK_VERSION}-scala_2.12-java17 diff --git a/build-images.sh b/build-images.sh index 7bdeb51..7836c18 100755 --- a/build-images.sh +++ b/build-images.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -sbt -Dsbt.supershell=false "show version" -sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 -sbt -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}' + VERSION=$(sbt --no-colors -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}') FLINK_VERSION=$(sbt --no-colors -Dsbt.supershell=false "show flinkV" | grep "info" | tail -1 | awk '{print $2}') echo "FLINK_SCALA_VERSION: ${VERSION}" @@ -14,7 +12,7 @@ else OUTPUT_TYPE="docker" fi -sbt "++clean;++assembly" +sbt "+clean; +assembly" IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.12" echo "Building Docker image with version: $IMAGE_TAG" From 8afd6766f67e4f117c816ea6a0ef91b02a4d4130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 12:03:49 +0200 Subject: [PATCH 14/27] sonatype credentials setup --- .github/workflows/publish.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 25874da..c26fda2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,9 +2,14 @@ name: Publish on: # workflow_dispatch: # todo: requires to be merged to default branch to work push: + branches: + - jdk17-scala2.12.15 jobs: build: runs-on: ubuntu-latest + env: + SONATYPE_USERNAME: ${{ secrets.sonatype_user }} + SONATYPE_PASSWORD: ${{ secrets.sonatype_password }} steps: - uses: actions/checkout@v1 - uses: actions/setup-java@v4 From e04033a465670937751f231488a6311f8018ef03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 14:25:19 +0200 Subject: [PATCH 15/27] secrets fixup --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c26fda2..3795586 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,8 +8,8 @@ jobs: build: runs-on: ubuntu-latest env: - SONATYPE_USERNAME: ${{ secrets.sonatype_user }} - SONATYPE_PASSWORD: ${{ secrets.sonatype_password }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} steps: - uses: actions/checkout@v1 - uses: actions/setup-java@v4 From 72d1773df32202f6ae4133a356afa596b8ab5737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 14:31:05 +0200 Subject: [PATCH 16/27] provoked change --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3795586..75ae82c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,6 @@ name: Publish on: -# workflow_dispatch: # todo: requires to be merged to default branch to work +# workflow_dispatch: # todo: requires to be merged to default branch to work push: branches: - jdk17-scala2.12.15 From 45f4cfa22a2d063120fd2481d952222bb67e7bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 15:26:14 +0200 Subject: [PATCH 17/27] provoked change --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 75ae82c..3795586 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,6 @@ name: Publish on: -# workflow_dispatch: # todo: requires to be merged to default branch to work +# workflow_dispatch: # todo: requires to be merged to default branch to work push: branches: - jdk17-scala2.12.15 From 7c921524f7750278371112b08a28e11ea39360ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 18:31:52 +0200 Subject: [PATCH 18/27] oss.sonatype.org changed to centra.sonatype.com --- build.sbt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index a123cba..79394c6 100644 --- a/build.sbt +++ b/build.sbt @@ -29,11 +29,10 @@ lazy val assemblySettings = Seq( lazy val publishSettings = Seq( publishMavenStyle := true, publishTo := { - val defaultNexusUrl = "https://oss.sonatype.org/" if (isSnapshot.value) - Some("snapshots" at defaultNexusUrl + "content/repositories/snapshots") + Some("snapshots" at "https://central.sonatype.com/repository/maven-snapshots/") else { - sonatypePublishToBundle.value + sonatypePublishToBundle.value //todo: full release not tested yet } }, Test / publishArtifact := false, From cbf6f647a4ae4a50e5786075b326ec9d4d4cef3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Mon, 23 Jun 2025 19:21:31 +0200 Subject: [PATCH 19/27] credential host fixup (maybe hack) --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 79394c6..6897308 100644 --- a/build.sbt +++ b/build.sbt @@ -28,6 +28,7 @@ lazy val assemblySettings = Seq( lazy val publishSettings = Seq( publishMavenStyle := true, + sonatypeCredentialHost := "central.sonatype.com", publishTo := { if (isSnapshot.value) Some("snapshots" at "https://central.sonatype.com/repository/maven-snapshots/") From e652754f6550a59ceaa3fe956d3c9d78d3f9d67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Tue, 24 Jun 2025 08:05:05 +0200 Subject: [PATCH 20/27] logging --- .github/workflows/publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3795586..90924a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,6 +18,8 @@ jobs: java-version: 11 cache: sbt - uses: sbt/setup-sbt@v1 + - name: Additional log + run: sbt -V - name: Build and publish maven artifacts run: sbt "+clean; +publish" - name: Setup buildx builder From 7b155837cfbe7a3c71e14e74e2fa6ba721bf7db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Tue, 24 Jun 2025 08:12:55 +0200 Subject: [PATCH 21/27] Logging removed --- .github/workflows/publish.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 90924a2..3795586 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,8 +18,6 @@ jobs: java-version: 11 cache: sbt - uses: sbt/setup-sbt@v1 - - name: Additional log - run: sbt -V - name: Build and publish maven artifacts run: sbt "+clean; +publish" - name: Setup buildx builder From a25f84489ad178ba5406232178f0f4effba55900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Tue, 24 Jun 2025 11:49:43 +0200 Subject: [PATCH 22/27] GH repo renamed + readme update --- README.md | 17 ++++++++++++----- build.sbt | 8 ++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index fbd4ebc..002b2bc 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,25 @@ -# flink-scala-2.13 +# flink-scala -For now Flink does not support Scala 2.13. For more refer to . +This module is a replacement for the `org.apache.flink:flink-scala` lib shipped with flink distribution, +and allows using flink with higher scala versions than 2.12.8. -Our solution to deploy Scala 2.13 code to Flink, until it's officially supported (or Flink becomes really scala-free): +For more refer to . +## Replacing flink-scala in flink distribution ```bash rm $FLINK_HOME/lib/flink-scala*.jar -wget https://repo1.maven.org/maven2/pl/touk/flink-scala-2-13_2.13/1.1.2/flink-scala-2-13_2.13-1.1.2-assembly.jar -O $FLINK_HOME/lib/flink-scala-2-13_2.13-1.1.1-assembly.jar + +wget https://central.sonatype.com/repository/maven-snapshots/pl/touk/flink-scala_2.13/1.1.3-SNAPSHOT/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar -O $FLINK_HOME/lib/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar ``` +## Using as a lib (probably only sufficient when child-first classloading is enabled on flink) ```scala -libraryDependencies += "pl.touk" %% "flink-scala-2-13" % "1.1.2" +libraryDependencies += "pl.touk" %% "flink-scala" % "1.1.3-SNAPSHOT" ``` +## Prebuild flink images +* we provide prebuild flink docker images for scala 2.12 and 2.13 on [Docker Hub](https://hub.docker.com/r/touk/flink) + ## Publishing ``` sbt publishSigned sonatypeBundleRelease diff --git a/build.sbt b/build.sbt index 6897308..0bdbd33 100644 --- a/build.sbt +++ b/build.sbt @@ -41,9 +41,9 @@ lazy val publishSettings = Seq( //we'll end up with two scm sections and invalid pom... pomExtra in Global := { - scm:git:github.com/TouK/flink-scala-2.13.git - scm:git:git@github.com:TouK/flink-scala-2.13.git - github.com/TouK/flink-scala-2.13 + scm:git:github.com/TouK/flink-scala.git + scm:git:git@github.com:TouK/flink-scala.git + github.com/TouK/flink-scala @@ -61,7 +61,7 @@ lazy val root = (project in file(".")) name := "flink-scala", organization := "pl.touk", licenses := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")), - homepage := Some(url("https://github.com/TouK/flink-scala-2.13")), + homepage := Some(url("https://github.com/TouK/flink-scala")), libraryDependencies ++= (forScalaVersion(scalaVersion.value) { case (2, 12) => Seq( From 1a2585f4df0d2d2271947c096bcd44dc8157581a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Tue, 24 Jun 2025 14:12:19 +0200 Subject: [PATCH 23/27] scala test bump --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 0bdbd33..e743818 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ crossScalaVersions := List(scala212, scala213) val flinkV = settingKey[String]("Flink version") // to extract using `show flinkV` flinkV := "1.20.1" -lazy val scalaTestV = "3.2.17" +lazy val scalaTestV = "3.2.19" lazy val assemblySettings = Seq( assembly / artifact := { From 7c0e8a47c82fcbbe1ce0b1ee6d3d6a4af79a9173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Tue, 24 Jun 2025 15:42:07 +0200 Subject: [PATCH 24/27] scala upgrade 2.12.15 -> 2.12.20 2.13.15 -> 2.13.16 --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index e743818..2b352bb 100644 --- a/build.sbt +++ b/build.sbt @@ -3,8 +3,8 @@ import sbtassembly.MergeStrategy name := "flink-scala" version := "1.1.3-SNAPSHOT" -val scala212 = "2.12.15" -val scala213 = "2.13.15" +val scala212 = "2.12.20" +val scala213 = "2.13.16" scalaVersion := scala212 crossScalaVersions := List(scala212, scala213) From 0e6e3653b9115895785359f28c13bc09e057e62b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Thu, 26 Jun 2025 09:45:33 +0200 Subject: [PATCH 25/27] review fix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2f8c202..c282f2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,5 +4,5 @@ FROM flink:${FLINK_VERSION}-scala_2.12-java17 RUN rm $FLINK_HOME/lib/flink-scala*.jar -ARG FLINK_SCALA_VERSION +ARG FLINK_SCALA_VERSION="invalid" COPY --chown=flink:flink flink-scala-assembly-${FLINK_SCALA_VERSION}.jar $FLINK_HOME/lib/ From f237c073a15f0e61e3a6dd75baedf5087530ba3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Thu, 26 Jun 2025 10:55:09 +0200 Subject: [PATCH 26/27] publishSigned configuration --- .github/workflows/publish.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3795586..f6ca4a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,8 +18,14 @@ jobs: java-version: 11 cache: sbt - uses: sbt/setup-sbt@v1 + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSWORD }} - name: Build and publish maven artifacts - run: sbt "+clean; +publish" + run: sbt "+clean; +publishSigned" - name: Setup buildx builder uses: docker/setup-buildx-action@v3 with: From d8bcd2b53e4276d0d6a2e784b448b43135ca8b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Solarski?= Date: Thu, 26 Jun 2025 11:00:05 +0200 Subject: [PATCH 27/27] publish as manual ci action --- .github/workflows/publish.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f6ca4a2..1bbe495 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,9 +1,6 @@ name: Publish on: -# workflow_dispatch: # todo: requires to be merged to default branch to work - push: - branches: - - jdk17-scala2.12.15 + workflow_dispatch: jobs: build: runs-on: ubuntu-latest