Finally working ProductBQRead WIP

build.sbt

@@ -76,7 +76,8 @@ lazy val root = tlCrossRootProject
     codegen,
     codegenTests,
     `transfer-client`,
-    docs)
+    docs
+  )
 
 lazy val core = crossProject(JVMPlatform)
   .withoutSuffixFor(JVMPlatform)
@@ -115,6 +116,7 @@ lazy val core = crossProject(JVMPlatform)
         )
       }
     },
+    Compile / sourceGenerators += (Compile / sourceManaged).map(Boilerplate.gen).taskValue,
     Compile / doc / scalacOptions ++= Seq(
       "-no-link-warnings" // Suppresses problems with Scaladoc @throws links
     ),

core/src/main/scala/no/nrk/bigquery/BQRead.scala

@@ -7,14 +7,12 @@
 package no.nrk.bigquery
 
 import io.circe.Json
-
 import org.apache.avro.util.Utf8
 import org.apache.avro
 
 import java.time.*
 import scala.collection.compat.*
 import scala.jdk.CollectionConverters.*
-
 import scala.reflect.ClassTag
 
 trait BQRead[A] { outer =>
@@ -30,7 +28,7 @@ trait BQRead[A] { outer =>
   def read(transportSchema: avro.Schema, value: Any): A
 }
 
-object BQRead extends BQReadCompat {
+object BQRead extends BQReadCompat with ProductBQRead {
 
   def apply[A: BQRead]: BQRead[A] = implicitly
 

generated/bq-query-static/default_value_in_case_class.sql

@@ -0,0 +1 @@
+SELECT * FROM UNNEST(ARRAY<STRUCT<a STRING, b BOOL>>[("a1", NULL), ("a2", TRUE)])

generated/bq-query-static/product_test_-_nested_structure.sql

@@ -0,0 +1 @@
+SELECT * FROM UNNEST(ARRAY<STRUCT<numberOfTests STRING, tests ARRAY<STRUCT<a STRING, b INT64>>>>[("a", [("b", 10), ("c", 11)])])

generated/bq-query/default_value_in_case_class.sql

@@ -0,0 +1 @@
+SELECT * FROM UNNEST(ARRAY<STRUCT<a STRING, b BOOL>>[("a1", NULL), ("a2", TRUE)])

generated/bq-query/product_test_-_nested_structure.sql

@@ -0,0 +1 @@
+SELECT * FROM UNNEST(ARRAY<STRUCT<numberOfTests STRING, tests ARRAY<STRUCT<a STRING, b INT64>>>>[("a", [("b", 10), ("c", 11)])])

project/Boilerplate.scala

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2020 NRK
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+/** Generate a boilerplate class that would be tedious to write and maintain by hand.
+  *
+  * Copied, with some modifications, from
+  * [[https://github.com/circe/circe/blob/series/0.14.x/project/Boilerplate.scala circe]].
+  *
+  * @author
+  *   Miles Sabin
+  * @author
+  *   Kevin Wright
+  */
+
+import sbt.*
+
+import java.io.File
+
+object Boilerplate {
+  import scala.StringContext.*
+
+  implicit class BlockHelper(val sc: StringContext) extends AnyVal {
+    def block(args: Any*): String = {
+      val interpolated = sc.standardInterpolator(treatEscapes, args)
+      val rawLines = interpolated.split('\n')
+      val trimmedLines = rawLines.map(_.dropWhile(_.isWhitespace))
+      trimmedLines.mkString("\n")
+    }
+  }
+
+  /** Return a sequence of the generated files.
+    *
+    * As a side-effect, it actually generates them...
+    */
+  def gen(dir: File): Seq[File] = {
+    val template = GenProductBqRead
+    val tgtFile = template.filename(dir)
+    print(tgtFile.toString)
+    IO.write(tgtFile, template.body)
+    Seq(tgtFile)
+  }
+
+  val header = "// auto-generated boilerplate"
+  val maxArity = 22
+
+  trait Template {
+    def filename(root: File): File
+
+    def content(arity: Int): String
+
+    def range: IndexedSeq[Int] = 1 to maxArity
+
+    def body: String = {
+      val headerLines = header.split('\n')
+      val raw = range.map(n => content(n).split('\n').filterNot(_.isEmpty))
+      val preBody = raw.head.takeWhile(_.startsWith("|")).map(_.tail)
+      val instances = raw.flatMap(_.filter(_.startsWith("-")).map(_.tail))
+      val postBody = raw.head.dropWhile(_.startsWith("|")).dropWhile(_.startsWith("-")).map(_.tail)
+      (headerLines ++ preBody ++ instances ++ postBody).mkString("\n")
+    }
+  }
+
+  object GenProductBqRead extends Template {
+    override def range: IndexedSeq[Int] = 1 to maxArity
+
+    def filename(root: File): File = root / "no" / "nrk" / "bigquery" / "ProductBQRead.scala"
+
+    def content(arity: Int): String = {
+      val synTypes = (0 until arity).map(n => s"A$n")
+      val `A..N` = synTypes.mkString(", ")
+      val instances = synTypes.map(tpe => s"bqRead$tpe: BQRead[$tpe]").mkString(", ")
+      val memberNames = synTypes.map(tpe => s"name$tpe: String").mkString(", ")
+      val fieldTypes = synTypes.map(tpe => s"name$tpe -> bqRead$tpe.bqType").mkString(", ")
+      val getNamedFields = synTypes.map(tpe => s"getField[$tpe](name$tpe)").mkString(", ")
+
+      block"""
+        |package no.nrk.bigquery
+        |
+        |import no.nrk.bigquery.BQRead.firstNotNullable
+        |import org.apache.avro.Schema
+        |import org.apache.avro.generic.GenericRecord
+        |
+        |private[bigquery] trait ProductBQRead {
+        -  /**
+        -   * @group Product
+        -   */
+        -  final def forProduct$arity[Target, ${`A..N`}]($memberNames)(f: (${`A..N`}) => Target)(implicit
+        -    $instances
+        -  ): BQRead[Target] = new BQRead[Target] {
+        -      override val bqType: BQType = BQType(
+        -          BQField.Mode.REQUIRED,
+        -          BQField.Type.STRUCT,
+        -          List($fieldTypes)
+        -       )
+        -      override def read(transportSchema: Schema, value: Any): Target =
+        -        value match {
+        -          case coll: GenericRecord =>
+        -            val schema = firstNotNullable(transportSchema).getOrElse(transportSchema)
+        -            def getField[A: BQRead](name: String) : A = implicitly[BQRead[A]].read(schema.getField(name).schema(), coll.get(name))
+        -            f($getNamedFields)
+        -          case other => sys.error(s"Unexpected: $${other.getClass.getSimpleName} $$other . Schema from BQ: $$transportSchema")
+        -        }
+        -    }
+        |}
+      """
+    }
+  }
+}

testing/src/test/scala/no/nrk/bigquery/ProductBQReadTest.scala

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 NRK
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+package no.nrk.bigquery
+
+import no.nrk.bigquery.syntax.bqShowInterpolator
+import no.nrk.bigquery.testing.BQSmokeTest
+
+class ProductBQReadTest extends BQSmokeTest(Http4sTestClient.testClient) {
+  case class SubStructure(a: String, b: Long)
+
+  object SubStructure {
+    implicit val bqRead: BQRead[SubStructure] = BQRead.derived
+  }
+
+  case class ComplexType(num: String, subs: List[SubStructure])
+
+  object ComplexType {
+    implicit val bqRead: BQRead[ComplexType] =
+      BQRead.forProduct2("numberOfTests", "tests")((n, tests) => ComplexType(n, tests))
+  }
+
+  bqTypeWithNameCheckTest("product test - nested structure") {
+    BQQuery[ComplexType](
+      bqsql"""SELECT * FROM UNNEST(ARRAY<STRUCT<numberOfTests STRING, tests ARRAY<STRUCT<a STRING, b INT64>>>>[("a", [("b", 10), ("c", 11)])])"""
+    )
+  }
+}