From d923382be60c61db79da99d03a371bf89d2f9201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 4 Nov 2026 20:11:07 +0100 Subject: [PATCH] feat(sql): ARRAY round trip, derived and WITH columns typed, struct order (SCHEMA-6) Story SCHEMA-6, rebased onto SCHEMA-5. Reviewed: r0:must-fix, r1:must-fix, r2:clean, r3:clean. Co-Authored-By: Claude Opus 5.5 --- .../elastic/client/AggregateReading.scala | 30 +- .../elastic/client/ElasticConversion.scala | 77 +- .../elastic/client/GatewayApi.scala | 19 +- .../elastic/client/HitReading.scala | 210 ++++- .../elastic/client/SearchApi.scala | 39 +- .../softnetwork/elastic/client/package.scala | 6 +- .../elastic/client/HitReadingSpec.scala | 36 +- .../elastic/client/MultiValuedFailSpec.scala | 287 +++++++ .../elastic/client/ResultSchemaSpec.scala | 9 +- .../client/SetOperationLegReadersSpec.scala | 5 +- .../client/SetOperationLegSchemasSpec.scala | 5 +- documentation/sql/ddl_statements.md | 47 ++ documentation/sql/dql_statements.md | 46 +- ...estHighLevelClientArrayRoundTripSpec.scala | 24 + .../client/java/JavaClientConversion.scala | 7 +- .../arrow/JavaClientSchemaConsumerSpec.scala | 42 + .../client/JavaClientArrayRoundTripSpec.scala | 24 + .../java/JavaClientConversionSpec.scala | 10 +- .../client/java/JavaClientConversion.scala | 7 +- .../java/JavaClientConversionSpec.scala | 10 +- .../softnetwork/elastic/schema/package.scala | 19 +- .../sql/function/aggregate/package.scala | 24 +- .../elastic/sql/function/cond/package.scala | 3 +- .../elastic/sql/function/math/package.scala | 15 +- .../elastic/sql/function/package.scala | 7 +- .../elastic/sql/function/time/package.scala | 7 +- .../operator/math/ArithmeticExpression.scala | 5 +- .../app/softnetwork/elastic/sql/package.scala | 16 +- .../elastic/sql/parser/type/package.scala | 13 +- .../softnetwork/elastic/sql/query/From.scala | 123 +++ .../elastic/sql/query/GroupBy.scala | 8 +- .../softnetwork/elastic/sql/query/Where.scala | 4 +- .../elastic/sql/query/package.scala | 142 +++- .../elastic/sql/schema/MappingsRules.scala | 11 + .../elastic/sql/schema/TableDiff.scala | 10 +- .../elastic/sql/schema/package.scala | 61 +- .../elastic/sql/type/SQLTypeUtils.scala | 79 +- .../elastic/sql/type/SQLTypes.scala | 16 +- .../elastic/sql/parser/ParserSpec.scala | 2 +- .../sql/query/AggregateReportedTypeSpec.scala | 47 +- .../query/DerivedColumnTypeMatrixSpec.scala | 730 ++++++++++++++++++ .../sql/query/ElementViewTwinSpec.scala | 329 ++++++++ .../sql/schema/ArrayDeclarationDiffSpec.scala | 200 +++++ .../ScriptReferenceValidationSpec.scala | 195 +++++ .../sql/schema/StructFieldOrderSpec.scala | 203 +++++ .../sql/type/ArrayTypeMappingSpec.scala | 204 +++++ .../elastic/client/ArrayRoundTripSpec.scala | 559 ++++++++++++++ .../elastic/client/DottedColumnSpec.scala | 82 +- .../elastic/client/SetOperationTypeSpec.scala | 102 +-- 49 files changed, 3962 insertions(+), 194 deletions(-) create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/MultiValuedFailSpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientArrayRoundTripSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientArrayRoundTripSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/query/DerivedColumnTypeMatrixSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/query/ElementViewTwinSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/schema/ArrayDeclarationDiffSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/schema/StructFieldOrderSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/type/ArrayTypeMappingSpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/ArrayRoundTripSpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala b/core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala index 2fb5758bb..b2d689b43 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala @@ -102,7 +102,8 @@ private[client] object AggregateReading { single.fieldAliases, dates, HitReading.Default, - single.sources.headOption.getOrElse("") + single.sources.headOption.getOrElse(""), + HitReading.remedyOf(single.schema) ) val keys = new java.util.HashMap[String, KeyReader]() single.buckets.foreach { b => @@ -110,7 +111,8 @@ private[client] object AggregateReading { if (name.nonEmpty && !keys.containsKey(name)) { val reader = if (instants.contains(name)) InstantKey - else keyReader(name, MultiSearch.bucketType(b), dates, readers) + // a GROUP BY level's key by its type, a window level's by its partition type (A-28) + else keyReader(name, MultiSearch.levelType(single, b), dates, readers) if (reader ne null) keys.put(name, reader) } } @@ -133,7 +135,7 @@ private[client] object AggregateReading { case AggregationType.Count | AggregationType.Sum | AggregationType.Avg | AggregationType.Min | AggregationType.Max | AggregationType.BucketScript => owners.get(name).foreach { field => - val reader = metricReader(MultiSearch.fieldType(field)) + val reader = metricReader(MultiSearch.fieldType(field, single)) if (reader ne null) metrics.put(name, reader) } case AggregationType.FirstValue | AggregationType.LastValue | @@ -500,6 +502,11 @@ private[client] object PartitionKeys { def of(request: SingleSearch, context: ConversionContext): PartitionKeys = { val partitions = request.windowFunctions.flatMap(wf => wf.partitionBy.zip(wf.buckets)) val names = partitions.map(_._1.aliasOrName).distinct.toArray + // SCHEMA-6 (A-28) -- the base side of a selected ARRAY partition column: its value as read + val asRead: java.util.HashMap[String, String] = context match { + case s: SchemaColumns => s.reading.asReadKeys + case _ => null + } val instants: java.util.HashMap[String, String] = context match { case s: SchemaColumns => s.reading.instants case _ => null @@ -507,11 +514,12 @@ private[client] object PartitionKeys { // the base side of an instant column reads its doc value's key; the cache side, whose reading // holds no instant, reads the name, under which its key reader put the bucket's instant val rowKeys: Array[String] = - if ((instants eq null) || instants.isEmpty) names + if (((instants eq null) || instants.isEmpty) && ((asRead eq null) || asRead.isEmpty)) names else names.map { name => - val key = instants.get(name) - if (key eq null) name else key + val key = if (instants eq null) null else instants.get(name) + val asReadKey = if ((key ne null) || (asRead eq null)) null else asRead.get(name) + if (key ne null) key else if (asReadKey ne null) asReadKey else name } val keyOf = new Array[Any => Any](names.length) if (context.isInstanceOf[SchemaColumns]) { @@ -520,7 +528,7 @@ private[client] object PartitionKeys { val name = names(i) if (rowKeys(i) eq name) partitions.find(_._1.aliasOrName == name).foreach { case (_, bucket) => - MultiSearch.bucketType(bucket).sqlType match { + MultiSearch.partitionType(bucket).sqlType match { case SQLTypes.Timestamp | SQLTypes.Time => keyOf(i) = toMillis case _ => } @@ -591,7 +599,7 @@ private[client] object PartitionKeys { !id.nested && id.functions.isEmpty && name.nonEmpty && name.indexOf('.') < 0 && bucket.name == name plain && { - val t = MultiSearch.bucketType(bucket) + val t = MultiSearch.partitionType(bucket) t.sqlType match { case SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time => DateRoute.of(t.sqlType, t.format, dates).isInstanceOf[DateRoute.AsStored] @@ -605,6 +613,12 @@ private[client] object PartitionKeys { */ private[client] def instantKey(name: String): String = "\u0000instant:" + name + /** SCHEMA-6 (A-28) -- the base row key a selected `ARRAY` partition column's value is read under + * AS READ (0.24.0's reading, the one its window aggregation key has), beside the column's typed + * value: led by a NUL, so no column's name, and dropped by the final projection. + */ + private[client] def asReadKey(name: String): String = "\u0000asread:" + name + /** The instant of a base row's `epoch_millis` doc value (a one-element array of its text): the * epoch milliseconds' decimal text, the spelling of [[AggregateReading]] 's instant key -- any * sub-millisecond fraction (a `date_nanos` value) left out, as its aggregation buckets it. No diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index 444cce723..6268f071c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -550,6 +550,32 @@ trait ElasticConversion { if (found) Some(current) else None } + /** SCHEMA-6 (A-26) -- the values the path `segments` holds in `map` where a list stands on it (a + * list of objects): every list met flattened, in order, as Elasticsearch indexes the path; one + * value is that value, several a `List`, none `None`. A read with no reader of its own (a + * ranking window's sort key), as read. + */ + private[client] def collectedValue( + segments: Array[String], + map: Map[String, Any] + ): Option[Any] = { + val out = ListBuffer.empty[Any] + def walk(v: Any, i: Int): Unit = v match { + case null => () + case l: Seq[_] => l.foreach(walk(_, i)) + case m: Map[_, _] if i < segments.length => + m.asInstanceOf[Map[String, Any]].get(segments(i)).foreach(walk(_, i + 1)) + case other if i == segments.length => out += other + case _ => () + } + walk(map, 0) + out.toList match { + case Nil => None + case one :: Nil => Some(one) + case several => Some(several) + } + } + /** The value a row carries under a FLAT key, when it holds one (SCHEMA-12, A-3). A flat `null` is * no value: Elasticsearch indexes none for it (a document `{"obj.a": null, "obj": {"a": 4}}` * holds 4 in doc values, the `fields` API, SUM and COUNT), so the nested spelling is read @@ -757,7 +783,11 @@ trait ElasticConversion { } catch { case cell: HitReading.MultiValuedCell => throw new IllegalArgumentException( - HitReading.multiValuedMessage(cell, Option(hit.get("_id")).map(_.asText())) + HitReading.multiValuedMessage( + cell, + Option(hit.get("_id")).map(_.asText()), + Option(hit.get("_index")).map(_.asText()) + ) ) } } @@ -786,6 +816,9 @@ trait ElasticConversion { ): ListMap[String, Any] = { val entries = ListMap.newBuilder[String, Any] var folds: ListBuffer[(String, String, JsonNode)] = null + // SCHEMA-6 (A-26) -- the selected paths of a parent read whole that holds a list of objects + var collectedPaths: ListBuffer[(String, Any)] = null + val listPaths = reading.listPaths val it = node.properties().iterator() while (it.hasNext) { val e = it.next() @@ -796,6 +829,22 @@ trait ElasticConversion { if ((ignored ne null) && ignored.contains(key)) reading.onIgnored(key) else null val value = if (onIgnored ne null) onIgnored(e.getValue) else reader(e.getValue) entries += fieldAliases.getOrElse(key, key) -> value + // SCHEMA-6 (A-28) -- a selected ARRAY partition column's value as read, under the row key + // the window join reads + if (!reading.asReadKeys.isEmpty) { + val asReadKey = reading.asReadKeys.get(key) + if (asReadKey ne null) entries += asReadKey -> jsonNodeToAny(e.getValue, fieldAliases) + } + if (!listPaths.isEmpty && e.getValue.isArray) { + val selected = listPaths.get(key) + if (selected ne null) { + if (collectedPaths eq null) collectedPaths = ListBuffer.empty + selected.foreach { path => + val values = HitReading.collected(e.getValue, path.segments) + collectedPaths += path.key -> (if (values eq null) null else path.reader(values)) + } + } + } } else { val parent = reading.foldsUnder(key) if (parent ne null) { @@ -805,7 +854,15 @@ trait ElasticConversion { entries += fieldAliases.getOrElse(key, key) -> jsonNodeToAny(e.getValue, fieldAliases) } } - val row = entries.result() + val read = entries.result() + // a path collected under a list of objects answers under its own key, unless the hit holds a + // non-null flat value of it (A-3 precedence, `flatValue`) + val row = + if (collectedPaths eq null) read + else + collectedPaths.foldLeft(read) { case (acc, (key, value)) => + if (ElasticConversion.flatValue(acc, key).isDefined) acc else acc + (key -> value) + } if (folds eq null) row else folds.foldLeft(row) { case (acc, (parent, key, value)) => @@ -2028,7 +2085,21 @@ trait ElasticConversion { val processedHits = hits.map { hit => val source = if (byField eq null) extractSource(hit, fieldAliases) - else readElement(hit, fieldAliases, byField) + else + try readElement(hit, fieldAliases, byField) + catch { + // SCHEMA-6 (D6) -- a several-valued scalar column of a top hit (FIRST_VALUE, + // LAST_VALUE, ARRAY_AGG) fails as a row's does: the top hit's document and index, + // and the ALTER that declares the column a list + case cell: HitReading.MultiValuedCell => + throw new IllegalArgumentException( + HitReading.multiValuedMessage( + cell, + Option(hit.get("_id")).map(_.asText()), + Option(hit.get("_index")).map(_.asText()) + ) + ) + } if (hasMultipleValues) { if (isRanking) { val metadata = extractHitId(hit) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index c646145e0..ba491ad56 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -1116,8 +1116,23 @@ class TableExecutor( return Left(error) } // we update the schema based on the DQL select clause - fromSchema - .mergeWithSearch(single) + val merged = fromSchema.mergeWithSearch(single) + // SCHEMA-6 (D2) -- a column the SELECT declares a list of lists (`ARRAY_AGG` of a + // list column) is refused by name, before the target is created or replaced + merged.columns.iterator + .flatMap(app.softnetwork.elastic.sql.schema.nestedArrayRefusal(_)) + .toList + .headOption + .foreach { reason => + val error = ElasticError( + message = reason, + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Left(error) + } + merged .copy( name = indexName, // set index name settings = fromSchema.settings // remove settings that cannot be copied diff --git a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala index f9c9798fc..abf17ceba 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala @@ -80,7 +80,11 @@ private[client] final class HitReading( val keep: Set[String], private[client] val star: Boolean, private[client] val flatReaders: String => JsonNode => Any, - private[client] val instants: java.util.HashMap[String, String] + private[client] val instants: java.util.HashMap[String, String], + private[client] val listPaths: java.util.HashMap[String, Array[HitReading.ListPath]] = + new java.util.HashMap[String, Array[HitReading.ListPath]](), + private[client] val asReadKeys: java.util.HashMap[String, String] = + new java.util.HashMap[String, String]() ) { /** Whether a hit's `_ignored` must be read (D8): the plan reads some `_source` key as a date. */ @@ -103,8 +107,39 @@ private[client] final class HitReading( private[client] object HitReading { - /** SCHEMA-6 flips it to [[MultiValued.Fail]], in the same core release (D7). */ - private[client] val Default: MultiValued = MultiValued.AsRead + /** SCHEMA-6 (D6, lead Q1): several values in a column not declared `ARRAY` fail the statement, + * naming the `ALTER` that declares it a list -- the remedy ships in the same release. + */ + private[client] val Default: MultiValued = MultiValued.Fail + + /** SCHEMA-6 (A-26) -- a selected dotted column under a parent the plan reads WHOLE (`SELECT obj, + * obj.a`, `SELECT *, obj.a`): where the parent's `_source` value is a list of objects, the row + * key `key` holds the values the path `segments` collects there ([[Readers.collected]]), read by + * `reader`, the path's own reader -- as the parent read for its paths only answers them. + */ + private[client] final class ListPath( + val key: String, + val segments: Array[String], + val reader: JsonNode => Any + ) + + /** SCHEMA-6 (A-26) -- the values the sub-path `segments` holds under `node` (a list of objects), + * in order, every list met on the way -- an element's own list included -- flattened, as + * Elasticsearch indexes a path under a list of objects; `null` when it holds none. + */ + private[client] def collected(node: JsonNode, segments: Array[String]): JsonNode = { + val out = com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.arrayNode() + def walk(n: JsonNode, i: Int): Unit = + if (n != null && !n.isNull) { + if (n.isArray) { + val it = n.elements() + while (it.hasNext) walk(it.next(), i) + } else if (i == segments.length) out.add(n) + else if (n.isObject) walk(n.get(segments(i)), i + 1) + } + walk(node, 0) + if (out.size() == 0) null else out + } /** A bare `*` item. */ private[client] def isStar(field: Field): Boolean = @@ -114,15 +149,67 @@ private[client] object HitReading { private[client] def isComputed(field: Field): Boolean = field.identifier.functions.nonEmpty || field.isScriptField - /** Thrown by a scalar reader under [[MultiValued.Fail]]; the hit loop names the document. */ - private[client] final class MultiValuedCell(val column: String, val table: String) - extends RuntimeException(column, null, false, false) + /** Thrown by a scalar reader under [[MultiValued.Fail]]; the hit loop names the document and its + * index. `alter` is the `ALTER COLUMN …` clause that declares the column a list ([[remedyOf]]), + * empty for a name no schema column has (a computed item); `table` the statement's source, the + * index named when the hit names none. + */ + private[client] final class MultiValuedCell( + val column: String, + val alter: String, + val table: String + ) extends RuntimeException(column, null, false, false) + + /** The message a several-element cell fails with under [[MultiValued.Fail]] (SCHEMA-6 D6): the + * column, the document `_id` and the index the hit came from (`_index`, else the statement's + * source), and the `ALTER TABLE` statement -- one that parses and runs -- declaring it a list. + */ + private[client] def multiValuedMessage( + cell: MultiValuedCell, + id: Option[String], + index: Option[String] + ): String = { + val in = index.filter(_.nonEmpty).getOrElse(cell.table) + val where = + id.fold("")(i => s" in document $i") + (if (in.nonEmpty) s" of index $in" else "") + val remedy = + if (cell.alter.isEmpty || in.isEmpty) "" + else + s": declare it a list with ALTER TABLE ${app.softnetwork.elastic.sql.renderColumnName(in)} " + + cell.alter + s"Column ${cell.column} holds several values$where, but is declared as one value$remedy" + } - /** The message a several-element cell fails with under [[MultiValued.Fail]] (D7). */ - private[client] def multiValuedMessage(cell: MultiValuedCell, id: Option[String]): String = - s"Column ${cell.column} holds several values${id.fold("")(i => s" in document $i")}, but is " + - s"declared as one value: declare it a list with ALTER TABLE ${cell.table} ALTER COLUMN " + - s"${cell.column} TYPE ARRAY<...>" + /** The `ALTER COLUMN …` clause declaring the column at `path` of `schema` a list of its own type + * (SCHEMA-6 D6), read on the error path only: `ALTER COLUMN c SET DATA TYPE ARRAY` for a + * column, X its declared type; for a sub-field `p.f`, `ALTER COLUMN p SET FIELD `, `` the + * sub-field as its schema declares it -- options, DEFAULT, COMMENT, NOT NULL and multi-fields + * kept, `SET FIELD` replacing the whole sub-field -- with the type `ARRAY`. `None` for a name + * the schema does not have. + */ + private[client] def remedyOf( + schema: Option[app.softnetwork.elastic.sql.schema.Table] + ): String => Option[String] = + path => + schema.flatMap(_.find(path)).map { c => + val list = c.dataType match { + case declared: app.softnetwork.elastic.sql.`type`.SQLArray => declared + case declared => SQLTypes.Array(declared) + } + val dot = path.lastIndexOf('.') + if (dot < 0) + s"ALTER COLUMN ${app.softnetwork.elastic.sql.renderColumnName(path)} SET DATA TYPE " + + list.typeId + else { + val parent = path + .substring(0, dot) + .split("\\.") + .map(app.softnetwork.elastic.sql.renderColumnName) + .mkString(".") + val field = c.copy(dataType = list, struct = None).update() + s"ALTER COLUMN $parent SET FIELD ${field.sql.replaceAll("\\n\\t*", " ")}" + } + } /** A plan that reads nothing and projects STRICTLY, keeping `keep` (D9): the final projection of * a window-enriched statement (SCHEMA-2b, B7 d), whose rows were read by their base query's @@ -169,7 +256,7 @@ private[client] object HitReading { val known = itemTypes.get(f) if (known ne null) known else { - val t = MultiSearch.fieldType(f) + val t = MultiSearch.fieldType(f, single) itemTypes.put(f, t) t } @@ -187,7 +274,7 @@ private[client] object HitReading { val schema = ResultSchemas.of(single, dates, typeOf, columnTypeOf) val table = single.sources.headOption.getOrElse("") val aliases = single.fieldAliases - val readers = new Readers(aliases, dates, multiValued, table) + val readers = new Readers(aliases, dates, multiValued, table, remedyOf(single.schema)) val fields = single.select.fieldsWithComputedAliases val star = fields.exists(isStar) val source = new java.util.HashMap[String, JsonNode => Any]() @@ -254,12 +341,27 @@ private[client] object HitReading { } } // a dotted column's parent not read whole: its selected paths by their types, every other key - // as read + // as read; a parent read whole: its selected paths collected where it holds a list of objects + // (SCHEMA-6 A-26) + val listPaths = new java.util.HashMap[String, Array[ListPath]]() paths.foreach { case (parent, under) => if (!source.containsKey(parent)) source.put(parent, readers.paths(parent, under.toList)) + else + listPaths.put( + parent, + under.map { case (p, t) => + val path = s"$parent.$p" + new ListPath( + aliases.getOrElse(path, path), + p.split("\\."), + readers.column(path, t, Set.empty)._1 + ) + }.toArray + ) } // SCHEMA-2b (B7 a) -- each plain PARTITION BY column of the window statement: read by the type // its key is read with (`MultiSearch.bucketType`), a DATE / TIME one exactly (A-23), and kept + val asReadKeys = new java.util.HashMap[String, String]() val carriedKeys: Seq[String] = if (single.schema.isEmpty) Nil else @@ -268,9 +370,14 @@ private[client] object HitReading { val name = id.name if (id.nested || id.functions.nonEmpty || name.isEmpty || name.indexOf('.') >= 0) None else { - val t = MultiSearch.bucketType(b) + val t = MultiSearch.partitionType(b) t.sqlType match { case SQLTypes.Date | SQLTypes.Time => source.put(name, readers.exact(name, t)) + // SCHEMA-6 (A-28) -- an `ARRAY` partition column is read as read, as 0.24.0 read it; + // selected, it keeps its own reader and its partition value is read as read under + // its own row key, the one the join reads (`PartitionKeys.of`) + case _ if MultiSearch.arrayLevel(b) && source.containsKey(name) => + asReadKeys.put(name, PartitionKeys.asReadKey(name)) case _ => if (!source.containsKey(name)) put(name, t, keepUnder(name)) } Some(name) @@ -306,7 +413,8 @@ private[client] object HitReading { // normalizer -- an UNNEST its WHERE filters per element included (`UnnestScope.rowExplosion`) val keep: Set[String] = Set(ElasticConversion.DocumentIdField) ++ single.nestedHitsMappings.keySet ++ UnnestScope.rowExplosion(single).keySet ++ carriedKeys ++ - instantKeys.values().toArray(new Array[String](0)) + instantKeys.values().toArray(new Array[String](0)) ++ + asReadKeys.values().toArray(new Array[String](0)) // a flat dotted key folded under `*` (rare): its reader built once per path, on first use val flatCache = new java.util.concurrent.ConcurrentHashMap[String, JsonNode => Any]() val flatReaders: String => JsonNode => Any = path => { @@ -331,7 +439,9 @@ private[client] object HitReading { keep, star && schema.isDefined && fields.forall(isStar), flatReaders, - instantKeys + instantKeys, + listPaths, + asReadKeys ) } @@ -342,9 +452,14 @@ private[client] object HitReading { aliases: ListMap[String, String], dates: SetOperationValues.DateReading, multiValued: MultiValued, - table: String + table: String, + remedy: String => Option[String] = _ => None ) { + /** The failure of a several-element cell of `name` (SCHEMA-6 D6). */ + private def several(name: String): MultiValuedCell = + new MultiValuedCell(name, remedy(name).getOrElse(""), table) + /** Today's reading, the plan's answer for a key it does not type (ANY, NUMERIC, TEMPORAL) and * for a value a typed reader cannot read (D6, `Unreadable.KeepAsStored`). */ @@ -383,9 +498,13 @@ private[client] object HitReading { t.sqlType match { case SQLTypes.Array(_) => (list(name, t.element.getOrElse(SetOperationType(SQLTypes.Any))), None) - case SQLTypes.Struct if t.fields.nonEmpty => (scalar(name, struct(name, t, keepKeys)), None) - case SQLTypes.GeoPoint => (geo(name), None) - case _ if !typed(t) => (asRead, None) + // SCHEMA-6 (A-26): a list of objects in an `object` column answers a `List` of its struct + // reads whatever `MultiValued` says -- `ARRAY` is `nested`, and no in-place ALTER + // turns an object into one, so failing it would ship a refusal with no remedy + case SQLTypes.Struct if t.fields.nonEmpty => + (scalar(name, struct(name, t, keepKeys), MultiValued.AsRead), None) + case SQLTypes.GeoPoint => (geo(name), None) + case _ if !typed(t) => (asRead, None) case scalarType => val (one, ignoredOne) = scalarOne(name, scalarType, t.format) ( @@ -469,6 +588,11 @@ private[client] object HitReading { nested.foreach { case (k, sub) => if (!leaves.containsKey(k)) leaves.put(k, paths(s"$name.$k", sub.toList)) } + // SCHEMA-6 (A-26): the selected children, in selection order, read under a list of objects + val children: Array[(String, Array[String], JsonNode => Any)] = + (under.map(_._1).filter(_.indexOf('.') < 0) ++ nested.keys).distinct + .map(k => (k, Array(k), leaves.get(k))) + .toArray node => if (node.isObject) { val entries = ListMap.newBuilder[String, Any] @@ -481,6 +605,20 @@ private[client] object HitReading { ) } entries.result() + } else if (node.isArray && children.nonEmpty) { + // a list of objects: each selected path holds the values its elements hold, collected in + // order -- as Elasticsearch indexes `obj.a` = [5, 50] for `obj: [{a: 5}, {a: 50}]` --, + // read by the path's reader: one value is it, several fail a scalar path (Q1, its + // `SET FIELD` remedy) or are an `ARRAY` path's list, none is NULL + val entries = ListMap.newBuilder[String, Any] + var i = 0 + while (i < children.length) { + val (k, segments, reader) = children(i) + val values = collected(node, segments) + entries += aliases.getOrElse(k, k) -> (if (values eq null) null else reader(values)) + i += 1 + } + entries.result() } else asRead(node) } @@ -490,9 +628,21 @@ private[client] object HitReading { * with. */ def scalar(name: String, one: JsonNode => Any): JsonNode => Any = - node => scalarValue(name, one, node) + node => scalarValue(name, one, node, multiValued) + + /** [[scalar]] under the given [[MultiValued]] rather than the statement's. */ + def scalar(name: String, one: JsonNode => Any, mode: MultiValued): JsonNode => Any = + node => scalarValue(name, one, node, mode) private def scalarValue(name: String, one: JsonNode => Any, node: JsonNode): Any = + scalarValue(name, one, node, multiValued) + + private def scalarValue( + name: String, + one: JsonNode => Any, + node: JsonNode, + mode: MultiValued + ): Any = if (node == null || node.isNull) null else if (node.isArray) { val size = node.size() @@ -500,19 +650,19 @@ private[client] object HitReading { else if (size == 1) { val only = node.get(0) if (only != null && only.isArray) { - if (only.size() == 1) scalarValue(name, one, only) - else if (only.size() > 1 && multiValued == MultiValued.Fail) - throw new MultiValuedCell(name, table) + if (only.size() == 1) scalarValue(name, one, only, mode) + else if (only.size() > 1 && mode == MultiValued.Fail) + throw several(name) else asRead(node) } else element(one)(only) } else - multiValued match { + mode match { case MultiValued.AsRead => val values = ListBuffer.empty[Any] val it = node.elements() while (it.hasNext) values += element(one)(it.next()) values.toList - case MultiValued.Fail => throw new MultiValuedCell(name, table) + case MultiValued.Fail => throw several(name) } } else one(node) @@ -529,9 +679,11 @@ private[client] object HitReading { case _ if !typed(element) => asRead case scalarType => scalarOne(name, scalarType, element.format)._1 } + val points = element.sqlType == SQLTypes.GeoPoint node => if (node == null || node.isNull) null - else if (node.isArray) { + // SCHEMA-6: `[lon, lat]` in an `ARRAY` is ONE point, a list of one (R-34) + else if (node.isArray && !(points && coordinates(node))) { val values = ListBuffer.empty[Any] val it = node.elements() while (it.hasNext) { @@ -591,7 +743,7 @@ private[client] object HitReading { } values.toList } catch { case _: IllegalArgumentException => asRead(node) } - case MultiValued.Fail => throw new MultiValuedCell(name, table) + case MultiValued.Fail => throw several(name) } } else one(node) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 89226acf9..789f73db1 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -404,6 +404,38 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC } } + /** SCHEMA-6 (D11) -- `single` with each derived table and CTE reference typed from its body + * (`DerivedTable.attach`), every index source by [[lookupSchema]] (cached: one read per index + * per TTL): the typing API of the relational engine, which types the statements core refuses to + * run. It executes nothing, routes nothing and guards nothing -- the gateway's closure refusal + * and [[resolveWithSchema]] keep their order. A refused `UNION ALL` body fails with its refusal + * (400). + */ + private[elastic] def attachDerived(single: SingleSearch): ElasticResult[SingleSearch] = + app.softnetwork.elastic.sql.query.DerivedTable.attach(single, derivedLookup) match { + case Right(attached) => ElasticSuccess(attached) + case Left(refusal) => + ElasticFailure( + ElasticError(message = refusal, statusCode = Some(400), operation = Some("search")) + ) + } + + /** [[attachDerived]] of each branch of `multiple`. */ + private[elastic] def attachDerived(multiple: MultiSearch): ElasticResult[MultiSearch] = + multiple.requests + .foldLeft[ElasticResult[Vector[SingleSearch]]](ElasticSuccess(Vector.empty)) { + case (ElasticSuccess(done), branch) => attachDerived(branch).map(done :+ _) + case (failure, _) => failure + } + .map(branches => multiple.copy(requests = branches)) + + /** The schema of an index source for [[attachDerived]]: [[lookupSchema]]'s, `None` on a miss. */ + private def derivedLookup(sources: Seq[String]): Option[Schema] = + lookupSchema(sources) match { + case Some(Success(ElasticSuccess(schema))) => Some(schema) + case _ => None + } + /** The mapped type of a column PROJECTED BY AN INNER STATEMENT, when it can be known for free * (story 22.2). * @@ -2216,7 +2248,12 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC * reads the nested one, as Elasticsearch does. */ private def resolveSortKey(h: Map[String, Any], col: String): Any = - flatValue(h, col).orElse(findKeyValue(col, h)).orNull + flatValue(h, col) + .orElse(findKeyValue(col, h)) + // SCHEMA-6 (A-26): a path under a list of objects holds the values its elements hold, as + // Elasticsearch indexes -- and sorts -- it, never a NULL key + .orElse(ElasticConversion.collectedValue(col.split("\\."), h)) + .orNull private def extractRankings( row: ListMap[String, Any], diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index f38298c18..4c1285728 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -733,8 +733,10 @@ package object client extends SerializationApi { */ def instantRowKeys(context: ConversionContext): Seq[String] = context match { - case s: SchemaColumns if !s.reading.instants.isEmpty => - s.reading.instants.values().toArray(new Array[String](0)).toSeq + case s: SchemaColumns if !s.reading.instants.isEmpty || !s.reading.asReadKeys.isEmpty => + // SCHEMA-6 (A-28) -- the as-read partition keys are internal row keys too + s.reading.instants.values().toArray(new Array[String](0)).toSeq ++ + s.reading.asReadKeys.values().toArray(new Array[String](0)).toSeq case _ => Nil } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala index 7fb2bf0c9..1f0c88a18 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala @@ -95,6 +95,12 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { private def row(sql: String, source: String, extra: String = ""): ListMap[String, Any] = rows(sql, s"""{"_id": "a1", "_source": $source$extra}""").head + /** [[row]] under [[MultiValued.AsRead]]: a several-valued cell read as its list. SCHEMA-6 (D6) + * made [[MultiValued.Fail]] the default, so the cells pinning the list reading name the mode. + */ + private def rowAsRead(sql: String, source: String, extra: String = ""): ListMap[String, Any] = + rows(sql, s"""{"_id": "a1", "_source": $source$extra}""", multiValued = MultiValued.AsRead).head + private def utc(y: Int, mo: Int, d: Int, h: Int = 0, mi: Int = 0): ZonedDateTime = ZonedDateTime.of(y, mo, d, h, mi, 0, 0, ZoneOffset.UTC) @@ -166,9 +172,11 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { row("SELECT c_des FROM s2_t", """{"c_des": [[1706697000]]}""")("c_des") shouldBe LocalDate.of(2024, 1, 31) // several values in an array of one: as stored, its nesting kept (D6), never a failure - row("SELECT c_des FROM s2_t", """{"c_des": [[1706697000, 1706697001]]}""")("c_des") shouldBe + rowAsRead("SELECT c_des FROM s2_t", """{"c_des": [[1706697000, 1706697001]]}""")( + "c_des" + ) shouldBe List(List(1706697000, 1706697001)) - row("SELECT obj FROM s2_t", """{"obj": {"a": [[1, 2]], "d": [["2024-01-31"]]}}""")( + rowAsRead("SELECT obj FROM s2_t", """{"obj": {"a": [[1, 2]], "d": [["2024-01-31"]]}}""")( "obj" ) shouldBe ListMap("a" -> List(List(1, 2)), "d" -> utc(2024, 1, 31)) // ...a statement failing several values fails it too (D7) @@ -194,18 +202,20 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { } it should "read several values per MultiValued (multi)" in { - row("SELECT n FROM s2_t", """{"n": [1, "2"]}""")("n") shouldBe List(1, 2) + rows( + "SELECT n FROM s2_t", + """{"_id": "a1", "_source": {"n": [1, "2"]}}""", + multiValued = MultiValued.AsRead + ).head("n") shouldBe List(1, 2) + // SCHEMA-6 (D6, lead Q1): the default fails, naming the ALTER that parses and declares the list + HitReading.Default shouldBe MultiValued.Fail val failure = intercept[Exception] { - rows( - "SELECT n FROM s2_t", - """{"_id": "doc7", "_source": {"n": [1, 2]}}""", - multiValued = MultiValued.Fail - ) + rows("SELECT n FROM s2_t", """{"_id": "doc7", "_source": {"n": [1, 2]}}""") } val message = Option(failure.getCause).fold(failure.getMessage)(_.getMessage) message should include("Column n") message should include("doc7") - message should include("ALTER TABLE s2_t ALTER COLUMN n TYPE ARRAY") + message should include("ALTER TABLE s2_t ALTER COLUMN n SET DATA TYPE ARRAY") } it should "answer a date Elasticsearch ignored NULL, a keyword it ignored as stored (ignored)" in { @@ -220,7 +230,11 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { row("SELECT dm FROM s2_t", """{"dm": "not-a-date"}""")("dm") shouldBe "not-a-date" // Elasticsearch's verdict is per VALUE: in an array, the values it indexed keep theirs and only // the ones it refused are NULL - row("SELECT dm FROM s2_t", """{"dm": ["2024-01-01", "bad"]}""", """, "_ignored": ["dm"]""")( + rowAsRead( + "SELECT dm FROM s2_t", + """{"dm": ["2024-01-01", "bad"]}""", + """, "_ignored": ["dm"]""" + )( "dm" ) shouldBe List(utc(2024, 1, 1), null) row("SELECT dm FROM s2_t", """{"dm": ["bad"]}""", """, "_ignored": ["dm"]""")("dm") shouldBe @@ -278,7 +292,7 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { row("SELECT g FROM s2_t", """{"g": "not a point"}""")("g") shouldBe "not a point" row("SELECT c_c1 FROM s2_t", """{"c_c1": "31-01-2024"}""")("c_c1") shouldBe "31-01-2024" // several points, read one by one (D7's AsRead) - row("SELECT g FROM s2_t", """{"g": ["1.5,2.5", "3.5,4.5"]}""")("g") shouldBe List( + rowAsRead("SELECT g FROM s2_t", """{"g": ["1.5,2.5", "3.5,4.5"]}""")("g") shouldBe List( ListMap("lat" -> 1.5, "lon" -> 2.5), ListMap("lat" -> 3.5, "lon" -> 4.5) ) diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MultiValuedFailSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MultiValuedFailSpec.scala new file mode 100644 index 000000000..ab8c98924 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/MultiValuedFailSpec.scala @@ -0,0 +1,287 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{ + AlterColumnField, + AlterColumnType, + AlterTable, + SingleSearch +} +import app.softnetwork.elastic.sql.schema.Schema +import app.softnetwork.elastic.sql.`type`.SQLTypes +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.collection.immutable.ListMap + +/** SCHEMA-6 (D6, A-26, lead Q1) -- several values in a column NOT declared `ARRAY` fail the + * statement, by default, naming the column, the document, the index it came from and the `ALTER` + * that declares it a list -- a statement that parses and runs (meta-only, + * `ArrayDeclarationDiffSpec`). After that `ALTER`, the column reads its values as a list. + * + * Reached at both places a scalar reader meets several values: a hit's `_source` + * (`parseHitsByPlan`) and a top hit of FIRST_VALUE / LAST_VALUE / ARRAY_AGG (`readElement`), where + * the bare column name escaped before. A list of objects in an `object` column keeps answering its + * list of struct reads (no in-place remedy exists for it, A-26); a scalar path under it holds the + * values its elements hold -- Elasticsearch's `obj.a` -- and fails or reads them as any column + * does. An entity read (a WHERE subquery's inner statement) never goes through the plan (A-24). + */ +class MultiValuedFailSpec extends AnyFlatSpec with Matchers { + + private val version = "8.18.3" + + /** `zs_ext`, created by a raw mapping: no declaration but the mapping's. */ + private val raw: Schema = Index( + "zs_ext", + """{"zs_ext": {"aliases": {}, "mappings": {"properties": { + | "id": {"type": "keyword"}, "tags": {"type": "keyword"}, "n": {"type": "integer"}, + | "geo": {"type": "geo_point"}, "ts": {"type": "date"}, + | "obj": {"properties": {"a": {"type": "integer"}, "b": {"type": "keyword"}}} + |}}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + ).schema + + private def statement(sql: String, schema: Schema): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(schema)) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + private def outputs(single: SingleSearch): Seq[String] = { + val fields = single.select.fieldsWithComputedAliases + if (fields.size == 1 && fields.head.identifier.identifierName == "*") Seq.empty + else fields.map(_.outputName) + } + + private def rows( + sql: String, + hits: String, + schema: Schema = raw, + context: SingleSearch => ConversionContext = s => + SchemaColumns.of(s, NativeContext, Some(version)) + ): Seq[ListMap[String, Any]] = { + val single = statement(sql, schema) + val aggregations = ListMap(single.sqlAggregations.toSeq.map { case (k, v) => + k -> sqlAggregationToClientAggregation(v) + }: _*) + ElasticConversion + .parseResponse( + s"""{"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [$hits]}}""", + single.fieldAliases, + aggregations, + outputs(single) + )(context(single)) + .get + } + + private def failure(body: => Any): String = { + val e = intercept[Exception](body) + Option(e.getCause).fold(e.getMessage)(_.getMessage) + } + + /** The ALTER TABLE statement a failure names, re-parsed. */ + private def remedy(message: String): AlterTable = { + val at = message.indexOf("ALTER TABLE") + at should be >= 0 + Parser(message.substring(at)) match { + case Right(alter: AlterTable) => alter + case other => fail(s"the remedy of [$message] does not parse: $other") + } + } + + private def altered(schema: Schema, alter: AlterTable): Schema = schema.merge(alter.statements) + + private val doc = """"_index": "zs_ext", "_id": "d1"""" + + behavior of "several values in a scalar column (D6)" + + it should "fail by default, naming the column, the document, its index and a parseable ALTER" in { + HitReading.Default shouldBe MultiValued.Fail + val message = failure( + rows("SELECT id, tags FROM zs_ext", s"""{$doc, "_source": {"id": "1", "tags": ["a", "b"]}}""") + ) + message should include("Column tags") + message should include("document d1") + message should include("index zs_ext") + val alter = remedy(message) + alter shouldBe AlterTable( + "zs_ext", + ifExists = false, + List(AlterColumnType("tags", SQLTypes.Array(SQLTypes.Keyword))) + ) + alter.sql should include("SET DATA TYPE ARRAY") + } + + it should "read the list once the ALTER has declared it, a scalar as a list of one, NULL as NULL" in { + val alter = remedy( + failure(rows("SELECT id, tags FROM zs_ext", s"""{$doc, "_source": {"tags": ["a", "b"]}}""")) + ) + val declared = altered(raw, alter) + declared.find("tags").map(_.dataType) shouldBe Some(SQLTypes.Array(SQLTypes.Keyword)) + val read = rows( + "SELECT id, tags FROM zs_ext ORDER BY id", + """{"_id": "1", "_source": {"id": "1", "tags": ["a", "b"]}}, + |{"_id": "2", "_source": {"id": "2", "tags": "c"}}, + |{"_id": "3", "_source": {"id": "3"}}""".stripMargin, + declared + ) + read.map(_("tags")) shouldBe Seq(List("a", "b"), List("c"), null) + } + + it should "name the index the hit came from, not the statement's pattern" in { + val message = failure( + rows( + "SELECT id, n FROM zs_*", + """{"_index": "zs_ext_2024", "_id": "d9", "_source": {"n": [1, 2]}}""" + ) + ) + message should include("index zs_ext_2024") + remedy(message).table shouldBe "zs_ext_2024" + remedy(message).statements shouldBe List(AlterColumnType("n", SQLTypes.Array(SQLTypes.Int))) + } + + it should "name the declared TIMESTAMP of an undeclared date and the point of a geo column" in { + remedy( + failure( + rows( + "SELECT ts FROM zs_ext", + s"""{$doc, "_source": {"ts": ["2024-01-01", "2024-01-02"]}}""" + ) + ) + ).statements shouldBe List(AlterColumnType("ts", SQLTypes.Array(SQLTypes.Timestamp))) + val geo = remedy( + failure( + rows("SELECT geo FROM zs_ext", s"""{$doc, "_source": {"geo": [[1.0, 2.0], [3.0, 4.0]]}}""") + ) + ) + geo.statements shouldBe List(AlterColumnType("geo", SQLTypes.Array(SQLTypes.GeoPoint))) + // a [lon, lat] pair is ONE point, before and after the ALTER + rows("SELECT geo FROM zs_ext", s"""{$doc, "_source": {"geo": [2.0, 1.0]}}""") + .head("geo") shouldBe ListMap("lat" -> 1.0, "lon" -> 2.0) + val points = rows( + "SELECT geo FROM zs_ext", + s"""{$doc, "_source": {"geo": [2.0, 1.0]}}, {"_id": "d2", "_source": {"geo": [[2.0, 1.0], [4.0, 3.0]]}}""", + altered(raw, geo) + ) + points.map(_("geo")) shouldBe Seq( + List(ListMap("lat" -> 1.0, "lon" -> 2.0)), + List(ListMap("lat" -> 1.0, "lon" -> 2.0), ListMap("lat" -> 3.0, "lon" -> 4.0)) + ) + } + + it should "fail a several-valued top hit of FIRST_VALUE / ARRAY_AGG with the same message" in { + def global(sql: String, aggregations: String): Seq[ListMap[String, Any]] = { + val single = statement(sql, raw) + val clients = ListMap(single.sqlAggregations.toSeq.map { case (k, v) => + k -> sqlAggregationToClientAggregation(v) + }: _*) + ElasticConversion + .parseResponse( + s"""{"hits": {"total": {"value": 1, "relation": "eq"}, "hits": []}, + | "aggregations": {$aggregations}}""".stripMargin, + single.fieldAliases, + clients, + outputs(single) + )(SchemaColumns.of(single, NativeContext, Some(version))) + .get + } + val first = failure( + global( + "SELECT FIRST(tags) AS f FROM zs_ext", + """"f": {"hits": {"hits": [{"_index": "zs_ext", "_id": "t1", "_source": {"tags": ["a", "b"]}}]}}""" + ) + ) + first should include("Column tags") + first should include("document t1") + first should include("index zs_ext") + remedy(first).statements shouldBe List( + AlterColumnType("tags", SQLTypes.Array(SQLTypes.Keyword)) + ) + val agg = failure( + global( + "SELECT ARRAY_AGG(tags) AS a FROM zs_ext", + """"a": {"hits": {"hits": [{"_index": "zs_ext", "_id": "t2", "_source": {"tags": "x"}}, + | {"_index": "zs_ext", "_id": "t3", "_source": {"tags": ["a", "b"]}}]}}""".stripMargin + ) + ) + agg should include("document t3") + remedy(agg).statements shouldBe List(AlterColumnType("tags", SQLTypes.Array(SQLTypes.Keyword))) + } + + it should "keep an entity read (a WHERE subquery's inner statement) as read (A-24)" in { + rows( + "SELECT tags FROM zs_ext", + s"""{$doc, "_source": {"tags": ["a", "b"]}}""", + context = _ => EntityContext + ).head("tags") shouldBe List("a", "b") + } + + behavior of "a list of objects in an object column (A-26)" + + private val objects = + s"""{$doc, "_source": {"id": "1", "obj": [{"a": 5, "b": "x"}, {"a": 50}]}}""" + + it should "answer the STRUCT column's list of struct reads, as main did" in { + rows("SELECT id, obj FROM zs_ext", objects).head("obj") shouldBe List( + ListMap("a" -> 5, "b" -> "x"), + ListMap("a" -> 50) + ) + } + + it should "fail a scalar path under it, naming SET FIELD, then read the elements' values" in { + val message = failure(rows("SELECT id, obj.a FROM zs_ext", objects)) + message should include("Column obj.a") + val alter = remedy(message) + alter.statements match { + case List(AlterColumnField("obj", field, false)) => + field.name shouldBe "a" + field.dataType shouldBe SQLTypes.Array(SQLTypes.Int) + case other => fail(s"remedy $other") + } + val declared = altered(raw, alter) + rows("SELECT id, obj.a FROM zs_ext", objects, declared).head("obj.a") shouldBe List(5, 50) + // beside the parent read whole, in each spelling + val both = rows("SELECT id, obj, obj.a FROM zs_ext", objects, declared).head + both("obj.a") shouldBe List(5, 50) + // the parent's elements read `a` by its declared type too: a list of one + both("obj") shouldBe List(ListMap("a" -> List(5), "b" -> "x"), ListMap("a" -> List(50))) + rows("SELECT *, obj.a FROM zs_ext", objects, declared).head("obj.a") shouldBe List(5, 50) + failure(rows("SELECT id, obj, obj.a FROM zs_ext", objects)) should include("Column obj.a") + } + + it should "read one value under a list of objects as that value, none as NULL, never NULL by the list" in { + rows( + "SELECT id, obj.a FROM zs_ext", + s"""{$doc, "_source": {"obj": [{"a": 5}, {"b": "x"}]}}""" + ).head("obj.a") shouldBe 5 + rows( + "SELECT id, obj.a FROM zs_ext", + s"""{$doc, "_source": {"obj": [{"b": "x"}]}}""" + ).head("obj.a") shouldBe (null: Any) + // an element's own list is flattened, as Elasticsearch indexes `obj.a` + val declared = altered( + raw, + remedy(failure(rows("SELECT id, obj.a FROM zs_ext", objects))) + ) + rows( + "SELECT id, obj.a FROM zs_ext", + s"""{$doc, "_source": {"obj": [{"a": [1, 2]}, {"a": 3}]}}""", + declared + ).head("obj.a") shouldBe List(1, 2, 3) + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala index b760b4fd3..189a457cf 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala @@ -106,12 +106,15 @@ class ResultSchemaSpec extends AnyFlatSpec with Matchers { ) shouldBe Seq(SQLTypes.Varchar) } - it should "give a STRUCT its fields by name, a point {lat, lon}, a list its element (A-13)" in { + // SCHEMA-6 (D9) supersedes A-13's by-name order: a STRUCT's fields in its column's order -- the + // mapping's here, which has no `_meta.field_order` (Elasticsearch returns `properties` sorted, this + // hand-written mapping lists `z` first) + it should "give a STRUCT its fields in its column's order, a point {lat, lon}, a list its element (D9)" in { val obj = schemaOf("SELECT obj, g, items FROM s2_t").columns obj.head.sqlType shouldBe SQLTypes.Struct obj.head.fields.map(f => f.name -> f.sqlType) shouldBe Seq( - "a" -> SQLTypes.Int, - "z" -> SQLTypes.Varchar + "z" -> SQLTypes.Varchar, + "a" -> SQLTypes.Int ) obj(1).sqlType shouldBe SQLTypes.GeoPoint obj(1).fields.map(f => f.name -> f.sqlType) shouldBe Seq( diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala index 4a4ea65cf..27a73eb09 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala @@ -238,13 +238,14 @@ class SetOperationLegReadersSpec extends AnyFlatSpec with Matchers { if got != shown(value) } yield s"$item <- $json read $got instead of ${shown(value)}" wrong shouldBe empty - // a struct's ListMap keeps its order; a TIMESTAMP is in UTC + // a struct's ListMap keeps its order -- its column's declared order (SCHEMA-6 D9; by name + // before); a TIMESTAMP is in UTC legs.head.get .apply(12)(received(row(12))) .asInstanceOf[ListMap[String, Any]] .keys .toSeq shouldBe - Seq("e", "n", "p", "s") + Seq("n", "e", "s", "p") legs.head.get .apply(8)(received(row(8))) .asInstanceOf[ZonedDateTime] diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala index 902b4db33..3ec2947e7 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala @@ -407,9 +407,10 @@ class SetOperationLegSchemasSpec extends AnyFlatSpec with Matchers { wrong shouldBe empty slot(union("obj", "obj")).read(received("""{"n":1,"e":1706697000}""")) shouldBe ListMap("e" -> at1030, "n" -> 1) + // in the column's declared order (SCHEMA-6 D9; by name before) slot(union("obj", "obj")).sqlType.fields.map(f => f.name -> f.valueType) shouldBe Seq( - "e" -> SetOperationType(SQLTypes.Timestamp, format = Some("epoch_second")), - "n" -> SetOperationType(SQLTypes.Int) + "n" -> SetOperationType(SQLTypes.Int), + "e" -> SetOperationType(SQLTypes.Timestamp, format = Some("epoch_second")) ) slot(union("ae", "ae")).sqlType.element shouldBe Some(SetOperationType(SQLTypes.Timestamp, format = Some("epoch_second"))) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 883058440..ae926cb79 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -126,6 +126,30 @@ The SQL Gateway supports the following type system: | `STRUCT` | `object` with nested properties | | `ARRAY` | `nested` | | `GEO_POINT` | `geo_point` | +| `VARBINARY` | `binary` | +| `ARRAY` | the mapping of `X` (Elasticsearch has no list type: every field holds one value or several) | + +`ARRAY` is accepted for every type `X` above — `ARRAY`, `ARRAY`, `ARRAY`, +`ARRAY` (`binary`), `ARRAY` (`geo_point`), `ARRAY` (`nested`) — and +reads back as `ARRAY`: `DESCRIBE TABLE`, `SHOW CREATE TABLE`, the result schema and the type +checks all see the list, and each value of the column is a list of `X` values (a single value is +a list of one) (*since `0.25.0`*; the type read `ANY` before, and `CHAR` now reads back as `CHAR`). +A column declared `ARRAY` is used as an `X` where Elasticsearch works on each element: a +comparison (`WHERE tags = 'a'` matches a document holding `a` among its values), a function, a +`GROUP BY` key (its groups are the elements), a metric (`MIN(arr)` is the smallest element). +A computed column is the exception: its `SCRIPT AS` expression runs at ingest on the whole list, +not on one element, so a numeric or temporal function of an `ARRAY` column there +(`c INT SCRIPT AS (arr + 1)`, `SCRIPT AS (YEAR(ad))`) is refused by `CREATE TABLE` and by +`ALTER TABLE`, naming the column. So are a `NULLIF` that compares the list with a value +(`NULLIF(arr, arr)`, `NULLIF(n, arr)`, `NULLIF(tags, 'a')`) and date arithmetic on it +(`ad - d`): an `ALTER TABLE` of a table holding an `ARRAY` column judges its computed columns +exactly as `CREATE TABLE` does (*since `0.25.0`*; such an `ALTER TABLE` used to re-map the list +column to an `object` field). + +A **list of lists** (`ARRAY>`) is refused by every statement that declares a type — +`CREATE TABLE`, `ADD COLUMN`, `SET DATA TYPE`, `SET FIELD`, and a `CREATE TABLE … AS SELECT` whose +column would be one (`ARRAY_AGG` of a list column) —, naming the column: Elasticsearch flattens a +nested array into one list of values. A length, precision or scale may be written on any type SQL parameterises — `c VARCHAR(255)`, `c CHAR(1)`, `c DECIMAL(10,2)`, `c BIGINT(20)` — and is **accepted and ignored**, exactly as in a @@ -230,6 +254,12 @@ CREATE TABLE store ( - `OPTIONS` - `SCRIPT AS` — optionally `STORED` (except inside ARRAY\) - Multi-level nesting is supported. +- Sub-fields keep the order they are **declared** in: `DESCRIBE TABLE`, `SHOW CREATE TABLE` and + the result schema list `obj STRUCT FIELDS (y INT, b KEYWORD)` as `y, b` (*since `0.25.0`*). + Elasticsearch returns a mapping's properties sorted by name, so the declared order is recorded + in the table's `_meta` (`field_order`); a table that has none — created by an earlier version, + or by a raw mapping — keeps its mapping's order, and `ALTER COLUMN … SET FIELD` rewrites a + sub-field in place. --- @@ -715,6 +745,23 @@ Type changes fall into two categories: This is the only case where an `ALTER TABLE` operation can be rejected for safety reasons. All other ALTER operations (adding/dropping columns, renaming, modifying options, modifying nested fields, etc.) are allowed. +#### Declaring an existing field a list + +`ARRAY` maps to the same Elasticsearch field as `X`, so declaring an existing field a list of +its own type changes only the table's `_meta`: it is applied **in place**, with no reindex +(*since `0.25.0`*): + +```sql +ALTER TABLE logs ALTER COLUMN tags SET DATA TYPE ARRAY; +-- a sub-field: SET FIELD rewrites the whole sub-field, so it is written as declared, typed ARRAY +ALTER TABLE logs ALTER COLUMN obj SET FIELD a ARRAY; +``` + +This is the statement a query names when a column holds several values but is declared as one +(see [DQL: columns holding several values](dql_statements.md#columns-holding-several-values)). +Changing the element type of a list column (`ARRAY` → `ARRAY` or `ARRAY`), +or declaring a list of another Elasticsearch type (`KEYWORD` → `ARRAY`), stays `Impossible`. + --- ## DROP TABLE diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index a51050cf5..542c9bf3c 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -27,6 +27,7 @@ DQL supports: - [Qualified and quoted table names](#qualified-and-quoted-table-names) - [FROM-less SELECT (connection handshake)](#from-less-select-connection-handshake) - [WHERE](#where) +- [Columns holding several values](#columns-holding-several-values) - [ORDER BY](#order-by) - [LIMIT / OFFSET](#limit--offset) - [Set operators](#set-operators) @@ -552,6 +553,43 @@ WHERE event_ts >= '2026-06-04T00:00:00' -- what Elasticsearch's default --- +## Columns holding several values + +Elasticsearch has no list type: any field may hold one value or several (`"tags": ["a", "b"]`). A +column declared `ARRAY` reads each value as a list of `X` values — a single value is a list of +one —, and `DESCRIBE TABLE` and the result schema type it `ARRAY`. + +A column **not** declared a list that holds several values in a document fails the statement +(*since `0.25.0`*; earlier versions answered the list as read), naming the column, the document, +its index (the index the document is stored in, also when the statement reads an index pattern or +an alias) and the statement that declares the column a list: + +```sql +SELECT id, tags FROM logs; +-- Error: Column tags holds several values in document d1 of index logs, but is declared as one +-- value: declare it a list with ALTER TABLE logs ALTER COLUMN tags SET DATA TYPE ARRAY +``` + +That `ALTER` changes only the table's `_meta` and runs in place, with no reindex (see +[DDL: declaring an existing field a list](ddl_statements.md#declaring-an-existing-field-a-list)); +the statement then answers `List(a, b)`. A sub-field is declared a list with `SET FIELD`, which the +message spells out with the sub-field's own options: +`ALTER TABLE logs ALTER COLUMN obj SET FIELD a ARRAY`. The same failure is raised by a top hit +of `FIRST_VALUE`, `LAST_VALUE` or `ARRAY_AGG`, naming that hit's document. A `WHERE` subquery's +inner statement is not affected. + +A **list of objects** in a `STRUCT` column (an `object` mapping, `"obj": [{"a": 5}, {"a": 50}]`) +keeps answering: the column's value is the list of its objects, so a `STRUCT` column may hold a +list. A path under it (`SELECT obj.a`) holds the values its objects hold, in order — `[5, 50]`, as +Elasticsearch indexes `obj.a` (it answered NULL before) — and is read like any column: one value +is that value, several fail as above until the sub-field is declared `ARRAY`, none is NULL. + +A column declared `ARRAY` is used as an `X` wherever Elasticsearch works on each element — +`WHERE arr = 5`, `arr > 5` and `arr IN (…)` match a document holding a matching element, a function +reads one element, `GROUP BY arr` groups by element and `MIN(arr)` is the smallest element; a +`PARTITION BY` over it is read as the earlier versions read it. `ARRAY_AGG(x)` answers +`ARRAY`. + ## ORDER BY `ORDER BY` sorts the result set by one or more expressions. @@ -936,7 +974,10 @@ A value with several values — a multi-valued field — has no one value of its at a position that converts its values it fails the statement, in the words DuckDB uses for the same failure: `Conversion Error: Unimplemented type for cast (INTEGER -> VARCHAR[]) when casting from source column k` for a multi-valued keyword beside an `INT`. A position whose branches have -one type converts nothing, and answers such a value as its branch reads it. +one type converts nothing, and answers such a value as its branch reads it: a list for a column +declared `ARRAY`, and the failure of +[a column holding several values](#columns-holding-several-values) for any other (*since +`0.25.0`*). ```sql -- id is a KEYWORD, n an INT: one VARCHAR column ('r1', ..., '3', ...) @@ -1967,6 +2008,9 @@ Notes: - `PARTITION BY` defines the grouping key. - `ORDER BY` inside `OVER` defines the window ordering. - `LIMIT` inside `ARRAY_AGG` restricts the collected values. +- `ARRAY_AGG(x)` is typed `ARRAY` (*since `0.25.0`*; it was typed as `x`): `DESCRIBE`, + the result schema and a `CREATE TABLE … AS SELECT` column (`_meta` `ARRAY`, mapped + `keyword`) all read the list. Beside a scalar column in a set operation it is refused. - Frame clauses (`ROWS BETWEEN ...`) are not exposed; the engine uses a default frame per function semantics. --- diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientArrayRoundTripSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientArrayRoundTripSpec.scala new file mode 100644 index 000000000..c2342315a --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientArrayRoundTripSpec.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientArrayRoundTripSpec extends ArrayRoundTripSpec with ElasticDockerTestKit { + override lazy val client: GatewayApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 8f3532f56..6cd6d720b 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -90,8 +90,8 @@ trait JavaClientConversion { _: JavaClientCompanion => * * This is the single-parse hits path (softclient4es-arrow#160): each `_source` node is * re-parented into the envelope untouched — no serialization, no re-parse. The row parser reads - * exactly `_id`, `_source`, `_ignored`, `inner_hits` and `fields` per hit, so hits carrying - * `inner_hits` or `fields` (UNNEST legs, script fields) fall back to a whole-hit + * exactly `_id`, `_index`, `_source`, `_ignored`, `inner_hits` and `fields` per hit, so hits + * carrying `inner_hits` or `fields` (UNNEST legs, script fields) fall back to a whole-hit * [[convertToTree]] to keep full shape fidelity — still token-level, never a string. */ protected def hitsToResponseNode(hits: _root_.java.util.List[Hit[ObjectNode]]): ObjectNode = { @@ -101,6 +101,9 @@ trait JavaClientConversion { _: JavaClientCompanion => if (hit.innerHits().isEmpty && hit.fields().isEmpty) { val hitNode = hitsArray.addObject() Option(hit.id()).foreach(id => hitNode.put("_id", id)) + // SCHEMA-6 (D6) -- the index the hit came from: a several-valued cell names it, and the + // `ALTER TABLE` remedy it proposes, never the statement's pattern (which no ALTER accepts) + Option(hit.index()).foreach(index => hitNode.put("_index", index)) Option(hit.source()).foreach(source => hitNode.set[JsonNode]("_source", source)) // SCHEMA-2 (D8) -- the fields Elasticsearch ignored (`ignore_malformed`): core answers a // date it ignored as NULL, as the clients reading the raw body do diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala index 551704f84..d7fa3c62f 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala @@ -370,4 +370,46 @@ class JavaClientSchemaConsumerSpec extends AnyFlatSpecLike with ElasticDockerTes } counts shouldBe docs.size.toLong } + + // SCHEMA-6 (D11, R-6): the relational engine's typing call, as arrow will make it at SCHEMA-10 -- + // its branches with their schemas attached, then `attachDerived` for the derived ones, then + // `columnTree` over them. Arrow's own call today leaves a derived branch unattached: ANY. + "The relational engine" should "type a derived branch through attachDerived, unattached ANY (D11)" in { + val sql = + s"SELECT ts AS k FROM $table UNION ALL SELECT q.c1 FROM (SELECT n AS c1 FROM $table) q" + val statement = Parser(sql) match { + case Right(m: MultiSearch) => m + case other => fail(s"[$sql] $other") + } + // unattached: the derived column is ANY, nothing refused (R-6, as arrow runs today) + val unattached = statement.requests.map(_.update()) + MultiSearch.columnTree(unattached, statement.resolvedOperators).isRight shouldBe true + MultiSearch + .columnTypes(unattached, statement.resolvedOperators) + .map(_.head.branches.last.map(_.sqlType)) shouldBe Right( + Some(app.softnetwork.elastic.sql.`type`.SQLTypes.Any) + ) + // attached: the derived column is INT, the TIMESTAMP beside it #420 rule 1's refusal + client.attachDerived(statement) match { + case ElasticSuccess(attached) => + MultiSearch.columnTree(attached.requests, attached.resolvedOperators) shouldBe Left( + "Set operation branches must project compatible types at column 1: branch 1 'k' is " + + "TIMESTAMP, branch 2 'c1' is INT" + ) + case other => fail(s"attachDerived: $other") + } + // a KEYWORD beside it answers, typed VARCHAR (DuckDB 1.5.5) + Parser( + s"SELECT k FROM $table UNION ALL SELECT q.c1 FROM (SELECT n AS c1 FROM $table) q" + ) match { + case Right(m: MultiSearch) => + client.attachDerived(m) match { + case ElasticSuccess(attached) => + MultiSearch.columnTree(attached.requests, attached.resolvedOperators).isRight shouldBe + true + case other => fail(s"attachDerived: $other") + } + case other => fail(s"$other") + } + } } diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientArrayRoundTripSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientArrayRoundTripSpec.scala new file mode 100644 index 000000000..b8f696159 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientArrayRoundTripSpec.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientArrayRoundTripSpec extends ArrayRoundTripSpec with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala index 90d6148ed..d74dffe92 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala @@ -33,10 +33,11 @@ import _root_.java.util.{Collections, List => JList} /** Pins the envelope contract of the single-parse hits path (softclient4es-arrow#160). * - * The row parser (`ElasticConversion.parseSimpleHits`) reads exactly `_id`, `_source`, + * The row parser (`ElasticConversion.parseSimpleHits`) reads exactly `_id`, `_index`, `_source`, * `inner_hits` and `fields` per hit — `hitsToResponseNode` must provide those and may omit - * everything else. A hit carrying `inner_hits` or `fields` must fall back to a full-fidelity node, - * and `convertToTree` must produce the same tree `convertToJson` round-trips to. + * everything else (`_index` names the hit's index in a several-valued cell's failure, SCHEMA-6). A + * hit carrying `inner_hits` or `fields` must fall back to a full-fidelity node, and + * `convertToTree` must produce the same tree `convertToJson` round-trips to. */ class JavaClientConversionSpec extends AnyWordSpec with Matchers { @@ -87,7 +88,8 @@ class JavaClientConversionSpec extends AnyWordSpec with Matchers { val hitNode = hits.get(0) hitNode.path("_id").asText() shouldBe "1" hitNode.get("_source") should be theSameInstanceAs src - hitNode.has("_index") shouldBe false + // the hit's own index, never the statement's pattern (SCHEMA-6 D6) + hitNode.path("_index").asText() shouldBe "idx" } "omit _id when the hit has none" in { diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 8f3532f56..6cd6d720b 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -90,8 +90,8 @@ trait JavaClientConversion { _: JavaClientCompanion => * * This is the single-parse hits path (softclient4es-arrow#160): each `_source` node is * re-parented into the envelope untouched — no serialization, no re-parse. The row parser reads - * exactly `_id`, `_source`, `_ignored`, `inner_hits` and `fields` per hit, so hits carrying - * `inner_hits` or `fields` (UNNEST legs, script fields) fall back to a whole-hit + * exactly `_id`, `_index`, `_source`, `_ignored`, `inner_hits` and `fields` per hit, so hits + * carrying `inner_hits` or `fields` (UNNEST legs, script fields) fall back to a whole-hit * [[convertToTree]] to keep full shape fidelity — still token-level, never a string. */ protected def hitsToResponseNode(hits: _root_.java.util.List[Hit[ObjectNode]]): ObjectNode = { @@ -101,6 +101,9 @@ trait JavaClientConversion { _: JavaClientCompanion => if (hit.innerHits().isEmpty && hit.fields().isEmpty) { val hitNode = hitsArray.addObject() Option(hit.id()).foreach(id => hitNode.put("_id", id)) + // SCHEMA-6 (D6) -- the index the hit came from: a several-valued cell names it, and the + // `ALTER TABLE` remedy it proposes, never the statement's pattern (which no ALTER accepts) + Option(hit.index()).foreach(index => hitNode.put("_index", index)) Option(hit.source()).foreach(source => hitNode.set[JsonNode]("_source", source)) // SCHEMA-2 (D8) -- the fields Elasticsearch ignored (`ignore_malformed`): core answers a // date it ignored as NULL, as the clients reading the raw body do diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala index 90d6148ed..d74dffe92 100644 --- a/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala @@ -33,10 +33,11 @@ import _root_.java.util.{Collections, List => JList} /** Pins the envelope contract of the single-parse hits path (softclient4es-arrow#160). * - * The row parser (`ElasticConversion.parseSimpleHits`) reads exactly `_id`, `_source`, + * The row parser (`ElasticConversion.parseSimpleHits`) reads exactly `_id`, `_index`, `_source`, * `inner_hits` and `fields` per hit — `hitsToResponseNode` must provide those and may omit - * everything else. A hit carrying `inner_hits` or `fields` must fall back to a full-fidelity node, - * and `convertToTree` must produce the same tree `convertToJson` round-trips to. + * everything else (`_index` names the hit's index in a several-valued cell's failure, SCHEMA-6). A + * hit carrying `inner_hits` or `fields` must fall back to a full-fidelity node, and + * `convertToTree` must produce the same tree `convertToJson` round-trips to. */ class JavaClientConversionSpec extends AnyWordSpec with Matchers { @@ -87,7 +88,8 @@ class JavaClientConversionSpec extends AnyWordSpec with Matchers { val hitNode = hits.get(0) hitNode.path("_id").asText() shouldBe "1" hitNode.get("_source") should be theSameInstanceAs src - hitNode.has("_index") shouldBe false + // the hit's own index, never the statement's pattern (SCHEMA-6 D6) + hitNode.path("_index").asText() shouldBe "idx" } "omit _id when the hit has none" in { diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 516fdd82e..a462c830d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -133,10 +133,11 @@ package object schema { case _ => None } - val fields = + // in MAPPING order (SCHEMA-6 D9): a `Set` of the entries, mapped, lost it to a hash order + val mapped = Option(node.get("fields")) // multi-fields .orElse(Option(node.get("properties"))) // object/nested fields - .map(_.properties().asScala.map { entry => + .map(_.properties().asScala.toList.map { entry => val name = entry.getKey val value = entry.getValue val _meta = fields_meta.flatMap { @@ -148,9 +149,21 @@ package object schema { case _ => None } apply(name, value, _meta) - }.toList) + }) .getOrElse(Nil) + // SCHEMA-6 (D9): the sub-fields in the order the column DECLARED them (`_meta.field_order`), + // a name it does not list after them in mapping order, a listed name the mapping lacks + // ignored; without the list, the mapping's order (Elasticsearch's sorted `properties`) + val fields = _meta.flatMap(_.value.get("field_order")) match { + case Some(order: StringValues) if mapped.nonEmpty => + val declared = order.values.map(_.value) + val byName = mapped.map(f => f.name -> f).toMap + declared.distinct.flatMap(byName.get).toList ++ + mapped.filterNot(f => declared.contains(f.name)) + case _ => mapped + } + val options = extractObject(node, ignoredKeys = Set("type", "null_value", "fields", "properties")) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 5f8e6e06a..acefe9b63 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.query.{Bucket, BucketPath, Field, Limit, OrderBy, SingleSearch} import app.softnetwork.elastic.sql.{Expr, Identifier, Null, TokenRegex, Updateable} @@ -67,6 +67,13 @@ package object aggregate { * DATE))` is a DATE. * - every other aggregate answers its own declared type when it has one (STDDEV and the * percentiles are DOUBLE, the ranking windows BIGINT), else the aggregated chain's. + * - `ARRAY_AGG` collects the values it aggregates: `ARRAY` of the aggregated chain's + * REPORTED type (SCHEMA-6 D8, A-21); a chain whose type is unknown stays unknown (R-7). + * + * A metric (`MIN`, `MAX`, `SUM`, every statistic) is computed by Elasticsearch over each + * ELEMENT of a list field, so over an `ARRAY` column it reads X (SCHEMA-6 D7, + * `SQLTypeUtils.operandType`); `FIRST_VALUE`, `LAST_VALUE` and `ARRAY_AGG` return the top + * hits' `_source` values, the declared type. * * `None` when `aggregate` aggregates nothing. */ @@ -76,18 +83,27 @@ package object aggregate { aggregate .withFunctions(aggregate.functions.filterNot(_.isInstanceOf[AggregateFunction])) .reportedType + def aggregatedElement: SQLType = SQLTypeUtils.operandType(aggregated) function match { case COUNT | _: CountAgg => SQLTypes.BigInt case AVG | _: AvgAgg => SQLTypes.Double case SUM | _: SumAgg => - aggregated match { + aggregatedElement match { case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | SQLTypes.BigInt => SQLTypes.BigInt case _ => SQLTypes.Double } - case MIN | MAX | _: MinAgg | _: MaxAgg | _: FirstValue | _: LastValue => aggregated + case MIN | MAX | _: MinAgg | _: MaxAgg => aggregatedElement + case _: FirstValue | _: LastValue => aggregated + case _: ArrayAgg => + aggregated match { + case unknown if unknown.isUnknown => unknown + case element => SQLTypes.Array(element) + } case other => - Some(other.out).filterNot(t => t.isUnknown || t == SQLTypes.Null).getOrElse(aggregated) + Some(other.out) + .filterNot(t => t.isUnknown || t == SQLTypes.Null) + .getOrElse(aggregatedElement) } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index d54133df3..8ed3aab41 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -201,7 +201,8 @@ package object cond { else { val typed = arms .filterNot(a => ComparisonRule.stringLiteral(a).isDefined) - .map(_.reportedType) + // an `ARRAY` arm is one X element (SCHEMA-6 D7) + .map(a => SQLTypeUtils.operandType(a.reportedType)) .filterNot(t => t == SQLTypes.Null || t.isUnknown) .filter(family(_) >= 0) typed.headOption.flatMap { first => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 8043e1e09..dff7d79f8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -120,11 +120,12 @@ package object math { * argument whose type is not a known number keeps NUMERIC. * * The type REPORTED, not the one Painless is handed (`out`, still NUMERIC): what a consumer - * converts this value to is unchanged. + * converts this value to is unchanged. An `ARRAY` argument is one X element, the value + * Painless reads (SCHEMA-6 D7, `SQLTypeUtils.operandType`). */ override def reportedType: SQLType = mathOp match { case Abs => - arg.reportedType match { + SQLTypeUtils.operandType(arg.reportedType) match { case n: SQLNumeric => n case _ => super.reportedType } @@ -316,7 +317,8 @@ package object math { // narrower integer's rounded value always fits it, so it is not tested. In a query // only, the `domain` gate: an ingest processor and a transform keep their bytes. case _: SQLBigInt - if scale.exists(_ < 0) && arg.reportedType.isInstanceOf[SQLBigInt] && + if scale.exists(_ < 0) && + SQLTypeUtils.operandType(arg.reportedType).isInstanceOf[SQLBigInt] && SQLTypeUtils.convertsArms(context) => // the bounds of a `long` as a `double`: the lower one inclusive, the upper one // excluded (`2^63` is no `long`) @@ -348,8 +350,13 @@ package object math { * levels, aggregates, `CREATE TABLE … AS SELECT` and the overflow test of [[toPainlessCall]]; * `outputType` stays NUMERIC, so the Painless keeps its bytes (a whole scale answers a `Long`, * a DOUBLE column reads it as a `Double`). + * + * An `ARRAY` operand is one X element, the value Painless reads (SCHEMA-6 D7, + * `SQLTypeUtils.operandType`): `ROUND(arr, -1)` over an `ARRAY` reports the BIGINT its + * INT twin reports, in a query. Outside one (an ingest processor, a transform, the validation + * of a computed column) the operand is the whole list and keeps NUMERIC, as on 0.24.0. */ - override def reportedType: SQLType = arg.reportedType match { + override def reportedType: SQLType = SQLTypeUtils.operandType(arg.reportedType) match { case _: SQLTinyInt | _: SQLSmallInt | _: SQLInt if scale.exists(_ < 0) => SQLTypes.BigInt case whole @ (_: SQLTinyInt | _: SQLSmallInt | _: SQLInt | _: SQLBigInt) => whole case _: SQLReal => SQLTypes.Real diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 6e75d2836..dad6bdd7f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -308,8 +308,13 @@ package object function { /** [[argTypes]] 's counterpart for [[PainlessScript.reportedType]]: each argument's REPORTED * type, so a function that derives its own from its arguments can do so without the runtime * collapse leaking in from the leaves. + * + * An argument is read as Painless reads it (SCHEMA-6 D7, `SQLTypeUtils.operandType`): an + * `ARRAY` column passed to a scalar function is one X element (`doc['f'].value`), so + * `NULLIF(arr, 0)` over an `ARRAY` reports INT and its value is never wrapped in a list. */ - def reportedArgTypes: List[SQLType] = args.map(_.reportedType) + def reportedArgTypes: List[SQLType] = + args.map(a => app.softnetwork.elastic.sql.`type`.SQLTypeUtils.operandType(a.reportedType)) def argsSeparator: String = ", " diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 9c3562cc0..636ec5392 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -115,11 +115,14 @@ package object time { * DATE `COALESCE` it reported DATE: two answers for two DATE operands. Any other operand * reports as before, and so does every operand while an ingest processor or a transform * renders, whose bytes read this type (`GREATEST` / `LEAST` over it) and do not move - * (`SQLTypeUtils.rendersOutsideQuery`). + * (`SQLTypeUtils.rendersOutsideQuery`). An `ARRAY` operand is one DATE element, the + * value Painless reads (SCHEMA-6 D7, `SQLTypeUtils.operandType`), so it reports as a DATE + * does. */ def reportedOverDate: Option[SQLType] = this match { case f: FunctionWithIdentifier - if !SQLTypeUtils.rendersOutsideQuery && f.identifier.reportedType == SQLTypes.Date => + if !SQLTypeUtils.rendersOutsideQuery && + SQLTypeUtils.operandType(f.identifier.reportedType) == SQLTypes.Date => interval.checkType(SQLTypes.Date).toOption case _ => None } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 101221631..a58f32261 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -1123,8 +1123,11 @@ object ArithmeticExpression { * itself (`schema.IndexField`), so it keeps its time here as everywhere else, and every chain * over it follows: `MAX(ts)`, `COALESCE(ts, ts2)`, `CASE … THEN ts …`. */ + /** An operand's declared type as arithmetic reads it: an `ARRAY` column is one X element, the + * value Painless reads (SCHEMA-6 D7, `SQLTypeUtils.operandType`). + */ private def declaredType(operand: PainlessScript): SQLType = - ComparisonRule.declaredTypeOf(operand) + SQLTypeUtils.operandType(ComparisonRule.declaredTypeOf(operand)) /** `NULL`, bare or cast: its VALUE is NULL in every venue. */ private[sql] def nullLiteral(operand: PainlessScript): Boolean = operand match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index f9130460f..4fd868f57 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1912,7 +1912,9 @@ package object sql { */ def readingOf(literal: Token, against: => SQLType): Option[Reading] = stringLiteral(literal).flatMap { text => - against match { + // SCHEMA-6 (D7): beside an `ARRAY` column a literal is read as an X -- a term or range + // query matches any element of the field + SQLTypeUtils.operandType(against) match { case n: SQLNumeric => numberOf(text, n) case t: SQLTemporal if t != SQLTypes.Time && t != SQLTypes.Temporal => temporalOf(text, t) @@ -1936,12 +1938,18 @@ package object sql { def comparedType(reading: Option[Reading], operand: Token): SQLType = reading match { case Some(r) => r.sqlType - case None => declaredTypeOf(operand) + // SCHEMA-6 (D7): an `ARRAY` column is compared as its element X -- Elasticsearch matches + // a term or a range against each element, Painless reads one (`doc['f'].value`) + case None => SQLTypeUtils.operandType(declaredTypeOf(operand)) } - /** The two compared types ([[comparedType]]), when they cannot be compared. */ + /** The two compared types ([[comparedType]]), when they cannot be compared. Each side is judged + * as the element Elasticsearch compares (SCHEMA-6 D7, `SQLTypeUtils.operandType`): an + * `ARRAY` column against a number is an INT against it. + */ def mismatch(left: SQLType, right: SQLType): Option[(SQLType, SQLType)] = - if (comparable(left, right)) None else Some((left, right)) + if (comparable(SQLTypeUtils.operandType(left), SQLTypeUtils.operandType(right))) None + else Some((left, right)) /** The Painless a NUMBER or a BOOLEAN reading renders: the literal itself. */ def scalarPainless(reading: Reading): Option[String] = reading match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index 74208f478..e6aa89515 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -263,7 +263,18 @@ package object `type` { lazy val geo_point_type: PackratParser[SQLTypes.GeoPoint.type] = "(?i)(geo_point|geopoint)".r ^^ (_ => SQLTypes.GeoPoint) + /** A DDL list type: `ARRAY` of every type a column may declare, so `ARRAY` parses + * where a column is declared (SCHEMA-6 D3) -- the remedy a several-point geo field fails with + * names it. Tried FIRST by [[extension_type]]; a list of lists parses and is refused by the + * statement's validation, naming the column. `sql_type`'s `array_type` (a CAST target) is + * untouched. + */ + lazy val extension_array_type: PackratParser[SQLTypes.Array] = + "(?i)array<".r ~> extension_type <~ ">" ^^ { elementType => + SQLTypes.Array(elementType) + } + lazy val extension_type: PackratParser[SQLType] = - sql_type | geo_point_type + extension_array_type | sql_type | geo_point_type } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 9696431c3..c3eb00541 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -492,6 +492,129 @@ object DerivedTable { } } + /** SCHEMA-6 (D10) -- the table a derived table's body projects, each column typed as the + * projection item of the body that projects it ([[MultiSearch.fieldType]], the result schema's + * and the set operations' rule, #292), built by [[MultiSearch.columnOf]] and named by the body's + * output names: a `UNION ALL` body's columns typed over its branches + * ([[MultiSearch.columnTypes]], `Left` when the body is refused) under its first branch's names; + * a FROM-less body's items. + * + * `Right(None)` for an OPAQUE body, whose columns stay `ANY` (R-7): a bare `SELECT *` (no + * projection to type), a JOIN or several tables. The body is typed as it is: attach its own + * sources first ([[attach]]). + */ + private[elastic] def typedSchema(dt: DerivedTable): Either[String, Option[Schema]] = { + def single(s: SingleSearch): Option[Seq[(String, MultiSearch.SetOperationType)]] = + if ( + s.from.tables.size != 1 || + s.from.tables.exists(_.joins.exists(_.isInstanceOf[StandardJoin])) || + s.select.fields.exists(bareStar) + ) None + else + projected(s).map(names => names.zip(s.select.fields.map(f => MultiSearch.fieldType(f, s)))) + val columns: Either[String, Option[Seq[(String, MultiSearch.SetOperationType)]]] = + dt.query match { + case s: SingleSearch => Right(single(s)) + case m: MultiSearch => + m.requests.headOption + .filterNot(_.select.fields.exists(bareStar)) + .flatMap(projected) match { + case Some(names) => + MultiSearch + .columnTypes(m.requests, m.resolvedOperators) + .map(typed => + if (typed.size == names.size) Some(names.zip(typed.map(_.result))) else None + ) + case None => Right(None) + } + case f: FromlessSelect => + Right(Some(f.columnNames.zip(f.select.fields.map(f => MultiSearch.fieldType(f))))) + case _ => Right(None) + } + columns.map(_.map { typed => + app.softnetwork.elastic.sql.schema.Table( + dt.name, + typed.map { case (name, t) => MultiSearch.columnOf(name, t) }.toList + ) + }) + } + + /** SCHEMA-6 (D10) -- `s` with every FROM and JOIN source typed: each derived table's body -- a + * CTE reference is one -- attached first, innermost first, then its [[typedSchema]] registered + * under its alias (the key `From.tableAliases` gives it), and as `schema` when it is the main + * source; every index source with `lookup`'s schema. `Left` when a `UNION ALL` body is refused, + * with that refusal. Pure: nothing is executed, routed or refused beyond the typing, and plain + * `update` never types a derived table (R-6). WHERE-subquery bodies are not visited (A-24). + */ + private[elastic] def attach( + s: SingleSearch, + lookup: Seq[String] => Option[Schema] + ): Either[String, SingleSearch] = { + def attachBody(body: DqlStatement): Either[String, DqlStatement] = + body match { + case inner: SingleSearch => attach(inner, lookup) + case m: MultiSearch => + m.requests + .foldLeft[Either[String, Vector[SingleSearch]]](Right(Vector.empty)) { + case (Right(done), branch) => attach(branch, lookup).map(done :+ _) + case (refused, _) => refused + } + .map(branches => m.copy(requests = branches)) + case other => Right(other) + } + def attachDerived(dt: DerivedTable): Either[String, (DerivedTable, Option[Schema])] = + for { + body <- attachBody(dt.query) + typed <- typedSchema(dt.copy(query = body)) + } yield (dt.copy(query = body), typed) + val schemas = scala.collection.mutable.LinkedHashMap.empty[String, Schema] + def register(key: String, schema: Option[Schema]): Unit = + schema.foreach(t => schemas.put(key, t)) + def attachJoins(joins: Seq[Join]): Either[String, Vector[Join]] = + joins.foldLeft[Either[String, Vector[Join]]](Right(Vector.empty)) { + case (Right(done), sj: StandardJoin) => + sj.source match { + case dt: DerivedTable => + attachDerived(dt).map { case (attached, typed) => + register(attached.name, typed) + done :+ sj.copy(source = attached) + } + case other => + register(other.name, lookup(Seq(other.name))) + Right(done :+ sj) + } + case (Right(done), other) => Right(done :+ other) + case (refused, _) => refused + } + val attachedTables = + s.from.tables.foldLeft[Either[String, Vector[Table]]](Right(Vector.empty)) { + case (Right(done), table) => + val source: Either[String, Table] = table.derived match { + case Some(dt) => + attachDerived(dt).map { case (attached, typed) => + register(attached.name, typed) + table.copy(derived = Some(attached)) + } + case None => + register(table.name, lookup(Seq(table.name))) + Right(table) + } + for { + t <- source + joins <- attachJoins(t.joins) + } yield done :+ t.copy(joins = joins) + case (refused, _) => refused + } + attachedTables.map { tables => + val attached = s.copy(from = s.from.copy(tables = tables)) + val main = schemas.get(attached.from.mainTable.name).orElse(attached.schema) + attached.copy(schema = main, schemas = attached.schemas ++ schemas).update() + } + } + + private def bareStar(f: Field): Boolean = + f.identifier.name == "*" && f.identifier.functions.isEmpty + /** A bare `SELECT *` projects an un-enumerable list; anything else projects its items' output * names. The `*` test is `identifierName` with no functions — the same spelling * `SearchApi.extractOutputFieldNames` uses. diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 3316453ef..697e02210 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -27,6 +27,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTemporal, SQLTinyInt, SQLType, + SQLTypeUtils, SQLTypes, SQLVarchar } @@ -734,10 +735,13 @@ case class Bucket( // away from zero (`ComparisonRule.numberOf`, the one comparison rule): `HAVING k = '1.5'` keeps // the group 2, on a field level and on a scripted one alike, so the fold, `keyOf` and the // nameless-key gate, which read these lists, read that one value. A whole-number text keeps - // its path below. + // its path below. An `ARRAY` key is its element X, as the reading of the literal judges it + // (SCHEMA-6 D7, A-27: `SQLTypeUtils.operandType`, read by `ComparisonRule.readingOf`). case quoted: String if !ComparisonRule.spellsWholeNumber(quoted) && - ComparisonRule.wholeNumberType(ComparisonRule.declaredTypeOf(identifier)) => + ComparisonRule.wholeNumberType( + SQLTypeUtils.operandType(ComparisonRule.declaredTypeOf(identifier)) + ) => ComparisonRule.roundedWhole(quoted) match { case Some(rounded) => keyFilterTextsAsWritten(java.lang.Long.valueOf(rounded)) case None => keyFilterTextsAsWritten(quoted) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 1092a1f45..d37c36f0b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -1076,8 +1076,10 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { reading: Option[ComparisonRule.Reading] ): Option[String] = reading match { + // an `ARRAY` column is its element X, as the reading judges it (SCHEMA-6 D7: a term or a + // range query matches each element; `ComparisonRule.readingOf` reads `operandType` too) case Some(ComparisonRule.NumberReading(painless, false)) - if ComparisonRule.wholeNumberType(operandDeclaredType) => + if ComparisonRule.wholeNumberType(SQLTypeUtils.operandType(operandDeclaredType)) => ComparisonRule .stringLiteral(value) .filterNot(ComparisonRule.spellsWholeNumber) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index b1ad69bfd..e88d061f6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLReal, SQLType, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLArray, SQLReal, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.operator.{ AND, DIFF, @@ -34,6 +34,7 @@ import app.softnetwork.elastic.sql.operator.{ } import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.schema.{ + nestedArrayRefusal, sqlConfig, validateScriptReferences, Column, @@ -774,6 +775,12 @@ package object query { * Strings and `eq` only -- nothing here reads `Identifier.equals`. */ private[sql] def bucketFor(identifier: Identifier): Option[Bucket] = + keyedBucketFor(identifier).map(_.bucket) + + /** [[bucketFor]] 's decision with the bucket's kind: a GROUP BY key or a window level (SCHEMA-6 + * A-27 reads it to type a GROUP BY key item). + */ + private[query] def keyedBucketFor(identifier: Identifier): Option[KeyedBucket] = if (keyedBuckets.isEmpty) None else { val spelling = identifier.identifierName @@ -790,7 +797,7 @@ package object query { .orElse(identityRoute(leaf)) .orElse(today.filter(_.identity.forall(compatible(_, leaf)))) } - chosen.map(_.bucket) + chosen } /** Story IDENT-4 -- the identity of a column reference for the bucket lookup: its CORRELATION @@ -4163,47 +4170,130 @@ package object query { private def operandOf(branch: Int, field: Field): Operand = Operand(branch, field, declaredOperandType(field), identifierType(field.identifier)) + /** The operand of `field`, an item of the statement `s` (SCHEMA-6 A-27): a GROUP BY key of `s` + * is typed by its key's type ([[bucketType]]: X for an `ARRAY` column, Elasticsearch + * buckets the elements), declared and value type alike; every other item as [[operandOf]]. + */ + private def operandOf(branch: Int, field: Field, s: SingleSearch): Operand = + if (isGroupByKey(s, field)) { + val key = identifierType(field.identifier, element = true) + Operand(branch, field, SQLTypeUtils.operandType(declaredOperandType(field)), key) + } else operandOf(branch, field) + + /** Whether `field` is a GROUP BY key of `s` (SCHEMA-6 A-27): the bucket its identifier resolves + * to ([[SingleSearch.keyedBucketFor]], the lookup that attaches it) is a GROUP BY level -- + * never a window's PARTITION BY level, an aggregate or a window item. + */ + private def isGroupByKey(s: SingleSearch, field: Field): Boolean = + s.groupBy.isDefined && !field.isAggregation && !field.identifier.hasWindow && + s.keyedBucketFor(field.identifier).exists(_.groupByKey) + /** The type of ONE identifier as a set operation types a projection item of it: its declared * type ([[declaredOperandType]]) at every depth, a bare column's date format included * ([[positionType]]). */ - private def identifierType(id: Identifier): SetOperationType = { + private def identifierType(id: Identifier, element: Boolean = false): SetOperationType = { val column = id match { case g: GenericIdentifier if g.functions.isEmpty => g.col case _ => None } - positionType(declaredOperandType(id), column) + val declared = declaredOperandType(id) + positionType(if (element) SQLTypeUtils.operandType(declared) else declared, column) } /** SCHEMA-2b (B2) -- the type of the key of the aggregation level `b` (a GROUP BY level, a * window's PARTITION BY): the type a projection item of its identifier has ([[fieldType]]), so * the key of a selected level and its column are typed by ONE rule (#292). + * + * SCHEMA-6 (D7): an `ARRAY` column's key is an X -- a terms aggregation buckets each + * element -- and a GROUP BY key item is typed by this rule (A-27). A window level reads + * [[partitionType]]. + */ + private[elastic] def bucketType(b: Bucket): SetOperationType = + identifierType(b.identifier, element = true) + + /** SCHEMA-6 (A-28) -- the type of the key of a window's PARTITION BY level `b`, read by every + * window-level consumer: `ANY` (read as read, as 0.24.0 read every list) when `b`'s column is + * declared `ARRAY` -- Elasticsearch puts a several-valued document in several partitions and + * DuckDB in one, so core answers neither --, else [[bucketType]]. + */ + private[elastic] def partitionType(b: Bucket): SetOperationType = + if (arrayLevel(b)) SetOperationType(SQLTypes.Any) else bucketType(b) + + /** Whether the column of the aggregation level `b` is declared `ARRAY` (SCHEMA-6 A-28). */ + private[elastic] def arrayLevel(b: Bucket): Boolean = + declaredOperandType(b.identifier).isInstanceOf[SQLArray] + + /** The key type of the aggregation level `b` of `s`: a GROUP BY level's [[bucketType]], a + * window level's [[partitionType]] (SCHEMA-6 A-28). */ - private[elastic] def bucketType(b: Bucket): SetOperationType = identifierType(b.identifier) + private[elastic] def levelType(s: SingleSearch, b: Bucket): SetOperationType = + if (s.groupBy.exists(_.buckets.exists(_ eq b))) bucketType(b) else partitionType(b) /** SCHEMA-2 (D1) -- the type of ONE projection item, as a set operation types its position * ([[positionType]]): the result schema of a statement and the typing of its set-operation - * positions are this one rule (#292), never two. + * positions are this one rule (#292), never two. An item of no statement (a FROM-less one). */ private[elastic] def fieldType(field: Field): SetOperationType = operandOf(0, field).valueType + /** [[fieldType]] of an item of the statement `s`: a GROUP BY key of `s` has its key's type + * (SCHEMA-6 A-27). + */ + private[elastic] def fieldType(field: Field, s: SingleSearch): SetOperationType = + operandOf(0, field, s).valueType + /** SCHEMA-2 (D1) -- each item a statement DECLARES with its type ([[fieldType]]); `None` for a * statement that projects a bare `*`, whose width is its mapping's. */ private[elastic] def projectionTypes(s: SingleSearch): Option[Seq[(Field, SetOperationType)]] = - declared(s).map(_ => s.select.fieldsWithComputedAliases.map(f => f -> fieldType(f))) + declared(s).map(_ => s.select.fieldsWithComputedAliases.map(f => f -> fieldType(f, s))) /** SCHEMA-2 (D1) -- a mapped column's type, as [[positionType]] types it: `SELECT *`'s columns. */ private[elastic] def columnType(c: Column): SetOperationType = positionType(c.dataType, Some(c)) + /** SCHEMA-6 (D10) -- the column `name` of type `t`: the inverse of [[columnType]] + * (`columnType(columnOf(n, t)) == t`), so a derived table's column -- typed by the projection + * item of its body ([[fieldType]]) -- types an outer reference to it by the same rule. A + * `STRUCT`'s fields and a list of objects' element fields are its sub-fields, a temporal's + * date format its `format` option; a temporal of no format (an aggregate's) reads + * Elasticsearch's default one, as a mapped date does. + */ + private[elastic] def columnOf(name: String, t: SetOperationType): Column = { + def formatted(of: SetOperationType): ListMap[String, Value[_]] = + of.format.fold(ListMap.empty[String, Value[_]])(spec => + ListMap("format" -> StringValue(spec)) + ) + t.sqlType match { + case SQLTypes.GeoPoint => Column(name, SQLTypes.GeoPoint) + case SQLTypes.Struct => + Column( + name, + SQLTypes.Struct, + multiFields = t.fields.map(f => columnOf(f.name, f.valueType)).toList + ) + case array: SQLArray => + val element = t.element.getOrElse(SetOperationType(array.elementType)) + Column( + name, + array, + multiFields = element.fields + .filterNot(_ => element.sqlType == SQLTypes.GeoPoint) + .map(f => columnOf(f.name, f.valueType)) + .toList, + options = formatted(element) + ) + case scalar => Column(name, scalar, options = formatted(t)) + } + } + /** A declared type as a set operation types it, built once at every depth: a scalar folded * ([[unionOperandType]]); a list with its element's type; a `STRUCT` with its fields -- its - * column's, read from the schema, BY NAME: the schema read from a mapping holds an object's - * fields in no reliable order -- a list of objects' element too; a `GEO_POINT` as `{lat, - * lon}`. A `STRUCT` whose column is not known has no fields. + * column's, read from the schema, in the column's order ([[fieldsOf]]) -- a list of objects' + * element too; a `GEO_POINT` as `{lat, lon}`. A `STRUCT` whose column is not known has no + * fields. */ private def positionType(declared: SQLType, column: Option[Column]): SetOperationType = declared match { @@ -4232,8 +4322,11 @@ package object query { format = None ) + /** A `STRUCT` column's fields in its column's order (SCHEMA-6 D9): the order it DECLARED them + * in when it carries `_meta.field_order`, else its mapping's (`IndexField.apply`). + */ private def fieldsOf(column: Option[Column]): Seq[StructField] = - column.toSeq.flatMap(_.multiFields).sortBy(_.name).map { c => + column.toSeq.flatMap(_.multiFields).map { c => StructField(c.name, positionType(c.dataType, Some(c))) } @@ -4565,7 +4658,7 @@ package object query { ) { case (Right(columns), pos) => val leaves = requests.indices.map(branch => - byBranch.get(branch).map(f => operandOf(branch, f(pos))) + byBranch.get(branch).map(f => operandOf(branch, f(pos), requests(branch))) ) typeOf(tree, leaves) match { case Right(typed) => @@ -5458,6 +5551,9 @@ package object query { // the `schema` lazy val (`Table.update()`) during a parse that would not otherwise have // done so. Measured interleaved, 3 rounds: plain DDL 53.5-55.5 us control vs 55.4-56.4 us // branch -- overlapping, no signal; only `SCRIPT AS` DDL pays (204.6-208.4 -> 222.3-236.3). + // SCHEMA-6 (D2): a list of lists is refused before anything else, naming the column + case Right(cols) if cols.exists(nestedArrayRefusal(_).isDefined) => + Left(cols.flatMap(nestedArrayRefusal).head) case Right(cols) if cols.exists(hasScript) => validateScriptReferences(schema) case Right(_) => Right(()) } @@ -5616,6 +5712,28 @@ package object query { lazy val pipeline: IngestPipeline = IngestPipeline(s"alter-$table-${Instant.now}", IngestPipelineType.Custom, processors) + + /** SCHEMA-6 (D2): a column, a new type or a sub-field declared a list of lists is refused, + * naming the column -- `ADD COLUMN`, `SET DATA TYPE`, `SET FIELD` and `SET FIELDS` alike. + */ + override def validate(): Either[String, Unit] = + statements.iterator + .flatMap { + case AddColumn(column, _) => nestedArrayRefusal(column) + case AlterColumnType(name, newType, _) => nestedArrayRefusal(Column(name, newType)) + case AlterColumnField(name, field, _) => + nestedArrayRefusal(field).map(refusal => s"$name.$refusal") + case AlterColumnFields(name, fields, _) => + fields.iterator + .flatMap(nestedArrayRefusal(_)) + .map(refusal => s"$name.$refusal") + .toList + .headOption + case _ => None + } + .toList + .headOption + .toLeft(()) } sealed trait AlterTableStatement extends Token { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala index c6f00f333..fa7ed7abb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala @@ -42,10 +42,21 @@ object MappingsRules { "eager_global_ordinals" ) + /** A column's declared type and its sub-fields' declared order in the table's `_meta` + * (`_meta.columns.[.multi_fields.]*.data_type` / `.field_order`): written in place with + * the mapping, nothing indexed changes (SCHEMA-6 D4). A declared type whose Elasticsearch type + * changes is a column diff of its own (`ColumnTypeChanged`, `FieldAltered`), never this key + * alone. Every other `_meta` key keeps its safety. + */ + private val declarationKey = + """_meta\.columns\.[^.]+(\.multi_fields\.[^.]+)*\.(data_type|field_order)""".r + def isSafe(key: String): Boolean = { if (safeOptions.contains(key)) return true + if (declarationKey.pattern.matcher(key).matches()) return true + if (unsafeOptions.contains(key)) return false // Default: cautious → UNSAFE diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala index d7de20044..ed3e1f9b1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.sql.schema import app.softnetwork.elastic.sql.Value -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} +import app.softnetwork.elastic.sql.`type`.{SQLArray, SQLType, SQLTypeUtils} import app.softnetwork.elastic.sql.query._ sealed trait DiffSafety @@ -45,8 +45,14 @@ case class ColumnRemoved(name: String) extends ColumnDiff { case class ColumnTypeChanged(name: String, from: SQLType, to: SQLType) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnType(name, to) + /** SCHEMA-6 (D5): a change of an array column's element type -- or between a list and a scalar of + * another Elasticsearch type -- stays `Impossible`, as on 0.24.0, where every array column read + * `ANY` and `canConvert(ANY, _)` was false. Declaring an existing field a list of its own type + * is no `ColumnTypeChanged` at all (`Column.diff` compares Elasticsearch types). + */ override def safety: DiffSafety = - if (SQLTypeUtils.canConvert(from, to)) + if (from.isInstanceOf[SQLArray] || to.isInstanceOf[SQLArray]) Impossible + else if (SQLTypeUtils.canConvert(from, to)) UnsafeReindex else Impossible diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index c004d05b6..336ce2360 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLLiteral, SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLArray, SQLLiteral, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.config.ElasticSqlConfig import app.softnetwork.elastic.sql.function.{ Function => SQLFunction, @@ -59,6 +59,42 @@ package object schema { lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + /** `fields` with `field` in place of the sub-field of its name, which keeps its POSITION + * (SCHEMA-6 D9: the declared order is persisted, so `SET FIELD` -- the remedy a several-valued + * sub-field fails with -- must not move the field it rewrites); a new name is appended. + */ + private[schema] def replacedSubField(fields: List[Column], field: Column): List[Column] = + if (fields.exists(_.name == field.name)) + fields.map(f => if (f.name == field.name) field else f) + else fields :+ field + + /** The refusal of a column -- or of one of its sub-fields, at any depth -- declared a list of + * lists (SCHEMA-6 D2): Elasticsearch flattens a nested array into one list of values, so an + * `ARRAY>` cannot hold what it declares. ONE check, read by every DDL statement that + * declares a type and by `Table.mergeWithSearch` (CTAS). `None` when the column is accepted. + */ + private[elastic] def nestedArrayRefusal(column: Column): Option[String] = { + def nestedArray(t: SQLType): Boolean = t match { + case SQLTypes.Array(inner) => + inner match { + case _: SQLArray => true + case other => nestedArray(other) + } + case _ => false + } + def walk(c: Column, path: String): Option[String] = + if (nestedArray(c.dataType)) + Some( + s"$path: ${c.dataType.typeId} is not supported: Elasticsearch flattens nested arrays; " + + "declare ARRAY" + ) + else + c.multiFields.iterator.map(f => walk(f, s"$path.${f.name}")).collectFirst { + case Some(refusal) => refusal + } + walk(column, column.name) + } + // ======================================================================== // PIPELINE COMPONENTS // ======================================================================== @@ -955,8 +991,18 @@ package object schema { * collapses every temporal declaration to `Timestamp`, and the declaration is exactly what the * ingest emission needs -- a DATE column parses as a `LocalDate`, a TIMESTAMP as a * `ZonedDateTime`, and getting that wrong is the defect this rejects. + * + * 🔴 Runs in the scope of the script it judges (`SQLTypeUtils.validatingComputedColumns`): the + * processor reads `ctx.`, the whole value, so an `ARRAY` operand is its declared type here + * as in the processor's render, never D7's element (SCHEMA-6, lead R-66). Judged as the element, + * `CAST(arr AS BIGINT)` was refused ("declared ARRAY and not INT") on CREATE and on every + * later ALTER of a table holding it, while 0.24.0 accepted it. */ - private[elastic] def validateScriptReferences(schema: Schema): Either[String, Unit] = { + private[elastic] def validateScriptReferences(schema: Schema): Either[String, Unit] = + SQLTypeUtils.validatingComputedColumns(scriptReferenceErrors(schema)) + + /** [[validateScriptReferences]]'s rules, in the scope it sets. */ + private def scriptReferenceErrors(schema: Schema): Either[String, Unit] = { // 🔴 A REGULAR table only. Every other table type projects columns from somewhere else, so its // own column list is not the population a reference is checked against: a materialized view's // computed column legitimately reads a column of its SOURCE (`effective_date` from @@ -2058,6 +2104,12 @@ package object schema { field.name -> ObjectValue(field._meta) ): _* ) + ), + // SCHEMA-6 (D9): the sub-fields in their DECLARED order -- + // Elasticsearch returns `properties` sorted and a `_meta` + // object in no order, so only a list keeps it + "field_order" -> StringValues( + multiFields.map(field => StringValue(field.name)) ) ) else ListMap.empty[String, Value[_]]) ++ (if (lineage.nonEmpty) { @@ -2092,7 +2144,7 @@ package object schema { def updateStruct(): Column = { struct .map { st => - val updated = st.copy(multiFields = st.multiFields.filterNot(_.name == this.name) :+ this) + val updated = st.copy(multiFields = replacedSubField(st.multiFields, this)) updated.updateStruct() } .getOrElse(this) @@ -2998,8 +3050,7 @@ package object schema { // returning `table` made SET|ADD FIELD a no-op that still reported success. val updated = c .copy( - multiFields = - c.multiFields.filterNot(_.name == field.name) :+ field.update(Some(c)), + multiFields = replacedSubField(c.multiFields, field.update(Some(c))), script = None ) .updateStruct() diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 5db6faa6c..b2b74e305 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -77,7 +77,7 @@ object SQLTypeUtils { case BigInt | Array(BigInt) => "long" case Double | Array(Double) => "double" case Real | Array(Real) => "float" - case Numeric => "scaled_float" + case Numeric | Array(Numeric) => "scaled_float" // `Char` joins `Varchar`, not `Keyword`: `text` is what the DDL documentation already promises // for a character column. Without this arm `Char` fell to `case _ => "object"`, so // `CREATE TABLE t (c CHAR)` silently created an OBJECT-typed Elasticsearch field. That was @@ -98,8 +98,36 @@ object SQLTypeUtils { case GeoPoint | Array(GeoPoint) => "geo_point" case Array(Struct) => "nested" case Struct => "object" - case VarBinary => "binary" - case _ => "object" + // Elasticsearch indexes a list of base64 values into a `binary` field as it indexes one + // (SCHEMA-6 D2): an `ARRAY` is a `binary` field, not the `object` it fell to + case VarBinary | Array(VarBinary) => "binary" + case _ => "object" + } + + /** The type Elasticsearch works with PER ELEMENT for a column declared `t` (SCHEMA-6 D7): X for + * an `ARRAY` of a scalar X -- a term or range query matches any element, a metric aggregates + * the elements, a terms aggregation buckets them, Painless reads one (`doc['f'].value`) -- else + * `t` itself. A list of objects or of lists is no scalar, and keeps its type. + * + * Read where a value is used ONE element at a time (comparisons, function arguments, Painless, + * metric aggregations, GROUP BY keys); never where a column is DECLARED (the schema, DESCRIBE, + * CTAS, a set-operation position, the top hits FIRST / LAST / ARRAY_AGG return). + * + * 🔴 Never while a script that reads the WHOLE value is rendered or validated + * ([[readsWholeValues]]): an ingest processor or a transform reads `ctx.`, the list itself, + * not one element, so there an `ARRAY` column is its declared type and the script keeps + * 0.24.0's bytes and verdict. Read as the element there, `CAST(arrk AS INT)` parsed the list as + * one number and a CASE arm cast it to `long`: the processor failed on `["7"]` and `[5]`, and + * `ignore_failure` stored NULL. + */ + def operandType(t: SQLType): SQLType = t match { + case Array(element) if isScalarElement(element) && !readsWholeValues => element + case other => other + } + + private def isScalarElement(t: SQLType): Boolean = t match { + case _: SQLArray | Struct => false + case _ => true } /** The Elasticsearch types a field sort may name as its `unmapped_type`: each one sorts without @@ -142,8 +170,11 @@ object SQLTypeUtils { * Tableau emit constantly). * * Everything else is returned unchanged: a KEYWORD is a `String` at runtime too, an INT is an - * int. ARRAY types are deliberately NOT collapsed — `elasticType(Array(Date))` is also `"date"`, - * but the runtime value is a list, so the temporal arms must not fire on it. + * int. An `ARRAY` of a scalar X is ONE element at runtime (SCHEMA-6 D7): `doc['f'].value` + * reads one value of the field, an X -- a `ZonedDateTime` for an `ARRAY` -- so it is X's + * runtime type ([[operandType]]); a list of objects keeps its type. An ingest processor or a + * transform reads `ctx.`, the whole list, and keeps the declared `ARRAY` there + * ([[readsWholeValues]]). * * `date_nanos` needs no arm: `SQLTypes.apply` maps it to `Any` (BIDC-4's deliberate exclusion), * and `Any` is not temporal, so it falls to the identity case here. @@ -159,7 +190,9 @@ object SQLTypeUtils { case SQLTypes.Date | SQLTypes.Time | SQLTypes.DateTime | SQLTypes.Timestamp | SQLTypes.Temporal => SQLTypes.Timestamp - case other => other + // SCHEMA-6 (D7): `doc['f'].value` over an `ARRAY` field is ONE element, an X + case array @ Array(_) if operandType(array) ne array => runtimeType(operandType(array)) + case other => other } def matches(out: SQLType, in: SQLType): Boolean = @@ -304,8 +337,12 @@ object SQLTypeUtils { * * The reported type only: a node with an unknown arm renders as it did, and the other readers of * [[leastCommonSuperType]] (a comparison's operands, BETWEEN's bounds, arithmetic) keep theirs. + * + * Each arm is read as Painless reads it ([[operandType]], SCHEMA-6 D7): an `ARRAY` column is + * one X element, so `COALESCE(arr, 0)` over an `ARRAY` is an INT, never a list. */ - def armsReportedType(types: List[SQLType]): SQLType = { + def armsReportedType(arms: List[SQLType]): SQLType = { + val types = arms.map(operandType) val distinct = types.distinct val unknownBesideDate = distinct.contains(SQLTypes.Date) && @@ -463,6 +500,31 @@ object SQLTypeUtils { } } + /** Is the validation of a computed column (`SCRIPT AS`) running on this thread? Set by + * [[validatingComputedColumns]]. + */ + private[this] val computedColumnValidation = new scala.util.DynamicVariable[Boolean](false) + + /** Does the script being rendered or validated on this thread read a document's WHOLE value + * (`ctx.`), and not one element of it (`doc['f'].value`)? An ingest processor or a transform + * while it renders ([[rendersOutsideQuery]]), and a computed column while it is validated + * ([[validatingComputedColumns]]): its verdict is about the processor it installs. + * + * 🔴 SCHEMA-6 (lead R-66): ONE scope for D7's element view ([[operandType]]), which is off in + * it. Every reader of the element view goes through `operandType`, so the processor's bytes and + * the validation's verdict are 0.24.0's for an `ARRAY` column, and A-30's refusal of a + * NUMERIC or TEMPORAL input over it (`acceptableOperand`, the declared type) stays. + */ + def readsWholeValues: Boolean = outsideQuery.value || computedColumnValidation.value + + /** Runs `body`, the validation of computed columns, in the scope of [[readsWholeValues]]. Only + * D7's element view reads that scope: the rest of the validation reads the types it read before + * (a conditional answers its query type, [[rendersOutsideQuery]] stays false). + */ + def validatingComputedColumns[T](body: => T): T = + if (computedColumnValidation.value) body + else computedColumnValidation.withValue(true)(body) + /** The type an INGEST PROCESSOR (or a transform) writes for `expr`: its `out`, read in the scope * that script renders in ([[renderingIn]]), the type it rendered before a conditional answered * its reported type in a query. @@ -490,7 +552,8 @@ object SQLTypeUtils { * NULL arm has no type and is left out. */ def armsType(arms: List[PainlessScript], reported: SQLType): Option[SQLType] = { - val known = arms.map(_.reportedType).filterNot(_ == SQLTypes.Null) + // an `ARRAY` arm renders one X element (SCHEMA-6 D7, [[operandType]]) + val known = arms.map(a => operandType(a.reportedType)).filterNot(_ == SQLTypes.Null) def all(family: SQLType => Boolean): Boolean = known.forall(family) def dateCarrying(t: SQLType): Boolean = t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp || diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index 1f173ca4c..9e95a15c6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -58,6 +58,14 @@ object SQLTypes { case object VarBinary extends SQLVarBinary { val typeId = "VARBINARY" } + /** The type a TYPE NAME reads as: a declared `_meta` `data_type`, an Elasticsearch mapping type, + * a DDL type. THE read direction (SCHEMA-6 D13): `SQLTypeUtils.elasticType` is the write + * direction, and `IndexField.undeclaredType` the only seam before it. + * + * `ARRAY` reads `ARRAY` of X's reading (D1), so a column declared `ARRAY` reads back + * as one; an element read as `ANY`, or itself a list (a 0.24.0 `ARRAY>` declaration -- + * Elasticsearch has no list of lists), reads `ANY` (R-7), as every unknown name does. + */ def apply(typeName: String): SQLType = typeName.toLowerCase match { case "null" => Null case "boolean" => Boolean @@ -78,7 +86,13 @@ object SQLTypes { case "nested" | "array" => Array(Struct) case "geo_point" => GeoPoint case "binary" | "varbinary" => VarBinary - case _ => Any + case "char" => Char + case array if array.startsWith("array<") && array.endsWith(">") => + apply(array.substring(6, array.length - 1)) match { + case Any | _: SQLArray => Any + case element => Array(element) + } + case _ => Any } /** The Elasticsearch-mapping direction. A field's DECLARED type is preserved verbatim: when we diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 62b65dc65..7af253087 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1534,7 +1534,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" val indexMappings = schema.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"id":{"data_type":"INT","not_null":"true","comment":"user identifier"},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"birthdate":{"data_type":"DATE","not_null":"false"},"age":{"data_type":"INT","not_null":"false","script":{"sql":"TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5"}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4"}}}}},"type":"regular"}}""".stripMargin + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"id":{"data_type":"INT","not_null":"true","comment":"user identifier"},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}},"field_order":["raw"]},"birthdate":{"data_type":"DATE","not_null":"false"},"age":{"data_type":"INT","not_null":"false","script":{"sql":"TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5"}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4"}}},"field_order":["bio","followers","join_date","seniority"]}},"type":"regular"}}""".stripMargin val indexSettings = schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala index e7a4ec610..44f6b6fce 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.{ComparisonRule, Identifier} -import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction +import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, ArrayAgg} import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} @@ -221,7 +221,12 @@ class AggregateReportedTypeSpec extends AnyFlatSpec with Matchers { val (produced, column) = assertOneType(sql) withClue(s"[$sql] ") { produced shouldBe expected - SQLTypeUtils.elasticType(column.dataType) shouldBe kindOf(expected).es + // a list maps to its element's Elasticsearch type (SCHEMA-6 D2) + val es = expected match { + case SQLTypes.Array(element) => kindOf(element).es + case scalar => kindOf(scalar).es + } + SQLTypeUtils.elasticType(column.dataType) shouldBe es } case Agreed(_, holds) => holds(assertOneType(sql)._1) case Refused => () @@ -295,10 +300,10 @@ class AggregateReportedTypeSpec extends AnyFlatSpec with Matchers { "SELECT LAST_VALUE(k) OVER (PARTITION BY g ORDER BY n) AS a FROM t" -> Pinned( SQLTypes.Keyword ), - // DuckDB answers VARCHAR[], a list; core reports the element's type (which is also how - // Elasticsearch maps an array). Pre-existing. - "SELECT ARRAY_AGG(k) OVER (PARTITION BY g) AS a FROM t" -> Agreed( - "ARRAY_AGG: DuckDB VARCHAR[], core the element type" + // DuckDB answers VARCHAR[], a list, and so does core (SCHEMA-6 D8, A-21): ARRAY, + // mapped `keyword` as Elasticsearch maps a list + "SELECT ARRAY_AGG(k) OVER (PARTITION BY g) AS a FROM t" -> Pinned( + SQLTypes.Array(SQLTypes.Keyword) ), "SELECT ROW_NUMBER() OVER (PARTITION BY g ORDER BY n) AS a FROM t" -> Pinned(SQLTypes.BigInt), "SELECT RANK() OVER (PARTITION BY g ORDER BY n) AS a FROM t" -> Pinned(SQLTypes.BigInt) @@ -353,6 +358,36 @@ class AggregateReportedTypeSpec extends AnyFlatSpec with Matchers { } } + // SCHEMA-6 D12, the seam with SCHEMA-5: ARRAY_AGG collects what its chain REPORTS (D8), so over + // ROUND, DATEDIFF or a date part it is a list of SCHEMA-5's width -- `ARRAY_AGG(ROUND(n, -1))` + // over an INT a list of BIGINT, never of NUMERIC. ARRAY_AGG's grammar takes a column + // (`array_agg`), so the chain is composed here as the parser composes a window function over an + // identifier (`identifierWithWindowFunction`): the seam is `outputTypeOf`, which both read. + it should "declare ARRAY_AGG over ROUND, DATEDIFF or a date part a list of its type (D12)" in { + for ( + (chain, element) <- List( + "ROUND(n, -1)" -> SQLTypes.BigInt, + "ROUND(i8, -2)" -> SQLTypes.BigInt, + "ROUND(l, -1)" -> SQLTypes.BigInt, + "ROUND(n)" -> SQLTypes.Int, + "ROUND(x, 2)" -> SQLTypes.Double, + "ROUND(r, 1)" -> SQLTypes.Real, + "DATEDIFF(d, de)" -> SQLTypes.BigInt, + "YEAR(d)" -> SQLTypes.BigInt + ) + ) { + val (_, chainId) = resolved(s"SELECT $chain AS a FROM t") + val collected = chainId.withFunctions( + ArrayAgg(chainId, Seq.empty, None) +: chainId.functions + ) + withClue(s"[ARRAY_AGG($chain)] ") { + chainId.reportedType shouldBe element + AggregateFunction.outputTypeOf(collected) shouldBe Some(SQLTypes.Array(element)) + collected.reportedType shouldBe SQLTypes.Array(element) + } + } + } + // SCHEMA-2b (A-20): a whole SUM is a BIGINT whatever its column's width -- three INTs of // 2,000,000,000 sum beyond an INT -- and a fractional SUM a DOUBLE, a REAL one included // (Elasticsearch's `sum` is a double); a CTAS maps them `long` and `double` diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DerivedColumnTypeMatrixSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DerivedColumnTypeMatrixSpec.scala new file mode 100644 index 000000000..1b27579ba --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DerivedColumnTypeMatrixSpec.scala @@ -0,0 +1,730 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.MultiSearch.{SetOperationType, StructField} +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** SCHEMA-6 (D10, AC1-AC3, #419) -- a derived-table or CTE column is typed from its body, by the + * rule a projection item is typed by (`MultiSearch.fieldType`, #292), once the derived tables are + * attached (`DerivedTable.attach`, the seam `SearchApi.attachDerived` serves the relational + * engine). Plain `update` never types one (R-6): arrow's own call stays unattached until + * SCHEMA-10. + * + * 🔴 The ORACLE is DuckDB 1.5.5, MEASURED (its Python client, Python 3.12, one non-null row in a + * twin table: INT -> INTEGER, KEYWORD -> VARCHAR, VARBINARY -> BLOB, GEO_POINT -> `STRUCT(lat + * DOUBLE, lon DOUBLE)`, `obj` -> `STRUCT(y INTEGER, b VARCHAR)` in declared order, `ARRAY` -> + * `INTEGER[]`, nested -> `STRUCT(q INTEGER, p VARCHAR)[]`), embedded below and NOT derived from + * core: + * - P3: `DESCRIBE` of every item (a bare column of each kind, an alias, literals, casts, + * arithmetic, functions, aggregates under `GROUP BY g`, a list column grouped and under MIN) + * in every form -- F1 `q.c`, F2 unqualified `c`, F2b `q.*`, F3 WITH, F4 two levels, F5 a + * `UNION ALL` body; + * - F7: whether DuckDB runs a derived column of T1 beside a plain column of T2 (`UNION ALL`, + * `UNION`), and the list-of-objects merge of #419 item 2. + * + * Every cell equals DuckDB's answer, except the LISTED divergences, each with its rationale, and + * the NUMERIC / TEMPORAL cells SCHEMA-5 owns (D12), excluded and counted. + */ +class DerivedColumnTypeMatrixSpec extends AnyFlatSpec with Matchers { + + /** `zs_t` as core reads it; `l BIGINT` added for F7's BIGINT plain column. */ + private val zsT: SchemaTable = SchemaTable( + "zs_t", + columns = List( + Column("n", SQLTypes.Int), + Column("x", SQLTypes.Double), + Column("b", SQLTypes.Boolean), + Column("k", SQLTypes.Keyword), + Column("s", SQLTypes.Keyword), + Column("d", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("bin", SQLTypes.VarBinary), + Column("geo", SQLTypes.GeoPoint), + Column( + "obj", + SQLTypes.Struct, + multiFields = List(Column("y", SQLTypes.Int), Column("b", SQLTypes.Keyword)) + ), + Column( + "items", + SQLTypes.Array(SQLTypes.Struct), + multiFields = List(Column("q", SQLTypes.Int), Column("p", SQLTypes.Keyword)) + ), + Column("arr", SQLTypes.Array(SQLTypes.Int)), + Column("arrk", SQLTypes.Array(SQLTypes.Keyword)), + Column("ad", SQLTypes.Array(SQLTypes.Date)), + Column("g", SQLTypes.Keyword), + Column("l", SQLTypes.BigInt) + ) + ).update() + + private val zsU: SchemaTable = SchemaTable( + "zs_u", + columns = List( + Column( + "items", + SQLTypes.Array(SQLTypes.Struct), + multiFields = List(Column("r", SQLTypes.Int)) + ) + ) + ).update() + + private val lookup: Seq[String] => Option[SchemaTable] = + sources => Map("zs_t" -> zsT, "zs_u" -> zsU).get(sources.mkString(",")) + + /** MEASURED DuckDB 1.5.5: kind, form or operator, statement, `DESCRIBE` type or FAILS. + */ + private val measured: Seq[(String, String, String, String)] = + """ + |P3 F1 SELECT q.c FROM (SELECT n AS c FROM zs_t) q INTEGER + |P3 F2 SELECT c FROM (SELECT n AS c FROM zs_t) q INTEGER + |P3 F2b SELECT q.* FROM (SELECT n AS c FROM zs_t) q INTEGER + |P3 F3 WITH q AS (SELECT n AS c FROM zs_t) SELECT q.c FROM q INTEGER + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT n AS c FROM zs_t) q) q2 INTEGER + |P3 F5 SELECT q.c FROM (SELECT n AS c FROM zs_t UNION ALL SELECT n AS c FROM zs_t) q INTEGER + |P3 F1 SELECT q.c FROM (SELECT x AS c FROM zs_t) q DOUBLE + |P3 F2 SELECT c FROM (SELECT x AS c FROM zs_t) q DOUBLE + |P3 F2b SELECT q.* FROM (SELECT x AS c FROM zs_t) q DOUBLE + |P3 F3 WITH q AS (SELECT x AS c FROM zs_t) SELECT q.c FROM q DOUBLE + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT x AS c FROM zs_t) q) q2 DOUBLE + |P3 F5 SELECT q.c FROM (SELECT x AS c FROM zs_t UNION ALL SELECT x AS c FROM zs_t) q DOUBLE + |P3 F1 SELECT q.c FROM (SELECT b AS c FROM zs_t) q BOOLEAN + |P3 F2 SELECT c FROM (SELECT b AS c FROM zs_t) q BOOLEAN + |P3 F2b SELECT q.* FROM (SELECT b AS c FROM zs_t) q BOOLEAN + |P3 F3 WITH q AS (SELECT b AS c FROM zs_t) SELECT q.c FROM q BOOLEAN + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT b AS c FROM zs_t) q) q2 BOOLEAN + |P3 F5 SELECT q.c FROM (SELECT b AS c FROM zs_t UNION ALL SELECT b AS c FROM zs_t) q BOOLEAN + |P3 F1 SELECT q.c FROM (SELECT k AS c FROM zs_t) q VARCHAR + |P3 F2 SELECT c FROM (SELECT k AS c FROM zs_t) q VARCHAR + |P3 F2b SELECT q.* FROM (SELECT k AS c FROM zs_t) q VARCHAR + |P3 F3 WITH q AS (SELECT k AS c FROM zs_t) SELECT q.c FROM q VARCHAR + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT k AS c FROM zs_t) q) q2 VARCHAR + |P3 F5 SELECT q.c FROM (SELECT k AS c FROM zs_t UNION ALL SELECT k AS c FROM zs_t) q VARCHAR + |P3 F1 SELECT q.c FROM (SELECT s AS c FROM zs_t) q VARCHAR + |P3 F2 SELECT c FROM (SELECT s AS c FROM zs_t) q VARCHAR + |P3 F2b SELECT q.* FROM (SELECT s AS c FROM zs_t) q VARCHAR + |P3 F3 WITH q AS (SELECT s AS c FROM zs_t) SELECT q.c FROM q VARCHAR + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT s AS c FROM zs_t) q) q2 VARCHAR + |P3 F5 SELECT q.c FROM (SELECT s AS c FROM zs_t UNION ALL SELECT s AS c FROM zs_t) q VARCHAR + |P3 F1 SELECT q.c FROM (SELECT d AS c FROM zs_t) q DATE + |P3 F2 SELECT c FROM (SELECT d AS c FROM zs_t) q DATE + |P3 F2b SELECT q.* FROM (SELECT d AS c FROM zs_t) q DATE + |P3 F3 WITH q AS (SELECT d AS c FROM zs_t) SELECT q.c FROM q DATE + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT d AS c FROM zs_t) q) q2 DATE + |P3 F5 SELECT q.c FROM (SELECT d AS c FROM zs_t UNION ALL SELECT d AS c FROM zs_t) q DATE + |P3 F1 SELECT q.c FROM (SELECT ts AS c FROM zs_t) q TIMESTAMP + |P3 F2 SELECT c FROM (SELECT ts AS c FROM zs_t) q TIMESTAMP + |P3 F2b SELECT q.* FROM (SELECT ts AS c FROM zs_t) q TIMESTAMP + |P3 F3 WITH q AS (SELECT ts AS c FROM zs_t) SELECT q.c FROM q TIMESTAMP + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT ts AS c FROM zs_t) q) q2 TIMESTAMP + |P3 F5 SELECT q.c FROM (SELECT ts AS c FROM zs_t UNION ALL SELECT ts AS c FROM zs_t) q TIMESTAMP + |P3 F1 SELECT q.c FROM (SELECT bin AS c FROM zs_t) q BLOB + |P3 F2 SELECT c FROM (SELECT bin AS c FROM zs_t) q BLOB + |P3 F2b SELECT q.* FROM (SELECT bin AS c FROM zs_t) q BLOB + |P3 F3 WITH q AS (SELECT bin AS c FROM zs_t) SELECT q.c FROM q BLOB + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT bin AS c FROM zs_t) q) q2 BLOB + |P3 F5 SELECT q.c FROM (SELECT bin AS c FROM zs_t UNION ALL SELECT bin AS c FROM zs_t) q BLOB + |P3 F1 SELECT q.c FROM (SELECT geo AS c FROM zs_t) q STRUCT(lat DOUBLE, lon DOUBLE) + |P3 F2 SELECT c FROM (SELECT geo AS c FROM zs_t) q STRUCT(lat DOUBLE, lon DOUBLE) + |P3 F2b SELECT q.* FROM (SELECT geo AS c FROM zs_t) q STRUCT(lat DOUBLE, lon DOUBLE) + |P3 F3 WITH q AS (SELECT geo AS c FROM zs_t) SELECT q.c FROM q STRUCT(lat DOUBLE, lon DOUBLE) + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT geo AS c FROM zs_t) q) q2 STRUCT(lat DOUBLE, lon DOUBLE) + |P3 F5 SELECT q.c FROM (SELECT geo AS c FROM zs_t UNION ALL SELECT geo AS c FROM zs_t) q STRUCT(lat DOUBLE, lon DOUBLE) + |P3 F1 SELECT q.c FROM (SELECT obj AS c FROM zs_t) q STRUCT(y INTEGER, b VARCHAR) + |P3 F2 SELECT c FROM (SELECT obj AS c FROM zs_t) q STRUCT(y INTEGER, b VARCHAR) + |P3 F2b SELECT q.* FROM (SELECT obj AS c FROM zs_t) q STRUCT(y INTEGER, b VARCHAR) + |P3 F3 WITH q AS (SELECT obj AS c FROM zs_t) SELECT q.c FROM q STRUCT(y INTEGER, b VARCHAR) + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT obj AS c FROM zs_t) q) q2 STRUCT(y INTEGER, b VARCHAR) + |P3 F5 SELECT q.c FROM (SELECT obj AS c FROM zs_t UNION ALL SELECT obj AS c FROM zs_t) q STRUCT(y INTEGER, b VARCHAR) + |P3 F1 SELECT q.c FROM (SELECT items AS c FROM zs_t) q STRUCT(q INTEGER, p VARCHAR)[] + |P3 F2 SELECT c FROM (SELECT items AS c FROM zs_t) q STRUCT(q INTEGER, p VARCHAR)[] + |P3 F2b SELECT q.* FROM (SELECT items AS c FROM zs_t) q STRUCT(q INTEGER, p VARCHAR)[] + |P3 F3 WITH q AS (SELECT items AS c FROM zs_t) SELECT q.c FROM q STRUCT(q INTEGER, p VARCHAR)[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT items AS c FROM zs_t) q) q2 STRUCT(q INTEGER, p VARCHAR)[] + |P3 F5 SELECT q.c FROM (SELECT items AS c FROM zs_t UNION ALL SELECT items AS c FROM zs_t) q STRUCT(q INTEGER, p VARCHAR)[] + |P3 F1 SELECT q.c FROM (SELECT arr AS c FROM zs_t) q INTEGER[] + |P3 F2 SELECT c FROM (SELECT arr AS c FROM zs_t) q INTEGER[] + |P3 F2b SELECT q.* FROM (SELECT arr AS c FROM zs_t) q INTEGER[] + |P3 F3 WITH q AS (SELECT arr AS c FROM zs_t) SELECT q.c FROM q INTEGER[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT arr AS c FROM zs_t) q) q2 INTEGER[] + |P3 F5 SELECT q.c FROM (SELECT arr AS c FROM zs_t UNION ALL SELECT arr AS c FROM zs_t) q INTEGER[] + |P3 F1 SELECT q.c FROM (SELECT arrk AS c FROM zs_t) q VARCHAR[] + |P3 F2 SELECT c FROM (SELECT arrk AS c FROM zs_t) q VARCHAR[] + |P3 F2b SELECT q.* FROM (SELECT arrk AS c FROM zs_t) q VARCHAR[] + |P3 F3 WITH q AS (SELECT arrk AS c FROM zs_t) SELECT q.c FROM q VARCHAR[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT arrk AS c FROM zs_t) q) q2 VARCHAR[] + |P3 F5 SELECT q.c FROM (SELECT arrk AS c FROM zs_t UNION ALL SELECT arrk AS c FROM zs_t) q VARCHAR[] + |P3 F1 SELECT q.c FROM (SELECT ad AS c FROM zs_t) q DATE[] + |P3 F2 SELECT c FROM (SELECT ad AS c FROM zs_t) q DATE[] + |P3 F2b SELECT q.* FROM (SELECT ad AS c FROM zs_t) q DATE[] + |P3 F3 WITH q AS (SELECT ad AS c FROM zs_t) SELECT q.c FROM q DATE[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT ad AS c FROM zs_t) q) q2 DATE[] + |P3 F5 SELECT q.c FROM (SELECT ad AS c FROM zs_t UNION ALL SELECT ad AS c FROM zs_t) q DATE[] + |P3 F1 SELECT q.c FROM (SELECT g AS c FROM zs_t) q VARCHAR + |P3 F2 SELECT c FROM (SELECT g AS c FROM zs_t) q VARCHAR + |P3 F2b SELECT q.* FROM (SELECT g AS c FROM zs_t) q VARCHAR + |P3 F3 WITH q AS (SELECT g AS c FROM zs_t) SELECT q.c FROM q VARCHAR + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT g AS c FROM zs_t) q) q2 VARCHAR + |P3 F5 SELECT q.c FROM (SELECT g AS c FROM zs_t UNION ALL SELECT g AS c FROM zs_t) q VARCHAR + |P3 F1 SELECT q.c FROM (SELECT l AS c FROM zs_t) q BIGINT + |P3 F2 SELECT c FROM (SELECT l AS c FROM zs_t) q BIGINT + |P3 F2b SELECT q.* FROM (SELECT l AS c FROM zs_t) q BIGINT + |P3 F3 WITH q AS (SELECT l AS c FROM zs_t) SELECT q.c FROM q BIGINT + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT l AS c FROM zs_t) q) q2 BIGINT + |P3 F5 SELECT q.c FROM (SELECT l AS c FROM zs_t UNION ALL SELECT l AS c FROM zs_t) q BIGINT + |P3 F1 SELECT q.c FROM (SELECT 1 AS c FROM zs_t) q INTEGER + |P3 F2 SELECT c FROM (SELECT 1 AS c FROM zs_t) q INTEGER + |P3 F2b SELECT q.* FROM (SELECT 1 AS c FROM zs_t) q INTEGER + |P3 F3 WITH q AS (SELECT 1 AS c FROM zs_t) SELECT q.c FROM q INTEGER + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT 1 AS c FROM zs_t) q) q2 INTEGER + |P3 F5 SELECT q.c FROM (SELECT 1 AS c FROM zs_t UNION ALL SELECT 1 AS c FROM zs_t) q INTEGER + |P3 F1 SELECT q.c FROM (SELECT 'a' AS c FROM zs_t) q VARCHAR + |P3 F2 SELECT c FROM (SELECT 'a' AS c FROM zs_t) q VARCHAR + |P3 F2b SELECT q.* FROM (SELECT 'a' AS c FROM zs_t) q VARCHAR + |P3 F3 WITH q AS (SELECT 'a' AS c FROM zs_t) SELECT q.c FROM q VARCHAR + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT 'a' AS c FROM zs_t) q) q2 VARCHAR + |P3 F5 SELECT q.c FROM (SELECT 'a' AS c FROM zs_t UNION ALL SELECT 'a' AS c FROM zs_t) q VARCHAR + |P3 F1 SELECT q.c FROM (SELECT 1.5 AS c FROM zs_t) q DECIMAL(2,1) + |P3 F2 SELECT c FROM (SELECT 1.5 AS c FROM zs_t) q DECIMAL(2,1) + |P3 F2b SELECT q.* FROM (SELECT 1.5 AS c FROM zs_t) q DECIMAL(2,1) + |P3 F3 WITH q AS (SELECT 1.5 AS c FROM zs_t) SELECT q.c FROM q DECIMAL(2,1) + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT 1.5 AS c FROM zs_t) q) q2 DECIMAL(2,1) + |P3 F5 SELECT q.c FROM (SELECT 1.5 AS c FROM zs_t UNION ALL SELECT 1.5 AS c FROM zs_t) q DECIMAL(2,1) + |P3 F1 SELECT q.c FROM (SELECT TRUE AS c FROM zs_t) q BOOLEAN + |P3 F2 SELECT c FROM (SELECT TRUE AS c FROM zs_t) q BOOLEAN + |P3 F2b SELECT q.* FROM (SELECT TRUE AS c FROM zs_t) q BOOLEAN + |P3 F3 WITH q AS (SELECT TRUE AS c FROM zs_t) SELECT q.c FROM q BOOLEAN + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT TRUE AS c FROM zs_t) q) q2 BOOLEAN + |P3 F5 SELECT q.c FROM (SELECT TRUE AS c FROM zs_t UNION ALL SELECT TRUE AS c FROM zs_t) q BOOLEAN + |P3 F1 SELECT q.c FROM (SELECT NULL AS c FROM zs_t) q INTEGER + |P3 F2 SELECT c FROM (SELECT NULL AS c FROM zs_t) q INTEGER + |P3 F2b SELECT q.* FROM (SELECT NULL AS c FROM zs_t) q INTEGER + |P3 F3 WITH q AS (SELECT NULL AS c FROM zs_t) SELECT q.c FROM q INTEGER + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT NULL AS c FROM zs_t) q) q2 INTEGER + |P3 F5 SELECT q.c FROM (SELECT NULL AS c FROM zs_t UNION ALL SELECT NULL AS c FROM zs_t) q INTEGER + |P3 F1 SELECT q.c FROM (SELECT CAST(n AS BIGINT) AS c FROM zs_t) q BIGINT + |P3 F2 SELECT c FROM (SELECT CAST(n AS BIGINT) AS c FROM zs_t) q BIGINT + |P3 F2b SELECT q.* FROM (SELECT CAST(n AS BIGINT) AS c FROM zs_t) q BIGINT + |P3 F3 WITH q AS (SELECT CAST(n AS BIGINT) AS c FROM zs_t) SELECT q.c FROM q BIGINT + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT CAST(n AS BIGINT) AS c FROM zs_t) q) q2 BIGINT + |P3 F5 SELECT q.c FROM (SELECT CAST(n AS BIGINT) AS c FROM zs_t UNION ALL SELECT CAST(n AS BIGINT) AS c FROM zs_t) q BIGINT + |P3 F1 SELECT q.c FROM (SELECT n + 1 AS c FROM zs_t) q INTEGER + |P3 F2 SELECT c FROM (SELECT n + 1 AS c FROM zs_t) q INTEGER + |P3 F2b SELECT q.* FROM (SELECT n + 1 AS c FROM zs_t) q INTEGER + |P3 F3 WITH q AS (SELECT n + 1 AS c FROM zs_t) SELECT q.c FROM q INTEGER + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT n + 1 AS c FROM zs_t) q) q2 INTEGER + |P3 F5 SELECT q.c FROM (SELECT n + 1 AS c FROM zs_t UNION ALL SELECT n + 1 AS c FROM zs_t) q INTEGER + |P3 F1 SELECT q.c FROM (SELECT CONCAT(k, 'z') AS c FROM zs_t) q VARCHAR + |P3 F2 SELECT c FROM (SELECT CONCAT(k, 'z') AS c FROM zs_t) q VARCHAR + |P3 F2b SELECT q.* FROM (SELECT CONCAT(k, 'z') AS c FROM zs_t) q VARCHAR + |P3 F3 WITH q AS (SELECT CONCAT(k, 'z') AS c FROM zs_t) SELECT q.c FROM q VARCHAR + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT CONCAT(k, 'z') AS c FROM zs_t) q) q2 VARCHAR + |P3 F5 SELECT q.c FROM (SELECT CONCAT(k, 'z') AS c FROM zs_t UNION ALL SELECT CONCAT(k, 'z') AS c FROM zs_t) q VARCHAR + |P3 F1 SELECT q.c FROM (SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t) q INTEGER + |P3 F2 SELECT c FROM (SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t) q INTEGER + |P3 F2b SELECT q.* FROM (SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t) q INTEGER + |P3 F3 WITH q AS (SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t) SELECT q.c FROM q INTEGER + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t) q) q2 INTEGER + |P3 F5 SELECT q.c FROM (SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t UNION ALL SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t) q INTEGER + |P3 F1 SELECT q.c FROM (SELECT COALESCE(n, 0) AS c FROM zs_t) q INTEGER + |P3 F2 SELECT c FROM (SELECT COALESCE(n, 0) AS c FROM zs_t) q INTEGER + |P3 F2b SELECT q.* FROM (SELECT COALESCE(n, 0) AS c FROM zs_t) q INTEGER + |P3 F3 WITH q AS (SELECT COALESCE(n, 0) AS c FROM zs_t) SELECT q.c FROM q INTEGER + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT COALESCE(n, 0) AS c FROM zs_t) q) q2 INTEGER + |P3 F5 SELECT q.c FROM (SELECT COALESCE(n, 0) AS c FROM zs_t UNION ALL SELECT COALESCE(n, 0) AS c FROM zs_t) q INTEGER + |P3 F1 SELECT q.c FROM (SELECT COUNT(*) AS c FROM zs_t GROUP BY g) q BIGINT + |P3 F2 SELECT c FROM (SELECT COUNT(*) AS c FROM zs_t GROUP BY g) q BIGINT + |P3 F2b SELECT q.* FROM (SELECT COUNT(*) AS c FROM zs_t GROUP BY g) q BIGINT + |P3 F3 WITH q AS (SELECT COUNT(*) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q BIGINT + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT COUNT(*) AS c FROM zs_t GROUP BY g) q) q2 BIGINT + |P3 F5 SELECT q.c FROM (SELECT COUNT(*) AS c FROM zs_t GROUP BY g UNION ALL SELECT COUNT(*) AS c FROM zs_t GROUP BY g) q BIGINT + |P3 F1 SELECT q.c FROM (SELECT SUM(n) AS c FROM zs_t GROUP BY g) q HUGEINT + |P3 F2 SELECT c FROM (SELECT SUM(n) AS c FROM zs_t GROUP BY g) q HUGEINT + |P3 F2b SELECT q.* FROM (SELECT SUM(n) AS c FROM zs_t GROUP BY g) q HUGEINT + |P3 F3 WITH q AS (SELECT SUM(n) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q HUGEINT + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT SUM(n) AS c FROM zs_t GROUP BY g) q) q2 HUGEINT + |P3 F5 SELECT q.c FROM (SELECT SUM(n) AS c FROM zs_t GROUP BY g UNION ALL SELECT SUM(n) AS c FROM zs_t GROUP BY g) q HUGEINT + |P3 F1 SELECT q.c FROM (SELECT AVG(n) AS c FROM zs_t GROUP BY g) q DOUBLE + |P3 F2 SELECT c FROM (SELECT AVG(n) AS c FROM zs_t GROUP BY g) q DOUBLE + |P3 F2b SELECT q.* FROM (SELECT AVG(n) AS c FROM zs_t GROUP BY g) q DOUBLE + |P3 F3 WITH q AS (SELECT AVG(n) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q DOUBLE + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT AVG(n) AS c FROM zs_t GROUP BY g) q) q2 DOUBLE + |P3 F5 SELECT q.c FROM (SELECT AVG(n) AS c FROM zs_t GROUP BY g UNION ALL SELECT AVG(n) AS c FROM zs_t GROUP BY g) q DOUBLE + |P3 F1 SELECT q.c FROM (SELECT MIN(d) AS c FROM zs_t GROUP BY g) q DATE + |P3 F2 SELECT c FROM (SELECT MIN(d) AS c FROM zs_t GROUP BY g) q DATE + |P3 F2b SELECT q.* FROM (SELECT MIN(d) AS c FROM zs_t GROUP BY g) q DATE + |P3 F3 WITH q AS (SELECT MIN(d) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q DATE + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT MIN(d) AS c FROM zs_t GROUP BY g) q) q2 DATE + |P3 F5 SELECT q.c FROM (SELECT MIN(d) AS c FROM zs_t GROUP BY g UNION ALL SELECT MIN(d) AS c FROM zs_t GROUP BY g) q DATE + |P3 F1 SELECT q.c FROM (SELECT MAX(k) AS c FROM zs_t GROUP BY g) q VARCHAR + |P3 F2 SELECT c FROM (SELECT MAX(k) AS c FROM zs_t GROUP BY g) q VARCHAR + |P3 F2b SELECT q.* FROM (SELECT MAX(k) AS c FROM zs_t GROUP BY g) q VARCHAR + |P3 F3 WITH q AS (SELECT MAX(k) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q VARCHAR + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT MAX(k) AS c FROM zs_t GROUP BY g) q) q2 VARCHAR + |P3 F5 SELECT q.c FROM (SELECT MAX(k) AS c FROM zs_t GROUP BY g UNION ALL SELECT MAX(k) AS c FROM zs_t GROUP BY g) q VARCHAR + |P3 F1 SELECT q.c FROM (SELECT ARRAY_AGG(k) AS c FROM zs_t GROUP BY g) q VARCHAR[] + |P3 F2 SELECT c FROM (SELECT ARRAY_AGG(k) AS c FROM zs_t GROUP BY g) q VARCHAR[] + |P3 F2b SELECT q.* FROM (SELECT ARRAY_AGG(k) AS c FROM zs_t GROUP BY g) q VARCHAR[] + |P3 F3 WITH q AS (SELECT ARRAY_AGG(k) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q VARCHAR[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT ARRAY_AGG(k) AS c FROM zs_t GROUP BY g) q) q2 VARCHAR[] + |P3 F5 SELECT q.c FROM (SELECT ARRAY_AGG(k) AS c FROM zs_t GROUP BY g UNION ALL SELECT ARRAY_AGG(k) AS c FROM zs_t GROUP BY g) q VARCHAR[] + |P3 F1 SELECT q.c FROM (SELECT arr AS c FROM zs_t GROUP BY arr) q INTEGER[] + |P3 F2 SELECT c FROM (SELECT arr AS c FROM zs_t GROUP BY arr) q INTEGER[] + |P3 F2b SELECT q.* FROM (SELECT arr AS c FROM zs_t GROUP BY arr) q INTEGER[] + |P3 F3 WITH q AS (SELECT arr AS c FROM zs_t GROUP BY arr) SELECT q.c FROM q INTEGER[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT arr AS c FROM zs_t GROUP BY arr) q) q2 INTEGER[] + |P3 F5 SELECT q.c FROM (SELECT arr AS c FROM zs_t GROUP BY arr UNION ALL SELECT arr AS c FROM zs_t GROUP BY arr) q INTEGER[] + |P3 F1 SELECT q.c FROM (SELECT ad AS c FROM zs_t GROUP BY ad) q DATE[] + |P3 F2 SELECT c FROM (SELECT ad AS c FROM zs_t GROUP BY ad) q DATE[] + |P3 F2b SELECT q.* FROM (SELECT ad AS c FROM zs_t GROUP BY ad) q DATE[] + |P3 F3 WITH q AS (SELECT ad AS c FROM zs_t GROUP BY ad) SELECT q.c FROM q DATE[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT ad AS c FROM zs_t GROUP BY ad) q) q2 DATE[] + |P3 F5 SELECT q.c FROM (SELECT ad AS c FROM zs_t GROUP BY ad UNION ALL SELECT ad AS c FROM zs_t GROUP BY ad) q DATE[] + |P3 F1 SELECT q.c FROM (SELECT MIN(arr) AS c FROM zs_t GROUP BY g) q INTEGER[] + |P3 F2 SELECT c FROM (SELECT MIN(arr) AS c FROM zs_t GROUP BY g) q INTEGER[] + |P3 F2b SELECT q.* FROM (SELECT MIN(arr) AS c FROM zs_t GROUP BY g) q INTEGER[] + |P3 F3 WITH q AS (SELECT MIN(arr) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q INTEGER[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT MIN(arr) AS c FROM zs_t GROUP BY g) q) q2 INTEGER[] + |P3 F5 SELECT q.c FROM (SELECT MIN(arr) AS c FROM zs_t GROUP BY g UNION ALL SELECT MIN(arr) AS c FROM zs_t GROUP BY g) q INTEGER[] + |P3 F1 SELECT q.c FROM (SELECT MIN(ad) AS c FROM zs_t GROUP BY g) q DATE[] + |P3 F2 SELECT c FROM (SELECT MIN(ad) AS c FROM zs_t GROUP BY g) q DATE[] + |P3 F2b SELECT q.* FROM (SELECT MIN(ad) AS c FROM zs_t GROUP BY g) q DATE[] + |P3 F3 WITH q AS (SELECT MIN(ad) AS c FROM zs_t GROUP BY g) SELECT q.c FROM q DATE[] + |P3 F4 SELECT q2.c FROM (SELECT q.c FROM (SELECT MIN(ad) AS c FROM zs_t GROUP BY g) q) q2 DATE[] + |P3 F5 SELECT q.c FROM (SELECT MIN(ad) AS c FROM zs_t GROUP BY g UNION ALL SELECT MIN(ad) AS c FROM zs_t GROUP BY g) q DATE[] + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t INTEGER + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t BIGINT + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t DOUBLE + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t INTEGER + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t BIGINT + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t BIGINT + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t DOUBLE + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t BIGINT + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t DOUBLE + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t DOUBLE + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t DOUBLE + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t DOUBLE + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t INTEGER + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t BIGINT + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t DOUBLE + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t BOOLEAN + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t DATE + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t TIMESTAMP + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t TIMESTAMP + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t TIMESTAMP + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t VARCHAR + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t BLOB + |F7 UNION ALL SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t BLOB + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t BLOB + |F7 UNION ALL SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT n FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT l FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT x FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT b FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT d FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT ts FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT k FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT bin FROM zs_t FAILS + |F7 UNION ALL SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION ALL SELECT arr FROM zs_t INTEGER[] + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT n FROM zs_t INTEGER + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT l FROM zs_t BIGINT + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT x FROM zs_t DOUBLE + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT b FROM zs_t INTEGER + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT d FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT ts FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT k FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT bin FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT n AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT n FROM zs_t BIGINT + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT l FROM zs_t BIGINT + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT x FROM zs_t DOUBLE + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT b FROM zs_t BIGINT + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT d FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT ts FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT k FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT bin FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT l AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT n FROM zs_t DOUBLE + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT l FROM zs_t DOUBLE + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT x FROM zs_t DOUBLE + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT b FROM zs_t DOUBLE + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT d FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT ts FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT k FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT bin FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT x AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT n FROM zs_t INTEGER + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT l FROM zs_t BIGINT + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT x FROM zs_t DOUBLE + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT b FROM zs_t BOOLEAN + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT d FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT ts FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT k FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT bin FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT b AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT n FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT l FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT x FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT b FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT d FROM zs_t DATE + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT ts FROM zs_t TIMESTAMP + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT k FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT bin FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT d AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT n FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT l FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT x FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT b FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT d FROM zs_t TIMESTAMP + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT ts FROM zs_t TIMESTAMP + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT k FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT bin FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT ts AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT n FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT l FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT x FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT b FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT d FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT ts FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT k FROM zs_t VARCHAR + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT bin FROM zs_t BLOB + |F7 UNION SELECT q.c FROM (SELECT k AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT n FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT l FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT x FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT b FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT d FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT ts FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT k FROM zs_t BLOB + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT bin FROM zs_t BLOB + |F7 UNION SELECT q.c FROM (SELECT bin AS c FROM zs_t) q UNION SELECT arr FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT n FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT l FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT x FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT b FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT d FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT ts FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT k FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT bin FROM zs_t FAILS + |F7 UNION SELECT q.c FROM (SELECT arr AS c FROM zs_t) q UNION SELECT arr FROM zs_t INTEGER[] + |F7 UNION ALL SELECT q.items FROM (SELECT items FROM zs_t) q UNION ALL SELECT items FROM zs_u STRUCT(q INTEGER, p VARCHAR, r INTEGER)[] + |""".stripMargin.split("\n").toSeq.filter(_.nonEmpty).map { line => + val Array(kind, form, sql, t) = line.split("\t") + (kind, form, sql, t) + } + + /** A core type spelled as DuckDB spells it. */ + private def duck(t: SetOperationType): String = + t.sqlType match { + case SQLTypes.TinyInt => "TINYINT" + case SQLTypes.SmallInt => "SMALLINT" + case SQLTypes.Int => "INTEGER" + case SQLTypes.BigInt => "BIGINT" + case SQLTypes.Real => "FLOAT" + case SQLTypes.Double => "DOUBLE" + case SQLTypes.Boolean => "BOOLEAN" + case SQLTypes.Varchar => "VARCHAR" + case SQLTypes.Date => "DATE" + case SQLTypes.Time => "TIME" + case SQLTypes.Timestamp => "TIMESTAMP" + case SQLTypes.VarBinary => "BLOB" + case SQLTypes.GeoPoint => "STRUCT(lat DOUBLE, lon DOUBLE)" + case SQLTypes.Struct => + t.fields.map(f => s"${f.name} ${duck(f.valueType)}").mkString("STRUCT(", ", ", ")") + case SQLTypes.Array(_) => + s"${duck(t.element.getOrElse(SetOperationType(SQLTypes.Any)))}[]" + case other => other.typeId + } + + private def unformatted(t: SetOperationType): SetOperationType = + t.copy( + fields = t.fields.map(f => f.copy(valueType = unformatted(f.valueType))), + element = t.element.map(unformatted), + format = None + ) + + /** The core type of the statement's first column, once its derived tables are attached. */ + private def coreType(sql: String): Either[String, SetOperationType] = + Parser(sql) match { + case Right(single: SingleSearch) => + DerivedTable.attach(single, lookup).flatMap { attached => + if (attached.select.fields.exists(_.identifier.identifierName.endsWith("*"))) + attached.schema + .flatMap(_.columns.headOption) + .map(MultiSearch.columnType) + .toRight(s"[$sql] no schema attached for its *") + else + MultiSearch + .projectionTypes(attached) + .flatMap(_.headOption.map(_._2)) + .toRight(s"[$sql] declares nothing") + } + case other => Left(s"[$sql] is not one SELECT: $other") + } + + /** The listed divergences, each with its rationale: (a fragment of the statement, the core type, + * DuckDB's type, why). + */ + private val divergences: Seq[(String, String, String, String)] = Seq( + // D7 / A-27: Elasticsearch buckets and aggregates a list field ELEMENT by element -- core + // answers the element, DuckDB a list it groups or orders as a whole (core cannot answer + // DuckDB's value) + ("SELECT arr AS c FROM zs_t GROUP BY arr", "INTEGER", "INTEGER[]", "A-27 GROUP BY key"), + ("SELECT ad AS c FROM zs_t GROUP BY ad", "DATE", "DATE[]", "A-27 GROUP BY key"), + ("SELECT MIN(arr) AS c FROM zs_t GROUP BY g", "INTEGER", "INTEGER[]", "D7 metric"), + ("SELECT MIN(ad) AS c FROM zs_t GROUP BY g", "DATE", "DATE[]", "D7 metric"), + // AggregateReportedTypeSpec (A-20): a whole SUM is DuckDB's HUGEINT, of which core's BIGINT + // holds every sum Elasticsearch computes below 2^63 + ("SELECT SUM(n) AS c FROM zs_t GROUP BY g", "BIGINT", "HUGEINT", "A-20 SUM"), + // #420: a bare NULL has no type in core -- it types nothing beside other branches + ("SELECT NULL AS c FROM zs_t", "NULL", "INTEGER", "#420 NULL"), + // PRE-EXISTING, inherited from the projection item (one rule, #292: the plain `SELECT ` + // reads the same, asserted below): a whole-number literal is a BIGINT operand in core's + // arithmetic and conditionals, an INTEGER in DuckDB's -- recorded, not fixed here + ("SELECT n + 1 AS c FROM zs_t", "BIGINT", "INTEGER", "literal width (pre-existing)"), + ( + "SELECT CASE WHEN b THEN n ELSE 0 END AS c FROM zs_t", + "BIGINT", + "INTEGER", + "literal width (pre-existing)" + ), + ("SELECT COALESCE(n, 0) AS c FROM zs_t", "BIGINT", "INTEGER", "literal width (pre-existing)") + ) + + it should "type every derived column as the plain projection item of its body (#292)" in { + val bodies = measured.filter(_._1 == "P3").map(_._3).collect { + case sql if sql.startsWith("SELECT q.c FROM (") && !sql.contains("UNION ALL") => + sql.stripPrefix("SELECT q.c FROM (").stripSuffix(") q") + } + bodies.size shouldBe 36 + bodies.foreach { body => + val plain = Parser(body) match { + case Right(s: SingleSearch) => s.update(Some(zsT)) + case other => fail(s"[$body] $other") + } + val projected = MultiSearch.projectionTypes(plain).flatMap(_.headOption.map(_._2)) + // a derived temporal column reads Elasticsearch's default format where its item had none (an + // aggregate): the column carries one, as every mapped date does + withClue(s"[$body] ") { + coreType(s"SELECT q.c FROM ($body) q").toOption.map(unformatted) shouldBe + projected.map(unformatted) + } + } + } + + /** SCHEMA-5's cells (D12): a NUMERIC / TEMPORAL answer on either side -- a DECIMAL literal, a + * number of unknown width. + */ + private def schema5Owned(core: String, duckdb: String): Boolean = + duckdb.startsWith("DECIMAL") || core == "NUMERIC" || core == "TEMPORAL" + + behavior of "a derived or CTE column, typed from its body (D10, AC3)" + + it should "equal DuckDB 1.5.5's DESCRIBE on every P3 cell, but the listed divergences" in { + val cells = measured.filter(_._1 == "P3") + cells.size shouldBe 216 + var excluded = 0 + var diverging = 0 + val mismatches = cells.flatMap { case (_, form, sql, expected) => + coreType(sql) match { + case Left(error) => Some(s"$form [$sql] refused: $error (DuckDB $expected)") + case Right(t) => + val actual = duck(t) + if (actual == expected) None + else if (schema5Owned(actual, expected)) { excluded += 1; None } + else + divergences.find { case (fragment, c, d, _) => + sql.contains(fragment) && c == actual && d == expected + } match { + case Some(_) => diverging += 1; None + case None => Some(s"$form [$sql] core $actual, DuckDB $expected") + } + } + } + withClue( + s"${mismatches.size} mismatches over ${cells.size} cells ($diverging listed divergences, " + + s"$excluded SCHEMA-5 cells excluded):\n${mismatches.mkString("\n")}\n" + ) { + mismatches shouldBe empty + } + // every listed divergence is met in every form + diverging shouldBe divergences.size * 6 + } + + it should "type Case 14 INT: qualified, unqualified, two levels, q.* and WITH (AC1)" in { + Seq( + "SELECT q.c1 FROM (SELECT n AS c1 FROM zs_t) q", + "SELECT c1 FROM (SELECT n AS c1 FROM zs_t) q", + "SELECT q2.c1 FROM (SELECT q.c1 FROM (SELECT n AS c1 FROM zs_t) q) q2", + "SELECT q.* FROM (SELECT n AS c1 FROM zs_t) q", + "WITH q AS (SELECT n AS c1 FROM zs_t) SELECT q.c1 FROM q" + ).foreach { sql => + withClue(s"[$sql] ")(coreType(sql).map(_.sqlType) shouldBe Right(SQLTypes.Int)) + } + } + + it should "leave an UNattached derived column ANY (R-6: plain update never types one)" in { + Parser("SELECT q.c1 FROM (SELECT n AS c1 FROM zs_t) q") match { + case Right(single: SingleSearch) => + val plain = single.update(Some(zsT)) + MultiSearch.projectionTypes(plain).map(_.map(_._2.sqlType)) shouldBe Some( + Seq(SQLTypes.Any) + ) + case other => fail(s"$other") + } + } + + behavior of "a derived column beside a plain one in a set operation (F7, AC2)" + + private def branches( + sql: String, + attached: Boolean + ): (Seq[SingleSearch], Seq[app.softnetwork.elastic.sql.operator.SetOperator]) = + Parser(sql) match { + case Right(m: MultiSearch) => + val requests = + if (attached) + m.requests.map(r => DerivedTable.attach(r, lookup).fold(e => fail(e), identity)) + else m.requests.map(_.update(Some(zsT))) + (requests, m.resolvedOperators) + case other => fail(s"[$sql] is not a set operation: $other") + } + + it should "refuse only what DuckDB fails on, and answer every statement DuckDB runs" in { + val cells = measured.filter(_._1 == "F7") + cells.size shouldBe 163 + val wrong = cells.flatMap { case (_, op, sql, expected) => + val (requests, operators) = branches(sql, attached = true) + val verdict = MultiSearch.branchTypes(requests, operators) + (verdict, expected) match { + case (Left(refusal), "FAILS") => None + case (Right(_), "FAILS") => None // answered as VARCHAR where DuckDB gives no type + case (Left(refusal), t) => Some(s"$op [$sql] refused ($refusal), DuckDB $t") + case (Right(_), _) => None + } + } + withClue(s"${wrong.size} wrong verdicts:\n${wrong.mkString("\n")}\n")(wrong shouldBe empty) + // and every refusal core makes is one DuckDB makes, at the column it makes it + val refused = cells.count { case (_, _, sql, _) => + val (requests, operators) = branches(sql, attached = true) + MultiSearch.branchTypes(requests, operators).isLeft + } + refused should be > 0 + } + + it should "refuse Case 15 with #420 rule 1's message, a DATE beside an INT" in { + val sql = + "SELECT d AS k FROM zs_t UNION ALL SELECT q.c1 FROM (SELECT n AS c1 FROM zs_t) q" + val (requests, operators) = branches(sql, attached = true) + MultiSearch.branchTypes(requests, operators) shouldBe Left( + "Set operation branches must project compatible types at column 1: branch 1 'k' is DATE, " + + "branch 2 'c1' is INT" + ) + // unattached, the derived branch is ANY and the statement passes (R-6) + val (plain, plainOperators) = branches(sql, attached = false) + MultiSearch.branchTypes(plain, plainOperators) shouldBe Right(()) + MultiSearch + .columnTypes(plain, plainOperators) + .map(_.head.branches.map(_.map(_.sqlType))) shouldBe Right( + Seq(Some(SQLTypes.Date), Some(SQLTypes.Any)) + ) + } + + it should "merge lists of objects field by field, as DuckDB does (#419 item 2)" in { + val (_, op, sql, expected) = measured.filter(_._1 == "F7").last + op shouldBe "UNION ALL" + expected shouldBe "STRUCT(q INTEGER, p VARCHAR, r INTEGER)[]" + val (requests, operators) = branches(sql, attached = true) + MultiSearch.columnTypes(requests, operators).map(columns => duck(columns.head.result)) shouldBe + Right(expected) + } + + behavior of "MultiSearch.columnOf" + + it should "be the inverse of columnType over every zs_t column and a derived struct" in { + val types: Seq[SetOperationType] = zsT.columns.map(MultiSearch.columnType) ++ Seq( + SetOperationType(SQLTypes.Null), + SetOperationType(SQLTypes.Any), + SetOperationType( + SQLTypes.Struct, + Seq( + StructField("a", SetOperationType(SQLTypes.Int)), + StructField("z", SetOperationType.GeoPoint) + ) + ), + SetOperationType( + SQLTypes.Array(SQLTypes.Date), + element = Some(SetOperationType(SQLTypes.Date, format = Some("yyyy-MM-dd"))) + ) + ) + types.foreach { t => + withClue(s"$t: ")(MultiSearch.columnType(MultiSearch.columnOf("c", t)) shouldBe t) + } + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ElementViewTwinSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ElementViewTwinSpec.scala new file mode 100644 index 000000000..93f1b9460 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ElementViewTwinSpec.scala @@ -0,0 +1,329 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.util.Try + +/** SCHEMA-6 (D7, A-27) -- an `ARRAY` column used where Elasticsearch works per ELEMENT reads as + * its element X: a term or range query matches any element, a metric aggregates the elements, a + * terms aggregation buckets them, and Painless reads ONE (`doc['f'].value`). + * + * The oracle is a TWIN: the same statement over a column of the scalar type X. Over every P2 + * statement and every element type, the `ARRAY` column must get the twin's verdict, the twin's + * Painless (projection and predicate), and -- for every item that is not the bare column itself (a + * computed item, an aggregate, a GROUP BY key) -- the twin's type. So no computed item over a list + * column reports `ARRAY` (which would make the scripted reader wrap ONE element in a list: + * `NULLIF(arr, 0)` on `[5, 9]` answering `List(5)`), and no comparison is refused for the column + * being a list. + * + * On 0.24.0 every array column read `ANY`; typing it `ARRAY` without this view refused or + * re-typed exactly these cells. + */ +class ElementViewTwinSpec extends AnyFlatSpec with Matchers { + + /** `extra`: items typed from their operand outside the shared argument helpers -- `ABS` and + * `DATE_ADD` / `DATE_SUB` read their operand's own type, so each needs its element read too. + */ + private final case class Element( + x: SQLType, + v: String, + w: String, + fn: String, + arith: Option[String], + extra: Seq[String] = Nil + ) + + /** SCHEMA-5's operand-typed numeric sites, read through the element too: ROUND takes its + * operand's type (to tens, hundreds, … a narrower integer is a BIGINT, a BIGINT tests its + * range), FLOOR and CEIL are BIGINT, and every consumer of ROUND's type follows (a metric, a + * conditional, arithmetic, a comparison whose quoted number is read beside it). + */ + private val rounding: Seq[String] = Seq( + "ROUND(c)", + "ROUND(c, 1)", + "ROUND(c, -1)", + "ROUND(c, -2)", + "FLOOR(c)", + "CEIL(c)", + "MAX(ROUND(c, -1))", + "SUM(ROUND(c, -1))", + "MIN(ROUND(c, 1))", + "ROUND(c, -1) + 1", + "COALESCE(ROUND(c, -1), 0)", + "NULLIF(ROUND(c, -1), 10)", + "CASE WHEN b THEN ROUND(c, -1) ELSE 0 END" + ) + + private val elements: Seq[Element] = Seq( + Element(SQLTypes.Int, "1", "5", "ABS(c)", Some("c + 1"), rounding), + Element(SQLTypes.BigInt, "1", "5", "ABS(c)", Some("c + 1"), rounding), + Element(SQLTypes.TinyInt, "1", "5", "ABS(c)", Some("c + 1"), rounding), + Element(SQLTypes.Double, "1.5", "9.5", "ABS(c)", Some("c + 1"), rounding), + Element(SQLTypes.Real, "1.5", "9.5", "ABS(c)", Some("c + 1"), rounding), + Element(SQLTypes.Keyword, "'a'", "'z'", "UPPER(c)", None), + Element( + SQLTypes.Date, + "'2024-01-01'", + "'2024-12-31'", + "YEAR(c)", + Some("c + 1"), + Seq( + "DATE_ADD(c, INTERVAL 1 DAY)", + "DATE_SUB(c, INTERVAL 1 DAY)", + // SCHEMA-5: DATE_TRUNC over a date reports TIMESTAMP in a query, DATEDIFF a BIGINT + "DATE_TRUNC(c, DAY)", + "DATEDIFF(c, '2024-01-01')", + "c + DATEDIFF(c, '2024-01-01')" + ) + ), + Element(SQLTypes.Boolean, "TRUE", "FALSE", "CAST(c AS VARCHAR)", None) + ) + + private def statements(e: Element): Seq[String] = + Seq( + s"SELECT g FROM t WHERE c = ${e.v}", + s"SELECT g FROM t WHERE c IN (${e.v}, ${e.w})", + s"SELECT g FROM t WHERE c BETWEEN ${e.v} AND ${e.w}", + s"SELECT g FROM t WHERE c > ${e.v}", + "SELECT g FROM t WHERE c LIKE 'a%'", + "SELECT g FROM t WHERE c IS NULL", + s"SELECT g FROM t WHERE COALESCE(c, c) = ${e.v}", + s"SELECT ${e.fn} AS r FROM t", + "SELECT CAST(c AS VARCHAR) AS r FROM t", + "SELECT c, COUNT(*) AS n FROM t GROUP BY c", + "SELECT g FROM t ORDER BY c", + "SELECT MIN(c) AS r FROM t", + "SELECT MAX(c) AS r FROM t", + "SELECT SUM(c) AS r FROM t", + "SELECT COUNT(c) AS r FROM t", + "SELECT g, MAX(c) AS r FROM t GROUP BY g", + s"SELECT NULLIF(c, ${e.v}) AS r FROM t", + "SELECT COALESCE(c, c) AS r FROM t", + "SELECT CASE WHEN b THEN c END AS r FROM t" + ) ++ e.arith.map(a => s"SELECT $a AS r FROM t") ++ e.extra.map(x => s"SELECT $x AS r FROM t") + + private def schema(c: SQLType): SchemaTable = + SchemaTable( + "t", + columns = List(Column("c", c), Column("b", SQLTypes.Boolean), Column("g", SQLTypes.Keyword)) + ) + + private final case class Outcome( + accepted: Boolean, + painless: Seq[Option[String]], + types: Seq[Option[MultiSearch.SetOperationType]], + reported: Seq[Option[SQLType]] + ) + + private def outcome(sql: String, against: SchemaTable): Outcome = + Parser(sql) match { + case Right(single: SingleSearch) => + val resolved = single.update(Some(against)) + val accepted = resolved.validateResolved().isRight + val fields = resolved.select.fieldsWithComputedAliases + // the bare column item is declared `ARRAY` on purpose; every other item is compared + def compared(f: Field): Boolean = + f.identifier.functions.nonEmpty || f.isAggregation || + resolved.groupBy.exists( + _.buckets.exists(_.identifier.identifierName == f.identifier.identifierName) + ) + val painless = fields.map { f => + Try { + val ctx = PainlessContext(PainlessContextType.Query) + val body = f.painless(Some(ctx)) + s"$ctx$body" + }.toOption + } ++ Seq( + resolved.where.flatMap(_.criteria).flatMap { criteria => + Try { + val ctx = PainlessContext(PainlessContextType.Query) + val body = criteria.painless(Some(ctx)) + s"$ctx$body" + }.toOption + } + ) + val types = + fields.map(f => if (compared(f)) Some(MultiSearch.fieldType(f, resolved)) else None) + // a GROUP BY key keeps its column's declared type as its REPORTED type (CTAS declares it, + // A-27); its position is typed by its key ([[MultiSearch.fieldType]], compared above) + def groupKey(f: Field): Boolean = + !f.isAggregation && f.identifier.functions.isEmpty && + resolved.groupBy.exists( + _.buckets.exists(_.identifier.identifierName == f.identifier.identifierName) + ) + val reported = + fields.map(f => + if (compared(f) && !groupKey(f)) Some(f.identifier.reportedType) else None + ) + Outcome(accepted, painless, types, reported) + case Right(other) => fail(s"[$sql] is not one SELECT: $other") + case Left(error) => Outcome(accepted = false, Nil, Nil, Nil) + } + + behavior of "an ARRAY column used per element" + + for (e <- elements) { + it should s"read as its twin X column for X = ${e.x.typeId} (D7, A-27)" in { + val failures = statements(e).flatMap { sql => + val list = outcome(sql, schema(SQLTypes.Array(e.x))) + val twin = outcome(sql, schema(e.x)) + Seq( + if (list.accepted != twin.accepted) + Some(s"[$sql] verdict: ARRAY ${list.accepted}, twin ${twin.accepted}") + else None, + if (list.painless != twin.painless) + Some(s"[$sql] Painless: ARRAY ${list.painless}, twin ${twin.painless}") + else None, + if (list.types != twin.types) + Some(s"[$sql] type: ARRAY ${list.types}, twin ${twin.types}") + else None, + if (list.reported != twin.reported) + Some(s"[$sql] reported: ARRAY ${list.reported}, twin ${twin.reported}") + else None + ).flatten + } + withClue( + s"${failures.size} differences over ${statements(e).size} statements:\n" + + failures.mkString("\n") + ) { + failures shouldBe empty + } + } + } + + it should "keep the declared ARRAY where a column is DECLARED: the bare item, CTAS, FIRST_VALUE" in { + val t = schema(SQLTypes.Array(SQLTypes.Int)) + val bare = Parser("SELECT c FROM t") match { + case Right(s: SingleSearch) => s.update(Some(t)) + case other => fail(s"$other") + } + MultiSearch.fieldType(bare.select.fields.head, bare).sqlType shouldBe + SQLTypes.Array(SQLTypes.Int) + t.mergeWithSearch(bare).columns.find(_.name == "c").map(_.dataType) shouldBe + Some(SQLTypes.Array(SQLTypes.Int)) + val first = Parser("SELECT FIRST_VALUE(c) OVER (PARTITION BY g ORDER BY b) AS r FROM t") match { + case Right(s: SingleSearch) => s.update(Some(t)) + case other => fail(s"$other") + } + first.select.fields.head.identifier.reportedType shouldBe SQLTypes.Array(SQLTypes.Int) + } + + it should "type a window PARTITION BY over an ARRAY column as 0.24.0 read it (A-28)" in { + val t = schema(SQLTypes.Array(SQLTypes.Int)) + val windowed = Parser("SELECT g, COUNT(*) OVER (PARTITION BY c) AS n FROM t") match { + case Right(s: SingleSearch) => s.update(Some(t)) + case other => fail(s"$other") + } + val level = windowed.windowFunctions.flatMap(_.buckets).head + MultiSearch.partitionType(level).sqlType shouldBe SQLTypes.Any + MultiSearch.levelType(windowed, level).sqlType shouldBe SQLTypes.Any + MultiSearch.bucketType(level).sqlType shouldBe SQLTypes.Int + val grouped = Parser("SELECT c, COUNT(*) AS n FROM t GROUP BY c") match { + case Right(s: SingleSearch) => s.update(Some(t)) + case other => fail(s"$other") + } + val key = grouped.groupBy.get.buckets.head + MultiSearch.levelType(grouped, grouped.buckets.head).sqlType shouldBe SQLTypes.Int + MultiSearch.bucketType(key).sqlType shouldBe SQLTypes.Int + } + + private def resolve(sql: String, t: SchemaTable): SingleSearch = + Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(t)) + case other => fail(s"[$sql] $other") + } + + /** What Elasticsearch is SENT for the statement's quoted numbers: a bare column's term, range, + * terms and BETWEEN bounds, and each GROUP BY level's key filter for `'1.5'`. + */ + private def sent(sql: String, t: SchemaTable): Seq[Any] = { + val s = resolve(sql, t) + (s.where.flatMap(_.criteria) match { + case Some(e: GenericExpression) => Seq(e.queryText(e.value)) + case Some(in: InExpr[_, _]) => in.queryValues + case Some(between: BetweenExpr) => + Seq(between.boundQueryText(between.fromTo.from), between.boundQueryText(between.fromTo.to)) + case None => Nil + case other => fail(s"[$sql] $other") + }) ++ s.groupBy.toSeq.flatMap(_.buckets).map(_.keyFilterTexts("1.5")) + } + + // SCHEMA-5 (R-54) x D7: a quoted number beside a whole column is the whole number it rounds to, + // and that number is what Elasticsearch is sent -- a term, a range, a terms query, a BETWEEN, a + // HAVING key filter. An `ARRAY` column is matched per element, so it sends what its X twin + // sends: sent as written, `'1.5'` matched no element of an integer field (term) or the group 1 + // (terms include, the floor of 1.5) where the twin keeps 2 + it should "send a quoted fractional number to Elasticsearch as its twin does (R-54)" in { + for (x <- Seq(SQLTypes.TinyInt, SQLTypes.SmallInt, SQLTypes.Int, SQLTypes.BigInt)) { + for ( + sql <- Seq( + "SELECT g FROM t WHERE c = '1.5'", + "SELECT g FROM t WHERE c > '1.5'", + "SELECT g FROM t WHERE c IN ('1.5', '3')", + "SELECT g FROM t WHERE c BETWEEN '1.5' AND '4.5'", + "SELECT c, COUNT(*) AS n FROM t GROUP BY c" + ) + ) withClue(s"[${x.typeId}] [$sql] ") { + val twin = sent(sql, schema(x)) + twin should not be empty + sent(sql, schema(SQLTypes.Array(x))) shouldBe twin + } + } + val ints = schema(SQLTypes.Array(SQLTypes.Int)) + sent("SELECT g FROM t WHERE c = '1.5'", ints) shouldBe Seq(Some("2")) + sent("SELECT c, COUNT(*) AS n FROM t GROUP BY c", ints) shouldBe Seq(Set("2")) + } + + // R-66 -- outside a query the element view is off: an ingest processor and a transform read the + // whole value (`ctx.`), and the validation of a computed column judges that processor. So + // ROUND over an `ARRAY` column renders and reports there what it does over an untyped column, + // the reading 0.24.0 gave every list column + it should "keep ROUND over an ARRAY column untyped outside a query (R-66)" in { + val processor = Some(PainlessContext(context = PainlessContextType.Processor)) + for ( + x <- Seq(SQLTypes.Int, SQLTypes.BigInt, SQLTypes.Real, SQLTypes.Double); + e <- Seq("ROUND(c)", "ROUND(c, 1)", "ROUND(c, -1)") + ) { + val sql = s"SELECT $e AS r FROM t" + def item(t: SchemaTable): Field = resolve(sql, t).select.fields.head + withClue(s"[${x.typeId}] [$e] ") { + val list = item(schema(SQLTypes.Array(x))) + val untyped = item(schema(SQLTypes.Any)) + val listContext = PainlessContext(context = PainlessContextType.Processor) + val untypedContext = PainlessContext(context = PainlessContextType.Processor) + s"$listContext${list.painless(Some(listContext))}" shouldBe + s"$untypedContext${untyped.painless(Some(untypedContext))}" + def inProcessor(f: Field): SQLType = + SQLTypeUtils.renderingIn(processor)(f.identifier.reportedType) + inProcessor(item(schema(SQLTypes.Array(x)))) shouldBe inProcessor(untyped) + SQLTypeUtils.validatingComputedColumns( + item(schema(SQLTypes.Array(x))).identifier.reportedType + ) shouldBe SQLTypes.Numeric + // in a query, the twin's + item(schema(SQLTypes.Array(x))).identifier.reportedType shouldBe + item(schema(x)).identifier.reportedType + } + } + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/ArrayDeclarationDiffSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/ArrayDeclarationDiffSpec.scala new file mode 100644 index 000000000..564c770f0 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/ArrayDeclarationDiffSpec.scala @@ -0,0 +1,200 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.schema + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{AlterTable, CreateTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import com.fasterxml.jackson.databind.ObjectMapper +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** SCHEMA-6 (D4, D5, D6) -- declaring an existing field a list is a META-ONLY change. + * + * The remedy a several-valued column fails with (lead Q1) is `ALTER TABLE t ALTER COLUMN c SET + * DATA TYPE ARRAY` -- and for a sub-field `ALTER COLUMN p SET FIELD >`. + * Its only diff is the declared type in the table's `_meta` (`Column.diff` compares Elasticsearch + * types, and `ARRAY` maps to X's), and that key is now `Safe` (it was UNSAFE by default, so the + * ALTER reindexed: MEASURED on main, ES 8, a `_meta`-only column diff logs `requires REINDEX`). A + * change of X itself on a list column stays `Impossible`, as on 0.24.0 where every list read ANY. + */ +class ArrayDeclarationDiffSpec extends AnyFlatSpec with Matchers { + + private val mapper = new ObjectMapper() + + /** The table core reads for an index created by a raw mapping (no `_meta`): `zs_ext`. */ + private val rawMapping: Table = + Index( + name = "zs_ext", + mappings = mapper.readTree( + """{"mappings": {"properties": { + | "id": {"type": "keyword"}, + | "tags": {"type": "keyword"}, + | "n": {"type": "integer"}, + | "s": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}}, + | "ts": {"type": "date"}, + | "geo": {"type": "geo_point"}, + | "obj": {"properties": { + | "a": {"type": "integer"}, + | "f": {"type": "keyword", "ignore_above": 256, "fields": {"raw": {"type": "keyword"}}}, + | "x": {"properties": {"y": {"type": "integer"}}} + | }} + |}}}""".stripMargin + ), + settings = mapper.readTree("""{"settings": {}}""") + ).schema + + private def alter(table: Table, sql: String): Table = + Parser(sql) match { + case Right(at: AlterTable) => table.merge(at.statements) + case other => fail(s"[$sql] is not an ALTER TABLE: $other") + } + + /** The remedy core names for the column at `path` (HitReading's rendering): a column's `SET DATA + * TYPE ARRAY`, a sub-field's `SET FIELD` with the sub-field as declared, typed `ARRAY`. + */ + private def remedy(table: Table, path: String): String = { + val c = table.find(path).getOrElse(fail(s"no column $path")) + val list = SQLTypes.Array(c.dataType) + val dot = path.lastIndexOf('.') + if (dot < 0) s"ALTER TABLE ${table.name} ALTER COLUMN $path SET DATA TYPE ${list.typeId}" + else { + val field = c.copy(dataType = list, struct = None).update() + s"ALTER TABLE ${table.name} ALTER COLUMN ${path.substring(0, dot)} SET FIELD " + + field.sql.replaceAll("\\n\\t*", " ") + } + } + + behavior of "declaring an existing field a list (D4)" + + it should "diff KEYWORD -> ARRAY as ONE Safe _meta data_type key" in { + val desired = alter(rawMapping, remedy(rawMapping, "tags")) + desired.find("tags").map(_.dataType) shouldBe Some(SQLTypes.Array(SQLTypes.Keyword)) + val diff = rawMapping.diff(desired) + diff.columns shouldBe empty + diff.mappings.collect { case MappingSet(key, value) => key -> value.value } shouldBe + List("_meta.columns.tags.data_type" -> "ARRAY") + diff.safety shouldBe Safe + } + + it should "be meta-only for every scalar zs_ext shape: a number, a dynamic string, a date, a point" in { + for (c <- Seq("n", "s", "ts", "geo")) { + val desired = alter(rawMapping, remedy(rawMapping, c)) + val diff = rawMapping.diff(desired) + withClue(s"$c: columns=${diff.columns} mappings=${diff.mappings}: ") { + diff.columns shouldBe empty + diff.mappings.collect { case MappingSet(key, _) => key } shouldBe + List(s"_meta.columns.$c.data_type") + diff.safety shouldBe Safe + } + } + // the dynamic string keeps its `keyword` multi-field and its `ignore_above` + val s = alter(rawMapping, remedy(rawMapping, "s")).find("s").get + s.dataType shouldBe SQLTypes.Array(SQLTypes.Text) + s.multiFields.map(_.name) shouldBe List("keyword") + } + + it should "be meta-only for a sub-field, its options and multi-fields kept by the rendered SET FIELD" in { + for (path <- Seq("obj.a", "obj.f", "obj.x.y")) { + val statement = remedy(rawMapping, path) + val desired = alter(rawMapping, statement) + val diff = rawMapping.diff(desired) + val segments = path.split("\\.") + val key = + s"_meta.columns.${segments.head}" + segments.tail.map(s => s".multi_fields.$s").mkString + + ".data_type" + withClue(s"$statement: columns=${diff.columns} mappings=${diff.mappings}: ") { + desired.find(path).map(_.dataType.typeId).getOrElse("") should startWith("ARRAY<") + // the Elasticsearch mapping is the same bytes: only `_meta` changes (a sub-field whose + // options re-parse as another number class reports a no-op `FieldAltered`, Safe) + desired.indexMappings.path("properties").toString shouldBe + rawMapping.indexMappings.path("properties").toString + diff.columns.filterNot(_.isInstanceOf[FieldAltered]) shouldBe empty + diff.mappings.collect { case MappingSet(k, _) => k } shouldBe List(key) + diff.safety shouldBe Safe + } + } + val f = alter(rawMapping, remedy(rawMapping, "obj.f")).find("obj.f").get + f.options.keySet should contain("ignore_above") + f.multiFields.map(_.name) shouldBe List("raw") + } + + it should "keep a sub-field's DEFAULT, COMMENT and NOT NULL (an SQL-created table)" in { + val declared = Parser( + "CREATE TABLE t (id INT, obj STRUCT FIELDS (" + + "f KEYWORD FIELDS (raw KEYWORD) DEFAULT 'none' NOT NULL COMMENT 'a note' " + + "OPTIONS (ignore_above = 256), a INT))" + ) match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + val statement = remedy(declared, "obj.f") + val desired = alter(declared, statement) + val diff = declared.diff(desired) + withClue(s"$statement: columns=${diff.columns} mappings=${diff.mappings}: ") { + desired.indexMappings.path("properties").toString shouldBe + declared.indexMappings.path("properties").toString + diff.columns.filterNot(_.isInstanceOf[FieldAltered]) shouldBe empty + diff.mappings.collect { case MappingSet(k, _) => k } shouldBe + List("_meta.columns.obj.multi_fields.f.data_type") + diff.safety shouldBe Safe + // the sub-field keeps its position (D9) + desired.find("obj").get.multiFields.map(_.name) shouldBe List("f", "a") + } + } + + it should "keep today's safety for every other _meta key (a COMMENT reindexes, as measured)" in { + val declared = Parser("CREATE TABLE t (id INT, c KEYWORD)") match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + val diff = declared.diff(alter(declared, "ALTER TABLE t ALTER COLUMN c SET COMMENT 'x'")) + diff.mappings.collect { case MappingSet(k, _) => k } should contain( + "_meta.columns.c.comment" + ) + diff.safety shouldBe UnsafeReindex + MappingsRules.isSafe("_meta.columns.c.comment") shouldBe false + MappingsRules.isSafe("_meta.columns.c.data_type") shouldBe true + MappingsRules.isSafe("_meta.columns.c.multi_fields.f.data_type") shouldBe true + MappingsRules.isSafe("_meta.columns.c.field_order") shouldBe true + MappingsRules.isSafe("_meta.columns.c.multi_fields.f.field_order") shouldBe true + MappingsRules.isSafe("_meta.primary_key") shouldBe false + MappingsRules.isSafe("_meta.columns.c.script") shouldBe false + } + + behavior of "changing X itself on a list column (D5)" + + it should "stay Impossible, as on 0.24.0" in { + val declared = Parser("CREATE TABLE t (id INT, arr ARRAY, k KEYWORD)") match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + Seq( + "ALTER TABLE t ALTER COLUMN arr SET DATA TYPE ARRAY", + "ALTER TABLE t ALTER COLUMN arr SET DATA TYPE ARRAY", + "ALTER TABLE t ALTER COLUMN k SET DATA TYPE ARRAY" + ).foreach { sql => + val diff = declared.diff(alter(declared, sql)) + withClue(s"$sql: ${diff.columns}: ") { + diff.columns.collect { case c: ColumnTypeChanged => c } should not be empty + diff.safety shouldBe Impossible + } + } + ColumnTypeChanged("c", SQLTypes.Int, SQLTypes.BigInt).safety shouldBe UnsafeReindex + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/ScriptReferenceValidationSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/ScriptReferenceValidationSpec.scala index dfa9732f5..80b6c19f4 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/ScriptReferenceValidationSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/ScriptReferenceValidationSpec.scala @@ -104,6 +104,201 @@ class ScriptReferenceValidationSpec } } + /** SCHEMA-6 (A-30) -- an `ARRAY` column handed to a NUMERIC or TEMPORAL function input is + * refused by its DECLARED type, as 0.24.0 refused it on CREATE. An ingest processor reads + * `ctx.`, the whole `_source` value -- the list -- and not one element, so the element view + * Painless has in a query cannot be honoured there: measured on ES 8.18.3, `c INT SCRIPT AS (arr + * + 1)` stored NULL for `[5, 9]` and for `[5]`, and `YEAR(ad)` NULL for every document, the + * processor's failure swallowed by `ignore_failure`. + */ + "a NUMERIC or TEMPORAL function over an ARRAY column" should "be refused on CREATE, naming the column" in { + forAll( + Table( + ("ddl", "column", "declared"), + ("CREATE TABLE t (arr ARRAY, c INT SCRIPT AS (arr + 1))", "arr", "ARRAY"), + ("CREATE TABLE t (ad ARRAY, cy INT SCRIPT AS (YEAR(ad)))", "ad", "ARRAY") + ) + ) { (ddl, column, declared) => + withClue(s"[$ddl] ") { + val msg = rejectionOf(ddl) + msg should include(s"'$column'") + msg should include(declared) + } + } + } + + it should "be accepted over the scalar twin" in { + parse("CREATE TABLE t (n INT, c INT SCRIPT AS (n + 1))") shouldBe a[Right[_, _]] + parse("CREATE TABLE t (d DATE, cy INT SCRIPT AS (YEAR(d)))") shouldBe a[Right[_, _]] + } + + /** The ALTER half: `GatewayApi` merges the ALTER into the table it LOADS from Elasticsearch, then + * re-validates the whole table. 0.24.0 read the `_meta` type `ARRAY` as ANY there and + * accepted the column; it is now read back `ARRAY` and refused like the CREATE. + */ + it should "be refused by validateScriptReferences on a table loaded from Elasticsearch and merged with ADD COLUMN" in { + loadedArrayTable.find("arr").map(_.dataType) shouldBe Some( + app.softnetwork.elastic.sql.`type`.SQLTypes + .Array(app.softnetwork.elastic.sql.`type`.SQLTypes.Int) + ) + forAll( + Table( + ("alter", "column", "declared"), + ("ALTER TABLE t ADD COLUMN c INT SCRIPT AS (arr + 1)", "arr", "ARRAY"), + ("ALTER TABLE t ADD COLUMN cy INT SCRIPT AS (YEAR(ad))", "ad", "ARRAY") + ) + ) { (alter, column, declared) => + withClue(s"[$alter] ") { + validateScriptReferences(mergedWith(alter)) match { + case Left(msg) => + msg should include(s"'$column'") + msg should include(declared) + case Right(_) => fail("ACCEPTED, expected a refusal") + } + } + } + validateScriptReferences( + mergedWith("ALTER TABLE t ADD COLUMN c INT SCRIPT AS (n + 1)") + ) shouldBe + Right(()) + validateScriptReferences( + mergedWith("ALTER TABLE t ADD COLUMN cy INT SCRIPT AS (YEAR(d))") + ) shouldBe + Right(()) + } + + /** The table `GatewayApi` LOADS from Elasticsearch before it merges an ALTER: `_meta` declares + * the `ARRAY` columns. + */ + private lazy val loadedArrayTable: Schema = { + val mapper = new com.fasterxml.jackson.databind.ObjectMapper() + app.softnetwork.elastic.schema + .Index( + name = "t", + mappings = mapper.readTree( + """{"mappings": { + | "_meta": {"columns": { + | "arr": {"data_type": "ARRAY"}, + | "arrk": {"data_type": "ARRAY"}, + | "ad": {"data_type": "ARRAY"}, + | "n": {"data_type": "INT"}, + | "d": {"data_type": "DATE"} + | }}, + | "properties": { + | "id": {"type": "keyword"}, + | "arr": {"type": "integer"}, + | "arrk": {"type": "keyword"}, + | "ad": {"type": "date"}, + | "n": {"type": "integer"}, + | "d": {"type": "date"} + | } + |}}""".stripMargin + ), + settings = mapper.readTree("""{"settings": {}}""") + ) + .schema + } + + /** [[loadedArrayTable]] merged with each ALTER in turn, as `GatewayApi` merges one. */ + private def mergedWith(alters: String*): Schema = + alters.foldLeft(loadedArrayTable) { (table, alter) => + Parser(alter) match { + case Right(at: app.softnetwork.elastic.sql.query.AlterTable) => table.merge(at.statements) + case other => fail(s"[$alter] is not an ALTER TABLE: $other") + } + } + + /** The ingest source of the computed column `column` of `table`. */ + private def processorSource(table: Schema, column: String): String = + table.defaultPipeline.processors + .collectFirst { case p: ScriptProcessor if p.column == column => p.source } + .getOrElse(fail(s"no script processor for '$column'")) + + /** SCHEMA-6 (lead R-66) -- a computed column over an `ARRAY` column keeps 0.24.0's verdict and + * bytes: its processor reads `ctx.`, the whole list, so D7's element view is off while the + * script renders and while it is validated. The sources pinned are 0.24.0's (and 5d0c46b2's) for + * the same CREATE, RENDERED by the processor census on both trees. Read as the element, the + * first two had moved to an element cast (`Integer.parseInt(param1)`, `(long) param2`) that + * fails on `["7"]` / `[5]` and stores NULL under `ignore_failure`, and the last three were + * refused ("declared ARRAY and not INT"). + */ + private val wholeValueScripts = Table( + ("script", "source"), + ( + "c INT SCRIPT AS (CAST(arrk AS INT))", + "def param1 = ctx.arrk; ctx.c = param1" + ), + ( + "c INT SCRIPT AS (CASE WHEN n > 0 THEN arr ELSE 0 END)", + "def param1 = ctx.n; def param2 = ctx.arr; def param3 = param1 == null ? false : (param1 > 0); ctx.c = param3 ? param2 : 0" + ), + ( + "c BIGINT SCRIPT AS (CAST(arr AS BIGINT))", + "def param1 = ctx.arr; ctx.c = param1" + ), + ( + "c VARCHAR SCRIPT AS (CAST(arr AS VARCHAR))", + "def param1 = ctx.arr; ctx.c = (param1 != null ? String.valueOf(param1) : null)" + ), + ( + "c TIMESTAMP SCRIPT AS (CAST(ad AS TIMESTAMP))", + "def param1 = ctx.ad; ctx.c = param1" + ) + ) + + "a computed column over an ARRAY column" should "keep 0.24.0's verdict and processor on CREATE" in { + forAll(wholeValueScripts) { (script, source) => + val ddl = + s"CREATE TABLE t (n INT, d DATE, arr ARRAY, arrk ARRAY, ad ARRAY, $script)" + withClue(s"[$ddl] ") { + parse(ddl) match { + case Right(ct) => processorSource(ct.schema, "c") shouldBe source + case Left(msg) => fail(s"REFUSED: $msg") + } + } + } + } + + it should "keep 0.24.0's verdict and processor on an ALTER of a table loaded from Elasticsearch" in { + forAll(wholeValueScripts) { (script, source) => + withClue(s"[ADD COLUMN $script] ") { + val merged = mergedWith(s"ALTER TABLE t ADD COLUMN $script") + validateScriptReferences(merged) shouldBe Right(()) + processorSource(merged, "c") shouldBe source + } + } + } + + it should "leave every later ALTER of a table that holds it accepted" in { + forAll(wholeValueScripts) { (script, _) => + withClue(s"[holding $script, then ADD COLUMN z INT] ") { + validateScriptReferences( + mergedWith(s"ALTER TABLE t ADD COLUMN $script", "ALTER TABLE t ADD COLUMN z INT") + ) shouldBe Right(()) + } + } + } + + /** The NULLIF type rule judges the declared types too, as on 0.24.0: an `ARRAY` beside a + * number is refused, where the element view had accepted it and its processor compared the list + * with `0` and wrote the list into an INT column. + */ + it should "keep 0.24.0's NULLIF refusal" in { + val msg = rejectionOf("CREATE TABLE t (arr ARRAY, c INT SCRIPT AS (NULLIF(arr, 0)))") + msg should include("NULLIF(arr, 0) compares ARRAY with BIGINT") + } + + /** The element view is off only while such a script renders or is validated: a query over the + * same column still reads the element (D7), so the scope is restored after the validation. + */ + it should "leave the element view of a query untouched" in { + import app.softnetwork.elastic.sql.`type`.{SQLTypeUtils, SQLTypes} + val arrayOfInt = SQLTypes.Array(SQLTypes.Int) + SQLTypeUtils.validatingComputedColumns(SQLTypeUtils.operandType(arrayOfInt)) shouldBe arrayOfInt + SQLTypeUtils.operandType(arrayOfInt) shouldBe SQLTypes.Int + SQLTypeUtils.runtimeType(arrayOfInt) shouldBe SQLTypes.Int + } + /** 🔴 The half that matters more than the rejections. Over-rejecting valid DDL would be worse * than the defect, and several of these are non-obvious. */ diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/StructFieldOrderSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/StructFieldOrderSpec.scala new file mode 100644 index 000000000..14184b93c --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/StructFieldOrderSpec.scala @@ -0,0 +1,203 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.schema + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.{ObjectValue, StringValue, StringValues} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{AlterTable, CreateTable, MultiSearch} +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.ObjectNode +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +/** SCHEMA-6 (D9, AC5) -- object sub-fields keep their DECLARED order. + * + * MEASURED (SCHEMA-2 T0-4, ES 8): Elasticsearch returns a mapping's `properties` SORTED by name + * and a `_meta` object in no reliable order, so `CREATE TABLE so_t (obj STRUCT FIELDS (y INT, b + * KEYWORD))` read back `b, y`. Only a LIST survives: `_meta.columns..field_order`. A table that + * has none (a raw mapping, or one created by 0.24.0) keeps its mapping's order, and an ALTER on it + * is `Safe`. + */ +class StructFieldOrderSpec extends AnyFlatSpec with Matchers { + + private val mapper = new ObjectMapper() + + private val ddl = + "CREATE TABLE so_t (id INT, obj STRUCT FIELDS (y INT, b KEYWORD), " + + "items ARRAY FIELDS (q INT, p KEYWORD))" + + private def declared(sql: String = ddl): Table = + Parser(sql) match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + + /** `node` as Elasticsearch returns a mapping: every `properties` object sorted by name. */ + private def sortedProperties(node: JsonNode): JsonNode = + node match { + case obj: ObjectNode => + val copy = mapper.createObjectNode() + obj.properties().asScala.foreach { e => + val value = e.getValue match { + case props: ObjectNode if e.getKey == "properties" => + val sorted = mapper.createObjectNode() + props.properties().asScala.toSeq.sortBy(_.getKey).foreach { p => + sorted.set[JsonNode](p.getKey, sortedProperties(p.getValue)) + } + sorted + case other => sortedProperties(other) + } + copy.set[JsonNode](e.getKey, value) + } + copy + case other => other + } + + /** The table core reads back for `table`'s index: mapping `properties` sorted, as Elasticsearch + * returns them; `withMeta` false drops the `_meta` (a raw mapping). + */ + private def readBack(table: Table, dropFieldOrder: Boolean = false): Table = { + val mapping = sortedProperties(table.indexMappings).asInstanceOf[ObjectNode] + if (dropFieldOrder) { + val columns = mapping.path("_meta").path("columns") + columns.properties().asScala.foreach { e => + e.getValue match { + case c: ObjectNode => c.remove("field_order") + case _ => + } + } + } + val mappings = mapper.createObjectNode() + mappings.set[JsonNode]("mappings", mapping) + val settings = mapper.createObjectNode() + settings.set[JsonNode]("settings", table.indexSettings) + Index(name = table.name, mappings = mappings, settings = settings).schema + } + + private def names(table: Table, column: String): List[String] = + table.find(column).map(_.multiFields.map(_.name)).getOrElse(fail(s"no column $column")) + + behavior of "a struct's declared field order" + + it should "be written to _meta.field_order, a list" in { + val t = declared() + t.find("obj").get._meta.get("field_order") shouldBe Some( + StringValues(Seq(StringValue("y"), StringValue("b"))) + ) + t.find("items").get._meta.get("field_order") shouldBe Some( + StringValues(Seq(StringValue("q"), StringValue("p"))) + ) + t.find("id").get._meta.get("field_order") shouldBe None + } + + it should "be read back in the declared order, although the mapping is sorted" in { + val read = readBack(declared()) + names(read, "obj") shouldBe List("y", "b") + names(read, "items") shouldBe List("q", "p") + // DESCRIBE TABLE rows, SHOW CREATE TABLE + // (the top-level columns of a table read back are in name order, as they always were) + read.describe + .map(_("Field").toString) + .filter(f => f.startsWith("obj") || f.startsWith("items")) shouldBe + Seq("items", "items.q", "items.p", "obj", "obj.y", "obj.b") + val rendered = read.sql + rendered.indexOf("y INT") should be < rendered.indexOf("b KEYWORD") + rendered.indexOf("q INT") should be < rendered.indexOf("p KEYWORD") + read.diff(declared()).columns shouldBe empty + read.diff(declared()).mappings shouldBe empty + } + + it should "type a STRUCT position's fields in that order (ResultSchema, columnTypes)" in { + val read = readBack(declared()) + MultiSearch.columnType(read.find("obj").get).fields.map(_.name) shouldBe Seq("y", "b") + MultiSearch.columnType(read.find("items").get).element.get.fields.map(_.name) shouldBe + Seq("q", "p") + } + + it should "keep the mapping's order without field_order (raw mapping, 0.24.0 table)" in { + val legacy = readBack(declared(), dropFieldOrder = true) + names(legacy, "obj") shouldBe List("b", "y") + names(legacy, "items") shouldBe List("p", "q") + MultiSearch.columnType(legacy.find("obj").get).fields.map(_.name) shouldBe Seq("b", "y") + // and an ALTER on it is Safe: its field_order is written as its order stands, no diff + legacy.diff(legacy.merge(Nil)).mappings shouldBe empty + val altered = + Parser( + "ALTER TABLE so_t ALTER COLUMN obj SET FIELD b KEYWORD OPTIONS (ignore_above = 10)" + ) match { + case Right(at: AlterTable) => legacy.merge(at.statements) + case other => fail(s"$other") + } + val diff = legacy.diff(altered) + withClue(s"columns=${diff.columns} mappings=${diff.mappings}: ")(diff.safety shouldBe Safe) + names(altered, "obj") shouldBe List("b", "y") + } + + it should "append a mapped field field_order does not list, and ignore a listed absent one" in { + val t = declared() + val mapping = sortedProperties(t.indexMappings).asInstanceOf[ObjectNode] + mapping + .path("properties") + .path("obj") + .path("properties") + .asInstanceOf[ObjectNode] + .set[JsonNode]("a", mapper.readTree("""{"type": "integer"}""")) + mapping + .path("_meta") + .path("columns") + .path("obj") + .asInstanceOf[ObjectNode] + .set[JsonNode]("field_order", mapper.readTree("""["y", "gone", "b"]""")) + val mappings = mapper.createObjectNode() + mappings.set[JsonNode]("mappings", mapping) + val read = Index( + name = "so_t", + mappings = mappings, + settings = mapper.readTree("""{"settings": {}}""") + ).schema + names(read, "obj") shouldBe List("y", "b", "a") + } + + it should "keep a rewritten sub-field in its position (SET FIELD), and append a new one" in { + val t = declared() + val rewritten = Parser("ALTER TABLE so_t ALTER COLUMN obj SET FIELD y BIGINT") match { + case Right(at: AlterTable) => t.merge(at.statements) + case other => fail(s"$other") + } + names(rewritten, "obj") shouldBe List("y", "b") + val added = Parser("ALTER TABLE so_t ALTER COLUMN obj ADD FIELD c KEYWORD") match { + case Right(at: AlterTable) => t.merge(at.statements) + case other => fail(s"$other") + } + names(added, "obj") shouldBe List("y", "b", "c") + // the new order is a Safe key of its own (the new field's `_meta` keeps today's safety) + val diff = t.diff(added) + diff.mappings.collect { case m: MappingSet => m.key } should contain( + "_meta.columns.obj.field_order" + ) + diff.mappings.collect { + case m: MappingSet if m.key.endsWith("field_order") => m.safety + } shouldBe + List(Safe) + ObjectValue(added.find("obj").get._meta).value("field_order") shouldBe StringValues( + Seq(StringValue("y"), StringValue("b"), StringValue("c")) + ) + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/type/ArrayTypeMappingSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/type/ArrayTypeMappingSpec.scala new file mode 100644 index 000000000..5fa80f1f5 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/type/ArrayTypeMappingSpec.scala @@ -0,0 +1,204 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.`type` + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{AlterTable, CreateTable, SingleSearch} +import app.softnetwork.elastic.sql.schema.{nestedArrayRefusal, Column, Table} +import com.fasterxml.jackson.databind.ObjectMapper +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** SCHEMA-6 (D1, D2, D3, AC0) -- a column declared `ARRAY` reads back as `ARRAY`, for every X + * the DDL accepts: ONE read function (`SQLTypes.apply(String)`), ONE write function + * (`SQLTypeUtils.elasticType`), and the DDL grammar accepting `ARRAY` of every column type. + * + * 0.24.0 read every `array<…>` name but `array` as ANY (`case _ => Any`): `DESCRIBE` + * answered ANY, the type checks saw ANY, and the rows were as stored. A list of lists is refused + * by every DDL statement that declares a type (Elasticsearch flattens nested arrays). + */ +class ArrayTypeMappingSpec extends AnyFlatSpec with Matchers { + + /** P1: every X the DDL accepts, with the Elasticsearch type its list maps to. */ + private val p1: Seq[(SQLType, String)] = Seq( + SQLTypes.TinyInt -> "byte", + SQLTypes.SmallInt -> "short", + SQLTypes.Int -> "integer", + SQLTypes.BigInt -> "long", + SQLTypes.Real -> "float", + SQLTypes.Double -> "double", + SQLTypes.Boolean -> "boolean", + SQLTypes.Keyword -> "keyword", + SQLTypes.Text -> "text", + SQLTypes.Varchar -> "text", + SQLTypes.Char -> "text", + SQLTypes.Date -> "date", + SQLTypes.Time -> "date", + SQLTypes.DateTime -> "date", + SQLTypes.Timestamp -> "date", + SQLTypes.VarBinary -> "binary", + SQLTypes.GeoPoint -> "geo_point", + SQLTypes.Struct -> "nested" + ) + + behavior of "SQLTypes.apply / SQLTypeUtils.elasticType over ARRAY (D1, D2)" + + it should "read every ARRAY type name back as ARRAY, whatever its case" in { + p1.size shouldBe 18 + for ((x, _) <- p1) { + val declared = SQLTypes.Array(x) + withClue(s"${declared.typeId}: ") { + SQLTypes(declared.typeId) shouldBe declared + SQLTypes(declared.typeId.toLowerCase) shouldBe declared + } + } + } + + it should "map ARRAY to the Elasticsearch type of X (VARBINARY binary, STRUCT nested)" in { + for ((x, es) <- p1) { + withClue(s"ARRAY<${x.typeId}>: ") { + SQLTypeUtils.elasticType(SQLTypes.Array(x)) shouldBe es + if (x != SQLTypes.Struct) SQLTypeUtils.elasticType(x) shouldBe es + } + } + } + + it should "read CHAR back as CHAR (it read ANY: no arm)" in { + SQLTypes("CHAR") shouldBe SQLTypes.Char + SQLTypes("char") shouldBe SQLTypes.Char + } + + it should "read an unknown element, or a list of lists, as ANY (R-7)" in { + SQLTypes("ARRAY") shouldBe SQLTypes.Any + SQLTypes("ARRAY") shouldBe SQLTypes.Any + // a 0.24.0 `_meta` could hold one: Elasticsearch has no list of lists + SQLTypes("ARRAY>") shouldBe SQLTypes.Any + SQLTypes("nested") shouldBe SQLTypes.Array(SQLTypes.Struct) + SQLTypes("array") shouldBe SQLTypes.Array(SQLTypes.Struct) + } + + it should "read a declared ARRAY back through the mapping and its _meta" in { + val mapper = new ObjectMapper() + for ((x, es) <- p1.filterNot(_._1 == SQLTypes.Struct)) { + val declared = Parser(s"CREATE TABLE t (id INT, c ARRAY<${x.typeId}>)") match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"ARRAY<${x.typeId}>: $other") + } + val mappings = mapper.createObjectNode() + mappings.set("mappings", declared.indexMappings) + val settings = mapper.createObjectNode() + settings.set("settings", declared.indexSettings) + val read = Index(name = "t", mappings = mappings, settings = settings).schema + withClue(s"ARRAY<${x.typeId}>: ") { + declared.indexMappings.path("properties").path("c").path("type").asText() shouldBe es + read.find("c").map(_.dataType) shouldBe Some(SQLTypes.Array(x)) + read.diff(declared).columns shouldBe empty + read.diff(declared).mappings shouldBe empty + } + } + } + + behavior of "the DDL grammar over ARRAY (D3)" + + it should "parse and re-render CREATE TABLE with ARRAY of every X, ARRAY included" in { + for ((x, _) <- p1) { + val fields = if (x == SQLTypes.Struct) " FIELDS (q INT, p KEYWORD)" else "" + val sql = s"CREATE TABLE t (c ARRAY<${x.typeId}>$fields)" + Parser(sql) match { + case Right(ct: CreateTable) => + withClue(s"$sql: ") { + ct.schema.find("c").map(_.dataType) shouldBe Some(SQLTypes.Array(x)) + Parser(ct.sql) shouldBe Right(ct) + // SHOW CREATE TABLE renders the table's own DDL, which re-parses to the same table + Parser(ct.schema.sql) match { + case Right(again: CreateTable) => + again.schema.find("c").map(_.dataType) shouldBe Some(SQLTypes.Array(x)) + case other => fail(s"SHOW CREATE TABLE of $sql: $other") + } + } + case other => fail(s"$sql: $other") + } + } + } + + it should "parse SET DATA TYPE ARRAY (the remedy of a several-point geo field)" in { + val sql = "ALTER TABLE t ALTER COLUMN geo SET DATA TYPE ARRAY" + Parser(sql) match { + case Right(at: AlterTable) => + Parser(at.sql) shouldBe Right(at) + at.sql should include("ARRAY") + case other => fail(s"$sql: $other") + } + } + + it should "keep a CAST to ARRAY as it was (sql_type's array_type is untouched)" in { + Parser("SELECT CAST(n AS ARRAY) AS a FROM t").isRight shouldBe + Parser("SELECT CAST(n AS INT) AS a FROM t").isRight + } + + behavior of "a list of lists (D2)" + + private val refusal = + "ARRAY> is not supported: Elasticsearch flattens nested arrays; declare ARRAY" + + it should "be refused at VALIDATE by CREATE TABLE, naming the column" in { + Parser("CREATE TABLE t (id INT, c ARRAY>)") match { + case Left(error) => error.msg should include(s"c: $refusal") + case other => fail(s"accepted: $other") + } + Parser( + "CREATE TABLE t (id INT, obj STRUCT FIELDS (y INT, z ARRAY>))" + ) match { + case Left(error) => error.msg should include(s"obj.z: $refusal") + case other => fail(s"accepted: $other") + } + } + + it should "be refused at VALIDATE by ADD COLUMN, SET DATA TYPE and SET FIELD, naming the column" in { + Seq( + "ALTER TABLE t ADD COLUMN c ARRAY>" -> s"c: $refusal", + "ALTER TABLE t ALTER COLUMN c SET DATA TYPE ARRAY>" -> s"c: $refusal", + "ALTER TABLE t ALTER COLUMN obj SET FIELD z ARRAY>" -> s"obj.z: $refusal", + "ALTER TABLE t ALTER COLUMN obj SET FIELDS (z ARRAY>)" -> s"obj.z: $refusal" + ).foreach { case (sql, expected) => + Parser(sql) match { + case Left(error) => withClue(s"$sql: ")(error.msg should include(expected)) + case other => fail(s"$sql accepted: $other") + } + } + } + + it should "be refused for a CTAS column (ARRAY_AGG of a list column), by the same check" in { + val source = Table( + "src", + List(Column("g", SQLTypes.Keyword), Column("arr", SQLTypes.Array(SQLTypes.Int))) + ) + val search = Parser("SELECT g, ARRAY_AGG(arr) AS a FROM src GROUP BY g") match { + case Right(s: SingleSearch) => s + case other => fail(s"$other") + } + val merged = source.mergeWithSearch(search) + merged.columns.flatMap(nestedArrayRefusal) shouldBe Seq(s"a: $refusal") + // ...and a list of a scalar is accepted + val scalar = Parser("SELECT g, ARRAY_AGG(g) AS a FROM src GROUP BY g") match { + case Right(s: SingleSearch) => s + case other => fail(s"$other") + } + source.mergeWithSearch(scalar).columns.flatMap(nestedArrayRefusal) shouldBe empty + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ArrayRoundTripSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ArrayRoundTripSpec.scala new file mode 100644 index 000000000..592718cac --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ArrayRoundTripSpec.scala @@ -0,0 +1,559 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + DdlResult, + ElasticFailure, + ElasticResult, + ElasticSuccess, + QueryResult, + QueryRows, + QueryStream +} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{CreateTable, MultiSearch, SingleSearch} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} + +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.util.Try + +/** SCHEMA-6 on a real cluster (AC0, AC1, AC5, AC6): + * + * - "round trip": a column declared `ARRAY`, for every X the DDL accepts, reads back as + * `ARRAY` (DESCRIBE TABLE), maps to X's Elasticsearch type, re-parses from SHOW CREATE + * TABLE, and reads its values as a `List` of the class its scalar twin reads; + * - "declaration": on `zs_ext`, created by a raw mapping, several values in a column fail the + * statement naming the ALTER that declares it a list; that ALTER runs IN PLACE (index uuid and + * mapping unchanged, `_meta` declared), and the column then reads its list; + * - "derived": a derived-table / CTE column typed from its body by `attachDerived` (Case 14, + * INT), core-alone execution still refused; + * - "struct order": sub-fields in their declared order (DESCRIBE TABLE, SHOW CREATE TABLE, the + * result schema), a raw mapping's in its order, an ALTER on it in place; + * - "array_agg": ARRAY_AGG answers `ARRAY`, every cell a list. + */ +trait ArrayRoundTripSpec extends GatewayIntegrationTestKit { + self: ElasticTestKit => + + private val mapper = new ObjectMapper() + + private lazy val api: ElasticClientApi = client.asInstanceOf[ElasticClientApi] + + private def run(sql: String): ElasticResult[QueryResult] = + Await.result(client.run(sql), 60.seconds) + + private def runOk(sql: String): QueryResult = + run(sql) match { + case ElasticSuccess(result) => result + case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") + } + + private def rows(sql: String): Seq[Map[String, Any]] = collectRows(System.nanoTime(), run(sql)) + + private def failureOf(sql: String): String = + run(sql) match { + case ElasticFailure(error) => error.message + case ElasticSuccess(result) => + Try(collectRows(System.nanoTime(), ElasticSuccess(result))).failed.toOption + .map(messageOf) + .getOrElse(fail(s"[$sql] answered $result instead of failing")) + } + + /** The failure in core's own words: a streamed result's failure reaches the test wrapped by + * `futureValue` ("The future returned an exception of type: …, with message: …."). + */ + private def messageOf(e: Throwable): String = e match { + case wrapped: org.scalatest.exceptions.TestFailedException if wrapped.getCause != null => + messageOf(wrapped.getCause) + case other => Option(other.getMessage).getOrElse(other.toString) + } + + private def drop(index: String): Unit = { + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $index"), 60.seconds)) + Try(api.deleteIndex(index)) + } + + private def mappingOf(index: String): JsonNode = + api.getMapping(index) match { + case ElasticSuccess(json) => + val root = mapper.readTree(json) + // `{index: {mappings: …}}`, `{mappings: …}` or the mapping itself, per client + Option(root.get(index)) + .map(_.path("mappings")) + .filterNot(_.isMissingNode) + .orElse(Option(root.get("mappings"))) + .getOrElse(root) + case other => fail(s"getMapping($index): $other") + } + + private def uuidOf(index: String): String = + api.getIndex(index) match { + case ElasticSuccess(Some(i)) => + val settings = i.settings + Seq( + settings.path("settings").path("index").path("uuid"), + settings.path("index").path("uuid"), + settings.path(index).path("settings").path("index").path("uuid") + ).find(n => !n.isMissingNode && n.asText().nonEmpty) + .map(_.asText()) + .getOrElse( + fail(s"no uuid in the settings of $index: $settings") + ) + case other => fail(s"getIndex($index): $other") + } + + private def load(index: String, mapping: String, docs: Seq[(String, String)]): Unit = { + api + .createIndex(index, settings = """{"number_of_shards": 1, "number_of_replicas": 0}""") + .get shouldBe true + api.setMapping(index, mapping).get shouldBe true + docs.foreach { case (id, source) => + api.index(index, id, source) match { + case ElasticSuccess(true) => + case other => fail(s"indexing $id into $index: $other") + } + } + api.refresh(index) + } + + private def byId(rs: Seq[Map[String, Any]]): Map[String, Map[String, Any]] = + rs.map(r => String.valueOf(r.getOrElse("id", null)) -> r).toMap + + private def cls(v: Any): String = if (v == null) "null" else v.getClass.getName + + // ----------------------------------------------------------------------------------------------- + // AC0 -- round trip of every X + // ----------------------------------------------------------------------------------------------- + + /** X, its Elasticsearch type, two values as JSON. */ + private val population: Seq[(SQLType, String, String, String)] = Seq( + (SQLTypes.TinyInt, "byte", "1", "2"), + (SQLTypes.SmallInt, "short", "1", "2"), + (SQLTypes.Int, "integer", "1", "2"), + (SQLTypes.BigInt, "long", "1", "2"), + (SQLTypes.Real, "float", "1.5", "2.5"), + (SQLTypes.Double, "double", "1.5", "2.5"), + (SQLTypes.Boolean, "boolean", "true", "false"), + (SQLTypes.Keyword, "keyword", "\"a\"", "\"b\""), + (SQLTypes.Text, "text", "\"a\"", "\"b\""), + (SQLTypes.Varchar, "text", "\"a\"", "\"b\""), + (SQLTypes.Char, "text", "\"a\"", "\"b\""), + (SQLTypes.Date, "date", "\"2024-01-01\"", "\"2024-01-02\""), + (SQLTypes.Time, "date", "37800000", "41400000"), + (SQLTypes.DateTime, "date", "\"2024-01-01T10:00:00\"", "\"2024-01-02T11:00:00\""), + (SQLTypes.Timestamp, "date", "\"2024-01-01T10:00:00Z\"", "\"2024-01-02T11:00:00Z\""), + (SQLTypes.VarBinary, "binary", "\"aGVsbG8=\"", "\"d29ybGQ=\""), + (SQLTypes.GeoPoint, "geo_point", "[2.0, 1.0]", "[4.0, 3.0]"), + (SQLTypes.Struct, "nested", """{"q": 1, "p": "a"}""", """{"q": 2, "p": "b"}""") + ) + + private def tableOf(x: SQLType): String = s"ar_${x.typeId.toLowerCase}" + + behavior of "an ARRAY column (AC0)" + + it should "read back as ARRAY: DESCRIBE TABLE, mapping, SHOW CREATE TABLE, rows" in { + population.foreach { case (x, es, v1, v2) => + val table = tableOf(x) + drop(table) + val struct = x == SQLTypes.Struct + val fields = if (struct) " FIELDS (q INT, p KEYWORD)" else "" + val twin = if (struct) "" else s", s ${x.typeId}" + runOk(s"CREATE TABLE $table (id KEYWORD, c ARRAY<${x.typeId}>$fields$twin, PRIMARY KEY (id))") + withClue(s"ARRAY<${x.typeId}>: ") { + // DESCRIBE TABLE + val describe = rows(s"DESCRIBE TABLE $table") + describe.find(_("Field") == "c").map(_("Type")) shouldBe Some(s"ARRAY<${x.typeId}>") + // the mapping + mappingOf(table).path("properties").path("c").path("type").asText() shouldBe es + // SHOW CREATE TABLE re-parses to the same column + val ddl = assertShowCreate(System.nanoTime(), run(s"SHOW CREATE TABLE $table")) + Parser(ddl) match { + case Right(ct: CreateTable) => + ct.schema.find("c").map(_.dataType) shouldBe Some(SQLTypes.Array(x)) + case other => fail(s"SHOW CREATE TABLE $table does not re-parse: $other\n$ddl") + } + // rows: [v1, v2], [v1], v1, null, absent + val twinOf = if (struct) "" else s""", "s": $v1""" + Seq( + "r1" -> s"""{"id": "r1", "c": [$v1, $v2]$twinOf}""", + "r2" -> s"""{"id": "r2", "c": [$v1]$twinOf}""", + "r3" -> s"""{"id": "r3", "c": $v1$twinOf}""", + "r4" -> s"""{"id": "r4", "c": null}""", + "r5" -> s"""{"id": "r5"}""" + ).foreach { case (id, source) => api.index(table, id, source).get shouldBe true } + api.refresh(table) + val read = byId(rows(s"SELECT id, c${if (struct) "" else ", s"} FROM $table")) + def list(id: String): Seq[Any] = read(id)("c") match { + case l: Seq[_] => l + case other => fail(s"$id: c = $other (${cls(other)}), not a List") + } + list("r1").size shouldBe 2 + list("r2").size shouldBe 1 + list("r3").size shouldBe 1 + read("r4")("c") shouldBe (null: Any) + read("r5").getOrElse("c", null) shouldBe (null: Any) + // every element in the class its scalar twin reads + if (!struct) { + val twinClass = cls(read("r1")("s")) + (list("r1") ++ list("r2") ++ list("r3")).foreach(e => cls(e) shouldBe twinClass) + list("r1").head shouldBe read("r1")("s") + } else + list("r1").foreach(e => e shouldBe a[scala.collection.Map[_, _]]) + } + drop(table) + } + } + + it should "refuse a list of lists in every DDL statement, naming the column" in { + val refusal = "ARRAY> is not supported" + Seq( + "CREATE TABLE ar_nested (id KEYWORD, c ARRAY>)", + "ALTER TABLE ar_any ADD COLUMN c ARRAY>", + "ALTER TABLE ar_any ALTER COLUMN c SET DATA TYPE ARRAY>", + "ALTER TABLE ar_any ALTER COLUMN obj SET FIELD z ARRAY>" + ).foreach { sql => + run(sql) match { + case ElasticFailure(error) => withClue(s"$sql: ")(error.message should include(refusal)) + case other => fail(s"$sql answered $other") + } + } + } + + // ----------------------------------------------------------------------------------------------- + // AC0 -- zs_ext: Q1's failure, its meta-only remedy, the list after + // ----------------------------------------------------------------------------------------------- + + private val ext = "zs_ext" + + private val extMapping = + """{"properties": { + | "id": {"type": "keyword"}, "tags": {"type": "keyword"}, "n": {"type": "integer"}, + | "s": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}}, + | "ts": {"type": "date"}, "geo": {"type": "geo_point"}, + | "obj": {"properties": {"a": {"type": "integer"}}} + |}}""".stripMargin + + private val extDocs = Seq( + "e1" -> """{"id": "e1", "tags": ["a", "b"], "n": [1, 2], "s": ["x", "y"], + | "ts": ["2024-01-01T10:00:00Z", "2024-01-02T10:00:00Z"], + | "geo": [[2.0, 1.0], [4.0, 3.0]], "obj": [{"a": 5}, {"a": 50}]}""".stripMargin, + "e2" -> """{"id": "e2", "tags": "c", "n": 3, "s": "z", "ts": "2024-01-03T10:00:00Z", + | "geo": [6.0, 5.0], "obj": {"a": 7}}""".stripMargin, + "e3" -> """{"id": "e3"}""" + ) + + behavior of "a field holding several values, not declared a list (AC0, Q1)" + + it should "fail naming the ALTER, which runs in place, then read the list" in { + drop(ext) + load(ext, extMapping, extDocs) + // the STRUCT column holding a list of objects answers it (A-26) + byId(rows(s"SELECT id, obj FROM $ext"))("e1")("obj") shouldBe List( + ListMap("a" -> 5), + ListMap("a" -> 50) + ) + Seq( + ("tags", "KEYWORD", "keyword", List("a", "b")), + ("n", "INT", "integer", List(1, 2)), + ("s", "TEXT", "text", List("x", "y")), + ("ts", "TIMESTAMP", "date", null), + ("geo", "GEO_POINT", "geo_point", null) + ).foreach { case (c, x, es, expected) => + withClue(s"$c: ") { + val message = failureOf(s"SELECT id, $c FROM $ext") + message should include(s"Column $c holds several values in document e1") + message should include(s"index $ext") + val remedy = message.substring(message.indexOf("ALTER TABLE")) + remedy shouldBe s"ALTER TABLE $ext ALTER COLUMN $c SET DATA TYPE ARRAY<$x>" + val uuid = uuidOf(ext) + runOk(remedy) shouldBe DdlResult(true) + uuidOf(ext) shouldBe uuid // in place: no reindex + val mapping = mappingOf(ext) + mapping.path("properties").path(c).path("type").asText() shouldBe es + mapping.path("_meta").path("columns").path(c).path("data_type").asText() shouldBe + s"ARRAY<$x>" + rows(s"DESCRIBE TABLE $ext").find(_("Field") == c).map(_("Type")) shouldBe + Some(s"ARRAY<$x>") + val read = byId(rows(s"SELECT id, $c FROM $ext")) + read("e1")(c) shouldBe a[Seq[_]] + read("e1")(c).asInstanceOf[Seq[_]].size shouldBe 2 + if (expected != null) read("e1")(c) shouldBe expected + read("e2")(c).asInstanceOf[Seq[_]].size shouldBe 1 + read("e3").getOrElse(c, null) shouldBe (null: Any) + } + } + // a scalar sub-field under the list of objects: SET FIELD + val message = failureOf(s"SELECT id, obj.a FROM $ext") + message should include("Column obj.a holds several values in document e1") + val remedy = message.substring(message.indexOf("ALTER TABLE")) + remedy shouldBe s"ALTER TABLE $ext ALTER COLUMN obj SET FIELD a ARRAY" + val uuid = uuidOf(ext) + runOk(remedy) shouldBe DdlResult(true) + uuidOf(ext) shouldBe uuid + val paths = byId(rows(s"SELECT id, obj.a FROM $ext")) + paths("e1")("obj.a") shouldBe List(5, 50) + paths("e2")("obj.a") shouldBe List(7) + byId(rows(s"SELECT id, obj FROM $ext"))("e1")("obj") shouldBe List( + ListMap("a" -> List(5)), + ListMap("a" -> List(50)) + ) + drop(ext) + } + + /** Under an index PATTERN the failure names the index the hit came from (`_index`), never the + * pattern: `ALTER TABLE "zs_ext_*" …` is refused by Elasticsearch (`Invalid index`), so naming + * the pattern shipped a remedy that cannot run on the commonest time-series shape. Each client + * must carry the hit's `_index` into the rows it parses (the ES 8 / 9 Java clients build their + * own hit envelope), on a plain search and on a sorted one alike. + */ + it should "name the hit's own index under an index pattern, and that ALTER runs in place" in { + val concrete = s"${ext}_p1" + val pattern = s"${ext}_p*" + drop(concrete) + load(concrete, extMapping, extDocs) + Seq(s"SELECT id, tags FROM $pattern", s"SELECT id, tags FROM $pattern ORDER BY id").foreach { + sql => + withClue(s"[$sql] ") { + val message = failureOf(sql) + message should include( + s"Column tags holds several values in document e1 of index $concrete" + ) + message.substring(message.indexOf("ALTER TABLE")) shouldBe + s"ALTER TABLE $concrete ALTER COLUMN tags SET DATA TYPE ARRAY" + } + } + val remedy = s"ALTER TABLE $concrete ALTER COLUMN tags SET DATA TYPE ARRAY" + val uuid = uuidOf(concrete) + runOk(remedy) shouldBe DdlResult(true) + uuidOf(concrete) shouldBe uuid // in place: no reindex + val read = byId(rows(s"SELECT id, tags FROM $pattern")) + read("e1")("tags") shouldBe List("a", "b") + read("e2")("tags") shouldBe List("c") + drop(concrete) + } + + // ----------------------------------------------------------------------------------------------- + // AC1 -- a derived / CTE column typed from its body + // ----------------------------------------------------------------------------------------------- + + private val zs = "zs_t" + + behavior of "a derived-table or CTE column (AC1)" + + it should "be typed INT from its body through attachDerived, while core alone still refuses it" in { + drop(zs) + runOk(s"CREATE TABLE $zs (id KEYWORD, n INT, d DATE, PRIMARY KEY (id))") + Seq( + s"SELECT q.c1 FROM (SELECT n AS c1 FROM $zs) q", + s"SELECT c1 FROM (SELECT n AS c1 FROM $zs) q", + s"SELECT q2.c1 FROM (SELECT q.c1 FROM (SELECT n AS c1 FROM $zs) q) q2", + s"WITH q AS (SELECT n AS c1 FROM $zs) SELECT q.c1 FROM q" + ).foreach { sql => + withClue(s"[$sql] ") { + val single = Parser(sql) match { + case Right(s: SingleSearch) => s + case other => fail(s"$other") + } + api.attachDerived(single) match { + case ElasticSuccess(attached) => + MultiSearch.projectionTypes(attached).map(_.map(_._2.sqlType)) shouldBe Some( + Seq(SQLTypes.Int) + ) + case other => fail(s"attachDerived: $other") + } + // core alone keeps 0.24.0's closure refusal (D11) + run(sql) match { + case ElasticFailure(_) => + case other => fail(s"core ran a derived statement: $other") + } + } + } + val star = Parser(s"SELECT q.* FROM (SELECT n AS c1 FROM $zs) q") match { + case Right(s: SingleSearch) => s + case other => fail(s"$other") + } + api.attachDerived(star) match { + case ElasticSuccess(attached) => + attached.schema.flatMap(_.find("c1")).map(_.dataType) shouldBe Some(SQLTypes.Int) + case other => fail(s"attachDerived: $other") + } + // Case 15: a DATE beside the derived INT is #420 rule 1's refusal + Parser(s"SELECT d AS k FROM $zs UNION ALL SELECT q.c1 FROM (SELECT n AS c1 FROM $zs) q") match { + case Right(m: MultiSearch) => + api.attachDerived(m) match { + case ElasticSuccess(attached) => + MultiSearch.branchTypes(attached.requests, attached.resolvedOperators) shouldBe Left( + "Set operation branches must project compatible types at column 1: branch 1 'k' " + + "is DATE, branch 2 'c1' is INT" + ) + case other => fail(s"attachDerived: $other") + } + case other => fail(s"$other") + } + drop(zs) + } + + // ----------------------------------------------------------------------------------------------- + // AC5 -- struct order + // ----------------------------------------------------------------------------------------------- + + private val so = "so_t" + private val soRaw = "so_raw" + + behavior of "a struct's sub-fields (AC5)" + + it should "keep their declared order: DESCRIBE TABLE, SHOW CREATE TABLE, the result schema" in { + drop(so) + runOk( + s"CREATE TABLE $so (id KEYWORD, obj STRUCT FIELDS (y INT, b KEYWORD), " + + "items ARRAY FIELDS (q INT, p KEYWORD), PRIMARY KEY (id))" + ) + api.index( + so, + "s1", + """{"id": "s1", "obj": {"b": "x", "y": 1}, "items": [{"p": "a", "q": 2}]}""" + ) + api.refresh(so) + // each struct's sub-fields in their declared order (D9); the columns themselves keep the + // mapping's order, which D9 does not touch + val described = rows(s"DESCRIBE TABLE $so").map(_("Field").toString) + described.filter(_.startsWith("obj.")) shouldBe Seq("obj.y", "obj.b") + described.filter(_.startsWith("items.")) shouldBe Seq("items.q", "items.p") + val ddl = assertShowCreate(System.nanoTime(), run(s"SHOW CREATE TABLE $so")) + ddl.indexOf("y INT") should be < ddl.indexOf("b KEYWORD") + ddl.indexOf("q INT") should be < ddl.indexOf("p KEYWORD") + // a statement with no LIMIT streams its rows: its schema is the stream's + val result = runOk(s"SELECT obj, items FROM $so") + val schema = result match { + case q: QueryRows => q.schema + case q: QueryStream => q.schema + case other => fail(s"$other") + } + val columns = schema.getOrElse(fail("no schema")).columns + columns.head.fields.map(_.name) shouldBe Seq("y", "b") + columns(1).element.map(_.fields.map(_.name)) shouldBe Some(Seq("q", "p")) + // the rows keep `_source`'s key order (A-14) + collectRows(System.nanoTime(), ElasticSuccess(result)) + .head("obj") + .asInstanceOf[scala.collection.Map[String, Any]] + .keys + .toSeq shouldBe Seq("b", "y") + drop(so) + } + + it should "keep a raw mapping's order without field_order, and ALTER it in place" in { + drop(soRaw) + load( + soRaw, + """{"properties": {"id": {"type": "keyword"}, + | "obj": {"properties": {"y": {"type": "integer"}, "b": {"type": "keyword"}}}}}""".stripMargin, + Seq("r1" -> """{"id": "r1", "obj": {"y": 1, "b": "x"}}""") + ) + rows(s"DESCRIBE TABLE $soRaw") + .map(_("Field").toString) + .filter(_.startsWith("obj.")) shouldBe Seq("obj.b", "obj.y") + val uuid = uuidOf(soRaw) + runOk( + s"ALTER TABLE $soRaw ALTER COLUMN obj SET FIELD b KEYWORD OPTIONS (ignore_above = 10)" + ) shouldBe + DdlResult(true) + uuidOf(soRaw) shouldBe uuid + drop(soRaw) + } + + // ----------------------------------------------------------------------------------------------- + // AC6 -- ARRAY_AGG declared ARRAY + // ----------------------------------------------------------------------------------------------- + + private val agg = "ar_agg" + + behavior of "ARRAY_AGG (AC6)" + + it should "answer ARRAY, every cell a list, a one-element group too" in { + drop(agg) + load( + agg, + """{"properties": {"id": {"type": "keyword"}, "g": {"type": "keyword"}, + | "k": {"type": "keyword"}}}""".stripMargin, + Seq( + "a1" -> """{"id": "a1", "g": "x", "k": "a"}""", + "a2" -> """{"id": "a2", "g": "x", "k": "b"}""", + "a3" -> """{"id": "a3", "g": "y", "k": "c"}""" + ) + ) + run(s"SELECT g, ARRAY_AGG(k) AS a FROM $agg GROUP BY g") match { + case ElasticSuccess(result) => + val rs = collectRows(System.nanoTime(), ElasticSuccess(result)) + rs.foreach(r => r("a") shouldBe a[Seq[_]]) + rs.find(_("g") == "y").map(_("a")) shouldBe Some(List("c")) + case other => fail(s"$other") + } + drop(agg) + } + + private val perElement = "ar_elem" + + behavior of "an ARRAY column read per element beside its scalar twin (D7)" + + // a quoted number beside a whole column is the whole number it rounds to, sent to Elasticsearch + // as that number, and ROUND takes its operand's type: an `ARRAY` column is matched and + // rounded per element, so it answers what its INT twin `n` (each document's first element) + // answers + it should "match a quoted number and round to tens as its twin does" in { + drop(perElement) + runOk(s"CREATE TABLE $perElement (id KEYWORD, arr ARRAY, n INT, PRIMARY KEY (id))") + Seq( + "e1" -> """{"id": "e1", "arr": [2, 9], "n": 2}""", + "e2" -> """{"id": "e2", "arr": [1], "n": 1}""", + "e3" -> """{"id": "e3", "arr": 3, "n": 3}""", + "e4" -> """{"id": "e4", "arr": [14, 20], "n": 14}""" + ).foreach { case (id, source) => api.index(perElement, id, source).get shouldBe true } + api.refresh(perElement) + def ids(where: String): Seq[Any] = + rows(s"SELECT id FROM $perElement WHERE $where").map(_("id")).sortBy(String.valueOf) + // '12.5' is 13 (half away from zero), so `> '12.5'` keeps 14 only + Seq("= '1.5'", "IN ('1.5', '3')", "BETWEEN '1.5' AND '2.5'", "> '12.5'").foreach { predicate => + withClue(s"[c $predicate] ") { + val twin = ids(s"n $predicate") + twin should not be empty + ids(s"arr $predicate") shouldBe twin + } + } + ids("arr = '1.5'") shouldBe Seq("e1") + ids("arr > '12.5'") shouldBe Seq("e4") + // a HAVING key filter on the element groups: '1.5' keeps the group 2, never the group 1 + def groups(c: String): Seq[(Any, Any)] = + rows(s"SELECT $c, COUNT(*) AS k FROM $perElement GROUP BY $c HAVING $c = '1.5'") + .map(r => (r(c), r("k"))) + groups("arr").map { case (key, count) => (String.valueOf(key), count) } shouldBe + groups("n").map { case (key, count) => (String.valueOf(key), count) } + groups("arr").map(g => String.valueOf(g._1)) shouldBe Seq("2") + // ROUND to tens: a BIGINT, a Long per row, the twin's value + val rounded = byId(rows(s"SELECT id, ROUND(arr, -1) AS r, ROUND(n, -1) AS t FROM $perElement")) + rounded.values.foreach { row => + withClue(s"[${row("id")}] ") { + row("r") shouldBe row("t") + cls(row("r")) shouldBe classOf[java.lang.Long].getName + } + } + drop(perElement) + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala index 9b79a8c47..604ab4f0b 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala @@ -55,9 +55,11 @@ case class DottedColumnRow(id: String, obj: Option[DottedColumnObj]) * * The documents are written as Elasticsearch stores them (a flat spelling stays flat), one per * shape the reader must handle: nested (`dn`), flat (`df`), both spellings (`db`), a flat `null` - * beside the nested value (`dz`) and its nested twin (`dn4`), an array of objects (`dl`), a - * sibling only (`ds`), missing (`dm`), a deeper path (`dx`) and a mixed spelling (`dmx`). The - * index has two shards, so the scroll is sliced on Elasticsearch 8. + * beside the nested value (`dz`) and its nested twin (`dn4`), a sibling only (`ds`), missing + * (`dm`), a deeper path (`dx`) and a mixed spelling (`dmx`). The index has two shards, so the + * scroll is sliced on Elasticsearch 8. An array of objects (`dl`) has its own index + * ([[listTable]]): its `obj.a` holds several values, which fails every `obj.a` statement until the + * column is declared a list (SCHEMA-6 A-26, lead Q1). * * PUBLIC API ONLY: the same file runs on a main clone, which is how the "census" (main's answers, * the canary) and the "population" (every statement on every route, as canonical rows) are @@ -76,6 +78,7 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma protected val copyTable = "dotted_cols_t2" protected val scriptedTable = "dotted_cols_t2s" protected val changelogTable = "dotted_cl" + protected val listTable = "dotted_cols_l" /** The documents of [[table]], as written (`_id` = `id`). */ protected val docs: Seq[(String, String)] = Seq( @@ -85,7 +88,6 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma "db" -> """{"id":"db","n":3,"obj.a":3,"obj":{"a":30}}""", "dz" -> """{"id":"dz","n":4,"obj.a":null,"obj":{"a":4}}""", "dn4" -> """{"id":"dn4","n":5,"obj":{"a":4}}""", - "dl" -> """{"id":"dl","n":6,"obj":[{"a":5},{"a":50}]}""", "ds" -> """{"id":"ds","n":7,"obj":{"b":6}}""", "dm" -> """{"id":"dm","n":8,"children":[{"name":"c3"}]}""", "dx" -> """{"id":"dx","obj":{"x":{"y":9}}}""", @@ -93,7 +95,7 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma ) /** What `obj.a` holds per document, as Elasticsearch reads it (D-z holds 4: a flat null is no - * value; D-l is multi-valued, which stays NULL until SCHEMA-6; D-mx is `obj.x.y`). + * value; D-mx is `obj.x.y`). */ protected val valueOfA: Map[String, Any] = Map( "dn" -> 1, @@ -101,13 +103,22 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma "db" -> 3, "dz" -> 4, "dn4" -> 4, - "dl" -> null, "ds" -> null, "dm" -> null, "dx" -> null, "dmx" -> null ) + /** [[listTable]] (SCHEMA-6 A-26): a list of objects in the `object` column `obj` (`dl`), one + * object (`dl1`), a list of one (`dl2`) and none (`dl3`). + */ + protected val listDocs: Seq[(String, String)] = Seq( + "dl" -> """{"id":"dl","n":6,"obj":[{"a":5},{"a":50}]}""", + "dl1" -> """{"id":"dl1","n":7,"obj":{"a":7}}""", + "dl2" -> """{"id":"dl2","n":8,"obj":[{"a":8}]}""", + "dl3" -> """{"id":"dl3","n":9}""" + ) + /** [[uniqueTable]]: every document carries one `obj.a` (nested or flat, no array, no flat null). */ protected val uniqueDocs: Seq[(String, String)] = Seq( @@ -155,9 +166,10 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma override def beforeAll(): Unit = { super.beforeAll() - Seq(table, uniqueTable, copyTable, scriptedTable, changelogTable).foreach(drop) + Seq(table, uniqueTable, copyTable, scriptedTable, changelogTable, listTable).foreach(drop) load(table, 2, docs) load(uniqueTable, 1, uniqueDocs) + load(listTable, 1, listDocs) runOk(s"CREATE TABLE $copyTable (id KEYWORD, obj STRUCT FIELDS(a INT), PRIMARY KEY (id))") Seq(scriptedTable, changelogTable).foreach { t => runOk( @@ -169,7 +181,7 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma } override def afterAll(): Unit = { - Seq(table, uniqueTable, copyTable, scriptedTable, changelogTable).foreach(drop) + Seq(table, uniqueTable, copyTable, scriptedTable, changelogTable, listTable).foreach(drop) super.afterAll() } @@ -530,7 +542,7 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma ) // SCHEMA-2 (D9): where a non-null flat spelling won, or the parent is an array, it is // dropped too -- SCHEMA-12 A-1's recorded leak ends: the projection's columns only - Seq("db", "dl").foreach(id => withClue(s"$id: ")(s2(id).keySet shouldBe Set("id", "obj.a"))) + Seq("db").foreach(id => withClue(s"$id: ")(s2(id).keySet shouldBe Set("id", "obj.a"))) val s3 = byId(rowsOk(s"SELECT id, obj.x.y FROM $table", route)) Seq("dx", "dmx").foreach(id => withClue(s"$id: ")(s3(id).keySet shouldBe Set("id", "obj.x.y")) @@ -606,6 +618,58 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma ) } + // ------------------------------------------------------------------------------------------- + // SCHEMA-6 (A-26, D6): a list of objects in an `object` column + // ------------------------------------------------------------------------------------------- + + "A list of objects in an object column" should "answer the column, fail its scalar path with its remedy, then read the path's values (A-26)" in { + // the STRUCT column answers its list of struct reads, as main did + rowRoutes.foreach { route => + withClue(s"[$route] ") { + val obj = byId(rowsOk(s"SELECT id, obj FROM $listTable", route)) + obj("dl")("obj") shouldBe List(ListMap("a" -> 5), ListMap("a" -> 50)) + obj("dl1")("obj") shouldBe ListMap("a" -> 7) + } + } + // a scalar path under it holds several values (Elasticsearch's `obj.a` = [5, 50]): Q1's + // failure, naming the document, the index and the ALTER that declares it a list + val failures = rowRoutes.map { route => + rowsOn(s"SELECT id, obj.a FROM $listTable", route) match { + case Left(message) => message + case Right(rows) => fail(s"[$route] answered $rows instead of the multi-valued failure") + } + } + failures.foreach { message => + message should include("Column obj.a holds several values in document dl") + message should include(s"ALTER TABLE $listTable ALTER COLUMN obj SET FIELD a ARRAY") + } + // beside the parent read whole too, never NULL + Seq(s"SELECT id, obj, obj.a FROM $listTable", s"SELECT *, obj.a FROM $listTable").foreach { + sql => + rowsOn(sql, "P1") match { + case Left(message) => message should include("Column obj.a holds several values") + case Right(rows) => fail(s"$sql answered $rows instead of the multi-valued failure") + } + } + // the remedy runs (meta-only) ... + val remedy = failures.head.substring(failures.head.indexOf("ALTER TABLE")) + runOk(remedy) + // ... and the path reads its values on every route, in each spelling + rowRoutes.foreach { route => + withClue(s"[$route] ") { + val a = byId(rowsOk(s"SELECT id, obj.a FROM $listTable", route)) + a("dl")("obj.a") shouldBe List(5, 50) + a("dl1")("obj.a") shouldBe List(7) + a("dl2")("obj.a") shouldBe List(8) + a("dl3")("obj.a") shouldBe (null: Any) + val both = byId(rowsOk(s"SELECT id, obj, obj.a FROM $listTable", route)) + both("dl")("obj.a") shouldBe List(5, 50) + byId(rowsOk(s"SELECT *, obj.a FROM $listTable", route))("dl")("obj.a") shouldBe + List(5, 50) + } + } + } + // core rendered `CAST(NULL AS VARCHAR)` as `String.valueOf(null)`: the TEXT "null" on every row, // alone in the projection (a regression vs 0.23.0) and beside other columns "A NULL cast to VARCHAR" should "answer NULL, alone and beside other columns, on every route" in { diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala index 8f7bb5a65..79207bc6d 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala @@ -1129,51 +1129,40 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit wrong shouldBe empty } - /** A value with SEVERAL values -- a multi-valued keyword -- in a column that converts its values - * fails the statement: the column holds one value of its type per row, and such a value has no - * one value to convert. The message is in DuckDB 1.5.5's words for the same column (MEASURED, - * each shape below). The CONTROL: the same value in a column whose branches have one type, which - * converts nothing, answers. + /** A value with SEVERAL values -- a multi-valued keyword -- in a column not declared a list fails + * the statement on every route, whether its column converts its values or not: since SCHEMA-6 + * (D6, lead Q1, G1) the branch reads the declared `KEYWORD` before any conversion, and fails in + * core's words, naming the document, the index and the ALTER that declares the column a list. + * 0.24.0 failed a converting column in DuckDB 1.5.5's words for a list column (`Conversion + * Error: Unimplemented type for cast (INTEGER -> VARCHAR[])`, MEASURED) and answered the list as + * read in the control, whose branches have one type and convert nothing. */ it should "fail a multi-valued value in a column that converts, on every route" in { - def failure(other: String): String = - s"Conversion Error: Unimplemented type for cast ($other -> VARCHAR[]) when casting from " + - "source column k" def limitedIf(limited: Boolean, statement: String): String = if (limited) statement.replace(" UNION ALL ", " LIMIT 100 UNION ALL ") + " LIMIT 100" else statement def union(left: String, right: String, rightTable: String = textTable): String = s"SELECT $left AS k FROM $textTable UNION ALL SELECT $right AS k FROM $rightTable" - val shapes: Seq[(String, String)] = Seq( - union("tags", "n", table) -> failure("INTEGER"), - union("tags", "ts") -> failure("TIMESTAMP"), - union("tags", "bin") -> failure("BLOB"), - union("x", "tags") -> failure("DOUBLE") + val several = s"Column tags holds several values in document t1 of index $textTable" + val remedy = s"ALTER TABLE $textTable ALTER COLUMN tags SET DATA TYPE ARRAY" + val shapes: Seq[String] = Seq( + union("tags", "n", table), + union("tags", "ts"), + union("tags", "bin"), + union("x", "tags"), + // the control: `tags` beside another KEYWORD converts nothing + union("tags", "kw") ) val wrong = for { - (statement, message) <- shapes - (route, limited) <- routes + statement <- shapes + (route, limited) <- routes sql = limitedIf(limited, statement) error <- rowsOf(sql, route) match { - case Left(msg) if msg.contains(message) => None - case other => Some(s"[$route] [$sql] answered $other instead of failing with $message") + case Left(msg) if msg.contains(several) && msg.endsWith(remedy) => None + case other => Some(s"[$route] [$sql] answered $other instead of the multi-valued failure") } } yield error - // the control: `tags` beside another KEYWORD, every row answered, the two values of t1 as the - // branch reads them - def bothTags(row: ListMap[String, Any]): Boolean = row.getOrElse("k", null) match { - case s: Seq[_] => s.map(e => String.valueOf(e)) == Seq("a", "b") - case l: java.util.List[_] => l.size == 2 && l.get(0) == "a" && l.get(1) == "b" - case _ => false - } - val unconverted = routes.flatMap { case (route, limited) => - val sql = limitedIf(limited, union("tags", "kw")) - rowsOf(sql, route) match { - case Right(rows) if rows.size == 2 * textDocs.size && rows.exists(bothTags) => None - case other => Some(s"[$route] [$sql] answered $other instead of every row, unconverted") - } - } - (wrong ++ unconverted) shouldBe empty + wrong shouldBe empty } /** A keyword core re-reads as a temporal when the response is parsed (issue #418) is spelled back @@ -1724,16 +1713,42 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit } /** A date core cannot read without guessing -- a region id, whose offset comes from zone data -- - * and a value of several values -- an array of arrays -- are answered as stored, at the column - * and at an object's field, on every route: no statement core answered fails. SCHEMA-2 (D12): at - * the column, such a format makes the column VARCHAR, as DuckDB types a text beside a date - * -- the other branch's date in DuckDB's spelling --, beside a DATE too, which failed before. + * is answered as stored, at the column and at an object's field, on every route. SCHEMA-2 (D12): + * at the column, such a format makes the column VARCHAR, as DuckDB types a text beside a date + * -- the other branch's date in DuckDB's spelling --, beside a DATE too, which failed before. A + * value of several values -- an array of arrays, which Elasticsearch flattens -- in a column not + * declared a list was answered as stored too; since SCHEMA-6 (D6, lead Q1, G1) it fails the + * statement on every route, naming the document, the index and the ALTER that declares the + * column -- or the object's field -- a list. */ it should "keep a date it cannot read as stored, its column VARCHAR at the top" in { def sql(left: String, right: String, limited: Boolean): String = Seq(left, right) .map(i => s"SELECT $i AS k FROM $zoneTable" + (if (limited) " LIMIT 100" else "")) .mkString(" UNION ALL ") + val several = for { + (left, right, column, remedy) <- Seq( + ("v", "ts", "v", "ALTER COLUMN v SET DATA TYPE ARRAY"), + ( + "fv", + "fts", + "fv.a", + "ALTER COLUMN fv SET FIELD a ARRAY OPTIONS (format = \"epoch_second\")" + ) + ) + (route, limited) <- routes + statement = sql(left, right, limited) + error <- rowsOf(statement, route) match { + case Left(e) + if e.contains( + s"Column $column holds several values in document z1 of index $zoneTable" + ) && + e.endsWith(s"ALTER TABLE $zoneTable $remedy") => + None + case other => + Some(s"[$route] [$statement] answered $other instead of the multi-valued failure") + } + } yield error val kept = for { (left, right, stored, partner) <- Seq( ("z", "ts", zoned: Any, "2024-02-01 00:00:00": Any), @@ -1742,19 +1757,6 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit "fts", ListMap("a" -> zoned): Any, ListMap("a" -> ZonedDateTime.parse("2024-02-01T00:00:00Z")): Any - ), - // an array of arrays: several values, as stored - ( - "v", - "ts", - List(List(1706697000, 1706697001)): Any, - ZonedDateTime.parse("2024-02-01T00:00:00Z"): Any - ), - ( - "fv", - "fts", - ListMap("a" -> List(List(1706697000, 1706697001))): Any, - ListMap("a" -> ZonedDateTime.parse("2024-02-01T00:00:00Z")): Any ) ) (route, limited) <- routes @@ -1784,7 +1786,7 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit case Left(e) => Some(s"[$route] [$statement] failed: $e") } } yield error - (kept ++ besideDate) shouldBe empty + (several ++ kept ++ besideDate) shouldBe empty } it should "refuse by name only a date beside a number, before anything runs" in {