From c9b8eb552ed50bd19dfa0771c6061f55cb375fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 31 Oct 2026 17:01:06 +0100 Subject: [PATCH 1/3] feat(core): aggregation values and window statements read by type (SCHEMA-2b) Story SCHEMA-2b. Reviewed: r0:must-fix, r1:must-fix, r2:clean. Co-Authored-By: Claude Opus 5.5 --- .../elastic/client/AggregateReading.scala | 652 ++++++++++++++++++ .../elastic/client/ElasticConversion.scala | 152 +++- .../elastic/client/HitReading.scala | 124 +++- .../elastic/client/ScrollApi.scala | 35 +- .../elastic/client/SearchApi.scala | 165 +++-- .../softnetwork/elastic/client/package.scala | 85 ++- .../elastic/client/AggregateReadingSpec.scala | 387 +++++++++++ .../client/WindowPartitionKeySpec.scala | 448 ++++++++++++ documentation/client/gateway.md | 28 +- documentation/sql/functions_aggregate.md | 8 +- .../arrow/JavaClientSchemaConsumerSpec.scala | 82 ++- .../sql/function/aggregate/package.scala | 17 +- .../elastic/sql/query/package.scala | 30 +- .../sql/query/AggregateReportedTypeSpec.scala | 40 +- .../sql/query/DateArithmeticTypeSpec.scala | 7 +- .../sql/query/TypeCheckResolutionSpec.scala | 3 +- .../elastic/client/ElasticClientSpec.scala | 9 +- .../elastic/client/SchemaReadingSpec.scala | 308 ++++++++- .../UnnestFunctionRefusalExecutionSpec.scala | 7 +- 19 files changed, 2430 insertions(+), 157 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/AggregateReadingSpec.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/WindowPartitionKeySpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala b/core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala new file mode 100644 index 00000000..8f6064e5 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/AggregateReading.scala @@ -0,0 +1,652 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.query.{Bucket, MultiSearch, SingleSearch, TemporalLiterals} +import app.softnetwork.elastic.sql.query.MultiSearch.SetOperationType +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} + +import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneOffset, ZonedDateTime} +import java.time.temporal.ChronoUnit +import scala.collection.immutable.ListMap + +/** SCHEMA-2b (B1-B6) -- how ONE statement reads the values of its aggregation response by its + * columns' declared types, where each is first read (a Jackson node): + * + * - `keys`: a level's bucket key, by the level's name (`Bucket.name`), read by the type of its + * key ([[MultiSearch.bucketType]], B2); + * - `metrics`: a metric's `value`, by aggregation name, read by the type of the SELECT item that + * owns it ([[MultiSearch.fieldType]], B3 / B4) -- exactly: a number becomes the class only + * where the class holds it unchanged; + * - `tops`: a `FIRST_VALUE` / `LAST_VALUE` / `ARRAY_AGG` hit's `_source`, by top_hits name, its + * aggregated column read by the column's reader (`HitReading.Readers`, B5). + * + * A value a reader cannot read answers [[AggregateReading.Unread]], and the caller reads it as it + * always did (A-12). Built ONCE per statement, by `SchemaColumns.of`; [[AggregateReading.Empty]] + * reads everything as 0.24.0 did. + */ +private[client] final class AggregateReading( + keys: java.util.HashMap[String, AggregateReading.KeyReader], + metrics: java.util.HashMap[String, JsonNode => Any], + tops: java.util.HashMap[String, java.util.HashMap[String, JsonNode => Any]] +) { + + /** The reader of the bucket keys of the level `name`, `null` for today's reading. */ + def key(name: String): AggregateReading.KeyReader = if (keys.isEmpty) null else keys.get(name) + + /** The reader of the `value` of the aggregation `name`, `null` for today's reading. */ + def metric(name: String): JsonNode => Any = if (metrics.isEmpty) null else metrics.get(name) + + /** The readers of the `_source` keys of the top_hits `name`, `null` for today's reading. */ + def top(name: String): java.util.HashMap[String, JsonNode => Any] = + if (tops.isEmpty) null else tops.get(name) + + /** Whether it reads nothing: every value as 0.24.0 did. */ + def isEmpty: Boolean = keys.isEmpty && metrics.isEmpty && tops.isEmpty +} + +private[client] object AggregateReading { + + /** What a reader answers for a value it cannot read: the caller reads it as it always did. */ + case object Unread + + /** Reads a bucket node; `doubleLevel` as `ElasticConversion.extractBucketKey` takes it. */ + trait KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any + } + + /** Today's reading everywhere. */ + val Empty: AggregateReading = + new AggregateReading( + new java.util.HashMap[String, KeyReader](), + new java.util.HashMap[String, JsonNode => Any](), + new java.util.HashMap[String, java.util.HashMap[String, JsonNode => Any]]() + ) + + private val DayMillis = 86400000L + + /** The readers of `single`'s aggregation response; [[Empty]] where it has no schema (D10: no + * declared type) or no aggregation. + * + * @param instants + * the levels whose key is read as the bucket's INSTANT ([[InstantKey]]): the partition columns + * of a window statement paired by instant ([[PartitionKeys.instantColumns]]), when `single` is + * its window aggregation request; none otherwise. + */ + def of( + single: SingleSearch, + dates: SetOperationValues.DateReading, + instants: Set[String] = Set.empty + ): AggregateReading = + if (single.schema.isEmpty || (single.buckets.isEmpty && single.sqlAggregations.isEmpty)) Empty + else { + val readers = + new HitReading.Readers( + single.fieldAliases, + dates, + HitReading.Default, + single.sources.headOption.getOrElse("") + ) + val keys = new java.util.HashMap[String, KeyReader]() + single.buckets.foreach { b => + val name = b.name + if (name.nonEmpty && !keys.containsKey(name)) { + val reader = + if (instants.contains(name)) InstantKey + else keyReader(name, MultiSearch.bucketType(b), dates, readers) + if (reader ne null) keys.put(name, reader) + } + } + // the SELECT item that owns each aggregation: its output name is the aggregation's name + val owners: Map[String, app.softnetwork.elastic.sql.query.Field] = + single.select.fieldsWithComputedAliases + .filter(f => f.isAggregation || f.isBucketScript || f.identifier.hasWindow) + .foldLeft(Map.empty[String, app.softnetwork.elastic.sql.query.Field]) { (acc, f) => + if (acc.contains(f.outputName)) acc else acc + (f.outputName -> f) + } + val metrics = new java.util.HashMap[String, JsonNode => Any]() + val tops = new java.util.HashMap[String, java.util.HashMap[String, JsonNode => Any]]() + single.sqlAggregations.foreach { case (name, aggregation) => + // an aggregation the client plan cannot hold is the search's own failure, not a reader's + val client: Option[ClientAggregation] = + if (aggregation.auxiliary) None + else scala.util.Try(sqlAggregationToClientAggregation(aggregation)).toOption + client.foreach { client => + client.aggType match { + case AggregationType.Count | AggregationType.Sum | AggregationType.Avg | + AggregationType.Min | AggregationType.Max | AggregationType.BucketScript => + owners.get(name).foreach { field => + val reader = metricReader(MultiSearch.fieldType(field)) + if (reader ne null) metrics.put(name, reader) + } + case AggregationType.FirstValue | AggregationType.LastValue | + AggregationType.ArrayAgg => + // the response names a top_hits by its last segment, a nested one included + val leaf = name.substring(name.lastIndexOf('.') + 1) + val source = aggregation.sourceField + single.schema.flatMap(_.find(source)).foreach { column => + val t = MultiSearch.columnType(column) + val key = aggregation.nestedElement match { + case Some(n) if source.startsWith(n.path + ".") => + source.substring(n.path.length + 1) + case Some(_) => source.substring(source.lastIndexOf('.') + 1) + case None => source + } + if (key.nonEmpty && !tops.containsKey(leaf)) { + val byField = new java.util.HashMap[String, JsonNode => Any]() + byField.put(key, readers.column(source, t, Set.empty)._1) + tops.put(leaf, byField) + } + } + case _ => // statistics, percentiles, rankings: as today + } + } + } + if (keys.isEmpty && metrics.isEmpty && tops.isEmpty) Empty + else new AggregateReading(keys, metrics, tops) + } + + // --------------------------------------------------------------------------------------------- + // Keys (B2) + // --------------------------------------------------------------------------------------------- + + /** The reader of a level's keys of type `t`; `null` for a type read as it comes (NUMERIC, + * TEMPORAL, ANY, nested types), whose keys keep today's reading. + */ + private def keyReader( + name: String, + t: SetOperationType, + dates: SetOperationValues.DateReading, + readers: HitReading.Readers + ): KeyReader = { + val route = DateRoute.of(t.sqlType, t.format, dates) + route match { + case DateRoute.AsStored(_) => TextKey + case DateRoute.Typed => + t.sqlType match { + case SQLTypes.Boolean => BooleanKey + case SQLTypes.TinyInt => + new WholeKey(Byte.MinValue.toLong, Byte.MaxValue.toLong, int = true) + case SQLTypes.SmallInt => + new WholeKey(Short.MinValue.toLong, Short.MaxValue.toLong, int = true) + case SQLTypes.Int => new WholeKey(Int.MinValue.toLong, Int.MaxValue.toLong, int = true) + case SQLTypes.BigInt => new WholeKey(Long.MinValue, Long.MaxValue, int = false) + case SQLTypes.Double => DoubleKey + case SQLTypes.Real => RealKey + case kind @ (SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time) => + // a scripted key's text: D5's reader of Elasticsearch's default format (no second + // date rule), or of a computed date where the major is not known + val default = Some(TemporalLiterals.DefaultDateFormat) + val text: JsonNode => Any = + if (DateRoute.of(kind, default, dates) == DateRoute.Typed) + readers.column(name, SetOperationType(kind, format = default), Set.empty)._1 + else readers.column(name, SetOperationType(kind), Set.empty)._1 + new TemporalKey(kind, text) + case SQLTypes.Varchar | SQLTypes.Keyword | SQLTypes.Text | SQLTypes.Char => TextKey + case _ => null + } + } + } + + /** `key_as_string` where the response has one (Elasticsearch's own text), else the key's text -- + * never re-read as a temporal. + */ + private object TextKey extends KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any = { + val text = bucket.get("key_as_string") + if (text != null && text.isTextual) text.asText() + else { + val key = bucket.get("key") + if (key != null && key.isValueNode && !key.isNull) key.asText() else Unread + } + } + } + + /** A window partition level paired by instant ([[PartitionKeys.instantColumns]]): the bucket's + * `key`, the epoch milliseconds of the instant Elasticsearch buckets the date by (aggregations + * are millisecond-resolution), as its decimal text -- the spelling of the base side's + * `epoch_millis` doc value ([[PartitionKeys.instantOf]]). Never `key_as_string`: the field's + * format at UTC spells one instant otherwise than the stored text does (`2024-01-31 10:30 Z` vs + * `... 11:30 CET`). + */ + private object InstantKey extends KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any = { + val key = bucket.get("key") + val millis = + if (key == null || !key.isNumber) null + else wholeOf(key, Long.MinValue, Long.MaxValue, text = false) + if (millis eq null) Unread else millis.toString + } + } + + /** A `boolean` level: the key `1` / `0` (`key_as_string` `"true"` / `"false"`). */ + private object BooleanKey extends KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any = { + val key = bucket.get("key") + if (key == null) Unread + else if (key.isBoolean) java.lang.Boolean.valueOf(key.booleanValue()) + else if (key.isNumber) { + val d = key.doubleValue() + if (d == 1d) java.lang.Boolean.TRUE else if (d == 0d) java.lang.Boolean.FALSE else Unread + } else if (key.isTextual) + SetOperationValues.boolean(key.asText()) match { + case b: java.lang.Boolean => b + case _ => Unread + } + else Unread + } + } + + /** A whole level: the key when it is a whole number within `[min, max]`, whatever its node kind + * (B3's exact rule); an `Integer` (`int`) or a `Long`. + */ + private final class WholeKey(min: Long, max: Long, int: Boolean) extends KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any = { + val key = bucket.get("key") + val whole = if (key == null) null else wholeOf(key, min, max, text = true) + if (whole eq null) Unread + else if (int) java.lang.Integer.valueOf(whole.intValue()) + else whole + } + } + + /** A `DOUBLE` level: the number; a non-finite key's text (`NonFiniteKeys`) is the Double. */ + private object DoubleKey extends KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any = { + val key = bucket.get("key") + if (key == null) Unread + else if (key.isNumber) java.lang.Double.valueOf(key.doubleValue()) + else if (key.isTextual && ElasticConversion.NonFiniteKeys.contains(key.asText())) + java.lang.Double.valueOf(key.asText()) + else Unread + } + } + + /** A `REAL` level: the number at `REAL`'s precision; a non-finite key's text is the Float. */ + private object RealKey extends KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any = { + val key = bucket.get("key") + if (key == null) Unread + else if (key.isNumber) java.lang.Float.valueOf(key.floatValue()) + else if (key.isTextual && ElasticConversion.NonFiniteKeys.contains(key.asText())) + java.lang.Float.valueOf(key.asText()) + else Unread + } + } + + /** A typed `DATE` / `TIMESTAMP` / `TIME` level: a whole key is epoch milliseconds at UTC (the + * bridge sends no `time_zone`), read EXACTLY (A-23): a `DATE` only at a whole UTC day, a `TIME` + * only on 1970-01-01, any other key as it always was; a text key (a scripted level) by `text`. + */ + private final class TemporalKey(kind: SQLType, text: JsonNode => Any) extends KeyReader { + def read(bucket: JsonNode, doubleLevel: Boolean): Any = { + val key = bucket.get("key") + if (key == null || key.isNull) Unread + else if (key.isNumber) { + val millis = wholeOf(key, Long.MinValue, Long.MaxValue, text = false) + if (millis eq null) Unread + else { + val value = exactly(kind, millis.longValue()) + if (value == null) Unread else value + } + } else if (key.isTextual) text(key) + else Unread + } + } + + /** `instant` as a value of `kind` when it is EXACTLY one (A-23): a `DATE` at a whole UTC day, a + * `TIME` on 1970-01-01, a `TIMESTAMP` always (at UTC); `null` otherwise. + */ + private[client] def exactly(kind: SQLType, instant: Instant): Any = + kind match { + case SQLTypes.Date => + if (instant.getNano == 0 && Math.floorMod(instant.getEpochSecond, 86400L) == 0L) + LocalDate.ofEpochDay(Math.floorDiv(instant.getEpochSecond, 86400L)) + else null + case SQLTypes.Time => + if (instant.getEpochSecond >= 0L && instant.getEpochSecond < 86400L) + LocalTime.ofNanoOfDay(instant.getEpochSecond * 1000000000L + instant.getNano) + else null + case _ => instant.atZone(ZoneOffset.UTC) + } + + private def exactly(kind: SQLType, millis: Long): Any = + kind match { + case SQLTypes.Date => + if (Math.floorMod(millis, DayMillis) == 0L) + LocalDate.ofEpochDay(Math.floorDiv(millis, DayMillis)) + else null + case SQLTypes.Time => + if (millis >= 0L && millis < DayMillis) LocalTime.ofNanoOfDay(millis * 1000000L) else null + case _ => Instant.ofEpochMilli(millis).atZone(ZoneOffset.UTC) + } + + // --------------------------------------------------------------------------------------------- + // Metrics (B3, B4) + // --------------------------------------------------------------------------------------------- + + /** The reader of a metric's `value` of type `t` -- EXACT: a number becomes the class only where + * the class holds it unchanged, else [[Unread]] (kept as stored, A-12); `null` for a type read + * as it comes. + */ + private def metricReader(t: SetOperationType): JsonNode => Any = + t.sqlType match { + case SQLTypes.TinyInt => wholeMetric(Byte.MinValue.toLong, Byte.MaxValue.toLong, int = true) + case SQLTypes.SmallInt => + wholeMetric(Short.MinValue.toLong, Short.MaxValue.toLong, int = true) + case SQLTypes.Int => wholeMetric(Int.MinValue.toLong, Int.MaxValue.toLong, int = true) + case SQLTypes.BigInt => wholeMetric(Long.MinValue, Long.MaxValue, int = false) + case SQLTypes.Double => + node => + if (node.isFloatingPointNumber) { + val d = node.doubleValue() + if (finite(d)) java.lang.Double.valueOf(d) else Unread + } else if (node.isIntegralNumber && node.canConvertToLong) { + val l = node.longValue() + val d = l.toDouble + if (d.toLong == l && d != 9.223372036854775807e18) java.lang.Double.valueOf(d) + else Unread + } else Unread + case SQLTypes.Real => + node => + if (node.isNumber && finite(node.doubleValue())) + java.lang.Float.valueOf(node.floatValue()) + else Unread + case SQLTypes.Boolean => + node => + if (node.isBoolean) java.lang.Boolean.valueOf(node.booleanValue()) + else if (node.isNumber) { + val d = node.doubleValue() + if (d == 1d) java.lang.Boolean.TRUE + else if (d == 0d) java.lang.Boolean.FALSE + else Unread + } else Unread + case kind @ (SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time) => + // epoch milliseconds, never `value_as_string`: a metric truncates to its class + node => + if (node.isNumber && finite(node.doubleValue())) { + val d = node.doubleValue() + if (d < -9.223372036854775808e18 || d >= 9.223372036854775808e18) Unread + else { + val instant = + Instant.ofEpochMilli(if (node.isIntegralNumber) node.longValue() else d.toLong) + kind match { + case SQLTypes.Date => instant.atZone(ZoneOffset.UTC).toLocalDate + case SQLTypes.Time => instant.atZone(ZoneOffset.UTC).toLocalTime + case _ => instant.atZone(ZoneOffset.UTC) + } + } + } else Unread + case SQLTypes.Varchar | SQLTypes.Keyword | SQLTypes.Text | SQLTypes.Char => + node => if (node.isTextual) node.asText() else Unread + case _ => null + } + + private def wholeMetric(min: Long, max: Long, int: Boolean): JsonNode => Any = + node => { + val whole = wholeOf(node, min, max, text = false) + if (whole eq null) Unread + else if (int) java.lang.Integer.valueOf(whole.intValue()) + else whole + } + + private def finite(d: Double): Boolean = !d.isNaN && !d.isInfinite + + /** `node` as a whole number within `[min, max]` -- an integral number, a finite floating-point + * number with no fraction, or (`text`) a text spelling one --, `null` otherwise: never rounded, + * never wrapped. + */ + private[client] def wholeOf(node: JsonNode, min: Long, max: Long, text: Boolean): java.lang.Long = + if (node.isIntegralNumber) { + if (node.canConvertToLong) { + val v = node.longValue() + if (v >= min && v <= max) java.lang.Long.valueOf(v) else null + } else null + } else if (node.isFloatingPointNumber) { + val d = node.doubleValue() + if ( + !finite(d) || d != Math.rint( + d + ) || d < -9.223372036854775808e18 || d >= 9.223372036854775808e18 + ) + null + else { + val v = d.toLong + if (v >= min && v <= max) java.lang.Long.valueOf(v) else null + } + } else if (text && node.isTextual) { + val parsed = + try new java.math.BigDecimal(node.asText()) + catch { case _: NumberFormatException => null } + if (parsed eq null) null + else { + val whole = + try parsed.toBigIntegerExact + catch { case _: ArithmeticException => null } + if ((whole eq null) || whole.bitLength() > 63) null + else { + val v = whole.longValue() + if (v >= min && v <= max) java.lang.Long.valueOf(v) else null + } + } + } else null +} + +/** SCHEMA-2b (B8) -- the partition key of ONE side of a window statement's join: ONE function per + * statement and side, the names of today's derivation hoisted, each value through its key function + * -- a `TIMESTAMP` / `TIME` value truncated to the millisecond under [[SchemaColumns]] (doc + * values, a bucket's `key` and a `date_nanos` aggregation are milliseconds, a `_source` may hold + * more), every other value as it is. Each name is read under its row key: the name itself, but on + * the base side of a partition column paired by instant ([[PartitionKeys.instantColumns]]), the + * key its `epoch_millis` doc value is read under ([[PartitionKeys.instantKey]]). + */ +private[client] final class PartitionKeys( + names: Array[String], + rowKeys: Array[String], + keyOf: Array[Any => Any] +) { + + /** The partition key of `row`: `__global__` without PARTITION BY; a name `row` lacks left out. */ + def of(row: ListMap[String, Any]): ListMap[String, Any] = + if (names.isEmpty) PartitionKeys.Global + else { + val builder = ListMap.newBuilder[String, Any] + var i = 0 + while (i < names.length) { + row.get(rowKeys(i)) match { + case Some(v) => + val keyFunction = keyOf(i) + builder += names(i) -> (if (keyFunction eq null) v else keyFunction(v)) + case None => + } + i += 1 + } + builder.result() + } +} + +private[client] object PartitionKeys { + + /** The one partition of a window statement with no PARTITION BY. */ + val Global: ListMap[String, Any] = ListMap("__global__" -> true) + + private val toMillis: Any => Any = { + case z: ZonedDateTime => z.truncatedTo(ChronoUnit.MILLIS) + case t: LocalTime => t.truncatedTo(ChronoUnit.MILLIS) + case l: LocalDateTime => l.truncatedTo(ChronoUnit.MILLIS) + case other => other + } + + /** The partition keys of `request`'s rows read in `context`: the window aggregation's (cache + * side) or the base query's (base side). + */ + def of(request: SingleSearch, context: ConversionContext): PartitionKeys = { + val partitions = request.windowFunctions.flatMap(wf => wf.partitionBy.zip(wf.buckets)) + val names = partitions.map(_._1.aliasOrName).distinct.toArray + val instants: java.util.HashMap[String, String] = context match { + case s: SchemaColumns => s.reading.instants + case _ => null + } + // the base side of an instant column reads its doc value's key; the cache side, whose reading + // holds no instant, reads the name, under which its key reader put the bucket's instant + val rowKeys: Array[String] = + if ((instants eq null) || instants.isEmpty) names + else + names.map { name => + val key = instants.get(name) + if (key eq null) name else key + } + val keyOf = new Array[Any => Any](names.length) + if (context.isInstanceOf[SchemaColumns]) { + var i = 0 + while (i < names.length) { + val name = names(i) + if (rowKeys(i) eq name) + partitions.find(_._1.aliasOrName == name).foreach { case (_, bucket) => + MultiSearch.bucketType(bucket).sqlType match { + case SQLTypes.Timestamp | SQLTypes.Time => keyOf(i) = toMillis + case _ => + } + } + i += 1 + } + } + new PartitionKeys(names, rowKeys, keyOf) + } + + // --------------------------------------------------------------------------------------------- + // Partition columns paired by instant + // --------------------------------------------------------------------------------------------- + + /** The PARTITION BY columns of the window statement `request` paired by INSTANT: each plain + * column whose key is a date read AS STORED (`DateRoute.AsStored`: core does not read every + * value of its format as the Elasticsearch major does). Its base row holds the stored text and + * its bucket's `key_as_string` is the field's format applied to the bucket's instant at UTC: two + * spellings of one instant (`2024-01-31 11:30 CET` stored, `2024-01-31 10:30 Z` from + * Elasticsearch 8.18.3), which never paired. Both sides pair instead by the instant + * Elasticsearch supplies -- the bucket's millisecond `key` (cache side, + * `AggregateReading.of(instants)`) and the column's `epoch_millis` doc value (base side, + * [[withInstantDocValues]], `HitReading.of(instants)`) --, as Elasticsearch partitions a date; + * core never parses the text. The displayed value stays the stored text. + * + * A column the base statement COMPUTES under the same name (`CONCAT(pz, '') AS pz`) is not + * paired by instant: its reader reads the hit's `fields` entry of that name, so a doc value + * asked there would be read as the item's value. Such an item keeps the window's partition + * bucket, so no script field is sent for it either (Elasticsearch 8.18.3): both sides read it as + * 0.24.0 did, and the column and its window values read NULL, as on 0.24.0. + * + * ONE function for both sides, its set taken as it is by each (`SchemaColumns.ofWindowBase`: the + * base reading and the base request's `docvalue_fields`; `SchemaColumns.ofWindowAggregation`: + * the bucket keys), decided per statement from the declared types, the mapping formats, the + * major (`dates`) and the base statement's projection. + */ + def instantColumns(request: SingleSearch, dates: SetOperationValues.DateReading): Set[String] = + if (request.schema.isEmpty) Set.empty + else { + val candidates = request.windowFunctions + .flatMap(wf => wf.partitionBy.zip(wf.buckets)) + .collect { + case (id, bucket) + if id.aliasOrName == bucket.identifier.name && byInstant(bucket, dates) => + id.aliasOrName + } + .toSet + if (candidates.isEmpty) candidates + else { + // the output names the base statement computes: `SingleSearch.withoutWindows`'s SELECT -- + // this statement's (updated) items but its windows, `__cN` numbered without them -- the + // projection its reading reads computed columns from (`HitReading.isComputed`) + val computed = request.select + .copy(fields = request.select.fields.filterNot(_.identifier.hasWindow)) + .fieldsWithComputedAliases + .filter(HitReading.isComputed) + .map(_.outputName) + .toSet + candidates.filterNot(computed.contains) + } + } + + /** A plain date column whose key is read as stored (see [[instantColumns]]). */ + private def byInstant(bucket: Bucket, dates: SetOperationValues.DateReading): Boolean = { + val id = bucket.identifier + val name = id.name + val plain = + !id.nested && id.functions.isEmpty && name.nonEmpty && name.indexOf('.') < 0 && + bucket.name == name + plain && { + val t = MultiSearch.bucketType(bucket) + t.sqlType match { + case SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time => + DateRoute.of(t.sqlType, t.format, dates).isInstanceOf[DateRoute.AsStored] + case _ => false + } + } + } + + /** The base row key the doc value of the instant column `name` is read under: led by a NUL, so no + * column's name, and dropped by the final projection. + */ + private[client] def instantKey(name: String): String = "\u0000instant:" + name + + /** The instant of a base row's `epoch_millis` doc value (a one-element array of its text): the + * epoch milliseconds' decimal text, the spelling of [[AggregateReading]] 's instant key -- any + * sub-millisecond fraction (a `date_nanos` value) left out, as its aggregation buckets it. No + * parse: Elasticsearch formats the number. A document holding several values keeps them all and + * pairs with no bucket (NULL, as on main). + */ + private[client] def instantOf(node: JsonNode): Any = + if (node.isArray) { + if (node.size() == 1) millisText(node.get(0)) + else { + val values = List.newBuilder[String] + val it = node.elements() + while (it.hasNext) values += millisText(it.next()) + values.result() + } + } else millisText(node) + + private def millisText(node: JsonNode): String = { + val text = node.asText() + val dot = text.indexOf('.') + if (dot < 0) text else text.substring(0, dot) + } + + /** `query` (a search body) also asking each column of `fields` as its `epoch_millis` doc value + * (`docvalue_fields`, accepted with a `format` from Elasticsearch 6.4 on, through 9): the base + * query of a window statement whose `fields` pair by instant. Built once per statement; `query` + * itself when `fields` is empty. + */ + private[client] def withInstantDocValues(query: String, fields: Seq[String]): String = + if (fields.isEmpty) query + else + JacksonConfig.objectMapper.readTree(query) match { + case root: ObjectNode => + val docValues = root.get("docvalue_fields") match { + case existing: ArrayNode => existing + case _ => root.putArray("docvalue_fields") + } + fields.foreach { field => + docValues.addObject().put("field", field).put("format", "epoch_millis") + () + } + JacksonConfig.objectMapper.writeValueAsString(root) + case _ => query + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index 1aa2eaa1..444cce72 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -437,6 +437,12 @@ trait ElasticConversion { case _ => Set.empty } + // SCHEMA-2b (B1) -- the statement's aggregation values by type; 0.24.0's reading elsewhere + val reading: AggregateReading = context match { + case s: SchemaColumns => s.aggregates + case _ => AggregateReading.Empty + } + val rows = (hitsNode, aggsNode) match { case (Some(hits), None) if hits.nonEmpty => // Case 1 : only hits @@ -447,34 +453,36 @@ trait ElasticConversion { case (None, Some(aggs)) => // Case 2 : only aggregations val ret = - parseAggregations( + parseAggregationsWith( aggs, rowInvariants, fieldAliases, aggregations, rootDocCount, - doubleKeys + doubleKeys, + reading ) combineAggregationRows(ret) case (Some(hits), Some(aggs)) if hits.isEmpty => // Case 3 : aggregations with no hits val ret = - parseAggregations( + parseAggregationsWith( aggs, rowInvariants, fieldAliases, aggregations, rootDocCount, - doubleKeys + doubleKeys, + reading ) combineAggregationRows(ret) case (Some(hits), Some(aggs)) if hits.nonEmpty => // Case 4 : Hits + global aggregations + top_hits aggregations - val globalMetrics = extractGlobalMetrics(aggs, aggregations) + val globalMetrics = extractGlobalMetricsWith(aggs, aggregations, reading) val allTopHits = extractAggregationValues( - extractAllTopHits(aggs, fieldAliases, aggregations), + extractAllTopHitsWith(aggs, fieldAliases, aggregations, reading), aggregations ) val parsed = parseSimpleHits(hits, fieldAliases, fields, retainDocumentId) @@ -819,13 +827,20 @@ trait ElasticConversion { reading: HitReading ): ListMap[String, Any] = { val entries = ListMap.newBuilder[String, Any] + val instants = reading.instants val it = node.properties().iterator() while (it.hasNext) { val e = it.next() - val reader = reading.scripted.get(e.getKey) - entries += fieldAliases.getOrElse(e.getKey, e.getKey) -> ( - if (reader ne null) reader(e.getValue) else jsonNodeToAny(e.getValue, fieldAliases) - ) + // SCHEMA-2b -- a partition column's `epoch_millis` doc value: its instant, under its own row + // key, beside the column's stored value (`PartitionKeys.instantColumns`) + val instantKey = if (instants.isEmpty) null else instants.get(e.getKey) + if (instantKey ne null) entries += instantKey -> PartitionKeys.instantOf(e.getValue) + else { + val reader = reading.scripted.get(e.getKey) + entries += fieldAliases.getOrElse(e.getKey, e.getKey) -> ( + if (reader ne null) reader(e.getValue) else jsonNodeToAny(e.getValue, fieldAliases) + ) + } } entries.result() } @@ -1321,6 +1336,29 @@ trait ElasticConversion { aggregations: ListMap[String, ClientAggregation], docCount: Option[Long] = None, doubleKeys: Set[String] = Set.empty + ): Seq[ListMap[String, Any]] = + parseAggregationsWith( + aggsNode, + parentContext, + fieldAliases, + aggregations, + docCount, + doubleKeys, + AggregateReading.Empty + ) + + /** [[parseAggregations]], each bucket key, metric and top_hits value read by `reading` + * (SCHEMA-2b, B2-B5) -- the statement's [[AggregateReading]], or [[AggregateReading.Empty]] for + * 0.24.0's. + */ + private[client] def parseAggregationsWith( + aggsNode: JsonNode, + parentContext: ListMap[String, Any], + fieldAliases: ListMap[String, String], + aggregations: ListMap[String, ClientAggregation], + docCount: Option[Long], + doubleKeys: Set[String], + reading: AggregateReading ): Seq[ListMap[String, Any]] = { if (aggsNode.isMissingNode || !aggsNode.isObject) { @@ -1402,19 +1440,20 @@ trait ElasticConversion { } // Recursively parse subaggregations - parseAggregations( + parseAggregationsWith( subAggsNode, currentContext, fieldAliases, aggregations, docCountOf(aggValue, "doc_count").orElse(docCount), - doubleKeys + doubleKeys, + reading ) } } else if (bucketAggs.isEmpty) { // No buckets : it is a leaf aggregation (metrics or top_hits) - val metrics = extractMetrics(aggsNode, aggregations, docCount) - val allTopHits = extractAllTopHits(aggsNode, fieldAliases, aggregations) + val metrics = extractMetricsWith(aggsNode, aggregations, docCount, reading) + val allTopHits = extractAllTopHitsWith(aggsNode, fieldAliases, aggregations, reading) if (allTopHits.nonEmpty) { Seq(parentContext ++ metrics ++ allTopHits) @@ -1426,12 +1465,21 @@ trait ElasticConversion { } else { // Handle each aggregation with buckets bucketAggs.flatMap { case (aggName, buckets, _) => + // SCHEMA-2b (B2) -- the level's key reader, looked up once per level + val keyReader = reading.key(aggName) + val doubleLevel = doubleKeys.contains(aggName) buckets.flatMap { bucket => val bucketDocCount = docCountOf(bucket, "doc_count") - val metrics = extractMetrics(bucket, aggregations, bucketDocCount) - val allTopHits = extractAllTopHits(bucket, fieldAliases, aggregations) + val metrics = extractMetricsWith(bucket, aggregations, bucketDocCount, reading) + val allTopHits = extractAllTopHitsWith(bucket, fieldAliases, aggregations, reading) - val bucketKey = extractBucketKey(bucket, doubleKeys.contains(aggName)) + val bucketKey = + if (keyReader eq null) extractBucketKey(bucket, doubleLevel) + else + keyReader.read(bucket, doubleLevel) match { + case AggregateReading.Unread => extractBucketKey(bucket, doubleLevel) + case typed => typed + } // The synthetic whole-table bucket a `HAVING` with no `GROUP BY` is wrapped in is // TRANSPARENT: it exists only to give the `having_filter` bucket_selector a multi-bucket @@ -1464,13 +1512,14 @@ trait ElasticConversion { /*subAggFields.foreach { entry => subAggsNode.set(entry.getKey, entry.getValue) // FIXME }*/ - parseAggregations( + parseAggregationsWith( subAggsNode, currentContext, fieldAliases, aggregations, bucketDocCount, - doubleKeys + doubleKeys, + reading ) } else { Seq(currentContext) @@ -1733,6 +1782,18 @@ trait ElasticConversion { aggsNode: JsonNode, aggregations: ListMap[String, ClientAggregation], docCount: Option[Long] = None + ): ListMap[String, Any] = + extractMetricsWith(aggsNode, aggregations, docCount, AggregateReading.Empty) + + /** [[extractMetrics]], each metric's `value` read by `reading` (SCHEMA-2b, B3 / B4) AFTER the + * empty-input and no-value arms, which stay first: Elasticsearch 8's typed client has turned a + * `null` into `0.0` before any reader could see it. + */ + private[client] def extractMetricsWith( + aggsNode: JsonNode, + aggregations: ListMap[String, ClientAggregation], + docCount: Option[Long], + reading: AggregateReading ): ListMap[String, Any] = { aggsNode match { case n: ObjectNode => @@ -1779,7 +1840,13 @@ trait ElasticConversion { .filter(_ => !emptyInput) .filter(!_.isNull) .map { metricValue => - val numericValue = if (aggregations.get(name).exists(_.temporal)) { + // SCHEMA-2b (B3) -- by its column's type, exactly; a value the class does not hold + // unchanged is kept as stored, below + val reader = reading.metric(name) + val typed = if (reader eq null) AggregateReading.Unread else reader(metricValue) + val numericValue = if (AggregateReading.Unread != typed) { + typed + } else if (aggregations.get(name).exists(_.temporal)) { temporalValueOf(value, metricValue) } else if (metricValue.isIntegralNumber) { metricValue.asLong() @@ -1899,6 +1966,18 @@ trait ElasticConversion { aggsNode: JsonNode, fieldAliases: ListMap[String, String], aggregations: ListMap[String, ClientAggregation] + ): ListMap[String, Any] = + extractAllTopHitsWith(aggsNode, fieldAliases, aggregations, AggregateReading.Empty) + + /** [[extractAllTopHits]], each `FIRST_VALUE` / `LAST_VALUE` / `ARRAY_AGG` hit's aggregated column + * read by its column's reader (SCHEMA-2b, B5) -- before `distinct`, which then runs on canonical + * values. A ranking window's hits are read as today (internal, never projected). + */ + private[client] def extractAllTopHitsWith( + aggsNode: JsonNode, + fieldAliases: ListMap[String, String], + aggregations: ListMap[String, ClientAggregation], + reading: AggregateReading ): ListMap[String, Any] = { aggsNode match { case n: ObjectNode => @@ -1944,8 +2023,12 @@ trait ElasticConversion { val isRanking = agg.exists(_.ranking) def optionalHitId(hit: JsonNode): ListMap[String, Any] = if (includeDocumentId) extractHitId(hit) else ListMap.empty + // SCHEMA-2b (B5) -- the aggregated column's reader, looked up once per top_hits + val byField = if (isRanking) null else reading.top(topHitName) val processedHits = hits.map { hit => - val source = extractSource(hit, fieldAliases) + val source = + if (byField eq null) extractSource(hit, fieldAliases) + else readElement(hit, fieldAliases, byField) if (hasMultipleValues) { if (isRanking) { val metadata = extractHitId(hit) @@ -2011,6 +2094,18 @@ trait ElasticConversion { def extractGlobalMetrics( aggsNode: JsonNode, aggregations: ListMap[String, ClientAggregation] + ): ListMap[String, Any] = + extractGlobalMetricsWith(aggsNode, aggregations, AggregateReading.Empty) + + /** [[extractGlobalMetrics]], each `value` read by `reading` (SCHEMA-2b, B6), looked up by the + * normalized name, under today's key. A temporal metric without `value_as_string` keeps today's + * reading: this path has no no-value arm, and Elasticsearch 8's missing `0.0` must not become + * the epoch. + */ + private[client] def extractGlobalMetricsWith( + aggsNode: JsonNode, + aggregations: ListMap[String, ClientAggregation], + reading: AggregateReading ): ListMap[String, Any] = { aggsNode match { case n: ObjectNode => @@ -2028,14 +2123,25 @@ trait ElasticConversion { ) { val metricValue = value.get("value") if (!metricValue.isNull) { - val value = if (metricValue.isIntegralNumber) { + val reader = reading.metric(key) + val typed = + if ( + (reader eq null) || (aggregations.get(key).exists(_.temporal) && { + val text = value.get("value_as_string") + text == null || text.isNull + }) + ) AggregateReading.Unread + else reader(metricValue) + val read = if (AggregateReading.Unread != typed) { + typed + } else if (metricValue.isIntegralNumber) { metricValue.asLong() } else if (metricValue.isFloatingPointNumber) { metricValue.asDouble() } else { metricValue.asText() } - entries ++= Seq(name -> value) + entries ++= Seq(name -> read) } } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala index 0bcd8b8e..f9c9798f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query.{ + Bucket, Field, GroupedRowOrder, MultiSearch, @@ -60,7 +61,11 @@ private[client] object MultiValued { * NULL; * - `strict` / `keep`: the projection drops every undeclared key, but `keep` (D9); * - `star`: a bare `SELECT *` -- unmapped top-level keys dropped, a flat dotted key folded into - * its mapped parent (A-3). + * its mapped parent (A-3); + * - `instants`: SCHEMA-2b -- on the base side of a window statement, each partition column + * paired by instant (`PartitionKeys.instantColumns`): its hit `fields` entry (the + * `epoch_millis` doc value the base query asks) by the row key it is read under + * (`PartitionKeys.instantKey`), beside the column's stored `_source` value. * * A key the plan does not hold is read as it always was (`ElasticConversion.jsonNodeToAny`). Built * ONCE per statement: per row, only the readers run. @@ -74,7 +79,8 @@ private[client] final class HitReading( val strict: Boolean, val keep: Set[String], private[client] val star: Boolean, - private[client] val flatReaders: String => JsonNode => Any + private[client] val flatReaders: String => JsonNode => Any, + private[client] val instants: java.util.HashMap[String, String] ) { /** Whether a hit's `_ignored` must be read (D8): the plan reads some `_source` key as a date. */ @@ -104,6 +110,10 @@ private[client] object HitReading { private[client] def isStar(field: Field): Boolean = field.identifier.identifierName == "*" && field.identifier.functions.isEmpty + /** A computed SELECT item: read from the hit's `fields` by its output name (`scripted`). */ + private[client] def isComputed(field: Field): Boolean = + field.identifier.functions.nonEmpty || field.isScriptField + /** Thrown by a scalar reader under [[MultiValued.Fail]]; the hit loop names the document. */ private[client] final class MultiValuedCell(val column: String, val table: String) extends RuntimeException(column, null, false, false) @@ -114,11 +124,43 @@ private[client] object HitReading { s"declared as one value: declare it a list with ALTER TABLE ${cell.table} ALTER COLUMN " + s"${cell.column} TYPE ARRAY<...>" + /** A plan that reads nothing and projects STRICTLY, keeping `keep` (D9): the final projection of + * a window-enriched statement (SCHEMA-2b, B7 d), whose rows were read by their base query's + * plan. + */ + private[client] def projection(keep: Set[String]): HitReading = + new HitReading( + None, + new java.util.HashMap[String, JsonNode => Any](), + new java.util.HashMap[String, JsonNode => Any](), + Map.empty, + new java.util.HashMap[String, JsonNode => Any](), + strict = true, + keep, + star = false, + _ => null, + new java.util.HashMap[String, String]() + ) + + /** @param carried + * SCHEMA-2b (B7 a) -- the PARTITION BY levels of the window-enriched statement whose base + * query `single` is: each plain column of them is read by its type -- a `DATE` / `TIME` one + * only where it is exactly a day / a time of day ([[Readers.exact]], A-23) -- and kept by the + * strict projection, so the base row carries the partition key the window values are joined + * on. + * @param instants + * the carried columns paired by instant (`PartitionKeys.instantColumns`, the statement's one + * decision, taken as it is): each one's hit `fields` entry -- its `epoch_millis` doc value -- + * read under `PartitionKeys.instantKey` and kept by the strict projection; its `_source` value + * stays as stored. + */ def of( single: SingleSearch, dates: SetOperationValues.DateReading, multiValued: MultiValued = Default, - elements: Set[String] = Set.empty + elements: Set[String] = Set.empty, + carried: Seq[Bucket] = Nil, + instants: Set[String] = Set.empty ): HitReading = { // each item's and each mapped column's type, computed ONCE per statement: the schema and the // readers read the same value; so do the date formats and routes, built once by `dates` @@ -185,7 +227,7 @@ private[client] object HitReading { scala.collection.mutable.LinkedHashMap.empty[String, ListBuffer[(String, SetOperationType)]] fields.filterNot(isStar).foreach { f => val id = f.identifier - if (id.functions.nonEmpty || f.isScriptField) { + if (isComputed(f)) { val name = f.outputName val t = typeOf(f) if (name.nonEmpty && !scripted.containsKey(name)) { @@ -216,6 +258,30 @@ private[client] object HitReading { paths.foreach { case (parent, under) => if (!source.containsKey(parent)) source.put(parent, readers.paths(parent, under.toList)) } + // SCHEMA-2b (B7 a) -- each plain PARTITION BY column of the window statement: read by the type + // its key is read with (`MultiSearch.bucketType`), a DATE / TIME one exactly (A-23), and kept + val carriedKeys: Seq[String] = + if (single.schema.isEmpty) Nil + else + carried.flatMap { b => + val id = b.identifier + val name = id.name + if (id.nested || id.functions.nonEmpty || name.isEmpty || name.indexOf('.') >= 0) None + else { + val t = MultiSearch.bucketType(b) + t.sqlType match { + case SQLTypes.Date | SQLTypes.Time => source.put(name, readers.exact(name, t)) + case _ => if (!source.containsKey(name)) put(name, t, keepUnder(name)) + } + Some(name) + } + } + // SCHEMA-2b -- each carried column paired by instant: its `epoch_millis` doc value, under its + // own row key (never over the stored value). `instants` is the statement's ONE decision + // (`PartitionKeys.instantColumns`), the window aggregation's too: taken as it is + val instantKeys = new java.util.HashMap[String, String]() + if (single.schema.isDefined) + instants.foreach(name => instantKeys.put(name, PartitionKeys.instantKey(name))) // UNNEST element fields, by inner hit; a column with an element conversion keeps its raw value val inner: Map[String, java.util.HashMap[String, JsonNode => Any]] = if (single.schema.isEmpty) Map.empty @@ -239,7 +305,8 @@ private[client] object HitReading { // `_id` when it is kept, and every inner hit the row explosion reads AFTER the scroll's // normalizer -- an UNNEST its WHERE filters per element included (`UnnestScope.rowExplosion`) val keep: Set[String] = Set(ElasticConversion.DocumentIdField) ++ - single.nestedHitsMappings.keySet ++ UnnestScope.rowExplosion(single).keySet + single.nestedHitsMappings.keySet ++ UnnestScope.rowExplosion(single).keySet ++ carriedKeys ++ + instantKeys.values().toArray(new Array[String](0)) // a flat dotted key folded under `*` (rare): its reader built once per path, on first use val flatCache = new java.util.concurrent.ConcurrentHashMap[String, JsonNode => Any]() val flatReaders: String => JsonNode => Any = path => { @@ -263,7 +330,8 @@ private[client] object HitReading { strict, keep, star && schema.isDefined && fields.forall(isStar), - flatReaders + flatReaders, + instantKeys ) } @@ -329,6 +397,50 @@ private[client] object HitReading { ) } + /** SCHEMA-2b (A-23) -- a `DATE` / `TIME` column read EXACTLY, as the key of its aggregation + * level is (`AggregateReading`): a value its format reads to an instant that is exactly a UTC + * day (`DATE`) or a time of day of 1970-01-01 (`TIME`) is that day / time; any other value as + * read (D6, `Unreadable.KeepAsStored`), so it pairs with a key read as it always was. Any + * other type -- and a date read as stored, or with no format -- is [[column]] 's reader. + */ + def exact(name: String, t: SetOperationType): JsonNode => Any = + (t.sqlType, t.format) match { + case (kind @ (SQLTypes.Date | SQLTypes.Time), Some(spec)) + if DateRoute.of(kind, t.format, dates) == DateRoute.Typed => + val instantOf = instants(spec) + scalar( + name, + node => { + val instant = instantOf(node) + val value = if (instant eq null) null else AggregateReading.exactly(kind, instant) + if (value != null) value else asRead(node) + } + ) + case _ => column(name, t, Set.empty)._1 + } + + /** The instant a date field's value is under its mapping format `spec` -- the format + * [[byFormat]] reads it with --, `null` where the format does not read it. + */ + private def instants(spec: String): JsonNode => Instant = { + val format = dates.format(spec).get + val fast = spec == app.softnetwork.elastic.sql.query.TemporalLiterals.DefaultDateFormat + node => { + val z = if (fast && node.isTextual) DefaultDates.read(node.asText()) else null + if (z ne null) z.toInstant + else { + val outcome = + if (node.isTextual) format.read(node.asText()) + else if (node.isNumber) format.readValue(node.numberValue()) + else MappedDateFormat.Rejected + outcome match { + case MappedDateFormat.Read(instant) => instant + case _ => null + } + } + } + } + /** A computed column's `fields` entry: Elasticsearch's array unwrapped (#418 part 1). */ def scripted(name: String, t: SetOperationType): JsonNode => Any = t.sqlType match { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 31425218..0f25b971 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -824,6 +824,19 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { sql = Some(baseQuery.sql) ) + // SCHEMA-2b (B7 a) -- the base rows read by their base query's plan, each PARTITION BY column + // carried by its type + val baseContext: ConversionContext = + SchemaColumns.ofWindowBase( + request, + baseQuery, + context, + this match { + case api: VersionApi => api.dateDialectVersion + case _ => None + } + ) + // Execute window aggregations first val windowCacheFuture: Future[ElasticResult[WindowCache]] = Future(executeWindowAggregations(request, aggRequest, aggQuery)) @@ -834,15 +847,19 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { // Determine if we should keep the document ID in the output val shouldKeepDocumentId = keepsDocumentId(outputFields) - // Built once per stream — never per row (#229) - val normalizeOutputRow = rowNormalizer(outputFields) + // Built once per stream — never per row (#229): the base side's partition keys (B8), its + // request -- every route's one, asking the doc values of the partition columns paired by + // instant -- and the final projection, which drops `_id` unless it is kept (B7 d) + val partitionKeys = PartitionKeys.of(request, baseContext) + val enrichedBaseQuery = windowBaseQuery(baseElasticQuery, baseContext) + val project = windowRowProjection(request, baseContext) Source .futureSource( windowCacheFuture.map { case ElasticSuccess(cache) => scrollWithMetrics( - baseElasticQuery, + enrichedBaseQuery, baseQuery.fieldAliases, baseQuery.sqlAggregations, // The base rows must carry `_id`: the ordinal lookup below matches each row to @@ -851,14 +868,12 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { baseQuery.sorts.nonEmpty, extractOutputFieldNames(baseQuery), baseQuery.nestedHitsMappings - ) + )(system, baseContext) .map { case (doc, metrics) => - val enrichedDoc = enrichDocumentWithWindowValues(doc, cache, request) - var normalizedDoc = normalizeOutputRow(enrichedDoc) - if (!shouldKeepDocumentId) { - normalizedDoc = normalizedDoc - ElasticConversion.DocumentIdField - } - (normalizedDoc, metrics) + ( + project(enrichDocumentWithWindowValues(doc, cache, request, partitionKeys)), + metrics + ) } case ElasticFailure(error) => diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index a2651ad5..d71d8b2d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -2044,15 +2044,26 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC logger.info(s"🪟 Detected ${request.windowFunctions.size} window functions") + // SCHEMA-2b (B7 a) -- the base rows read by their base query's plan, each PARTITION BY column + // carried by its type, so both sides of the join read a partition column by ONE rule + val baseQuery = createBaseQuery(request) + val baseContext = + SchemaColumns.ofWindowBase(request, baseQuery, context, setOperationVersion) + for { // Step 1: Execute window aggregations windowCache <- executeWindowAggregations(request) // Step 2: Execute base query (without window functions) - baseResponse <- executeBaseQuery(request) + baseResponse <- executeBaseQuery(baseQuery)(timestamp, baseContext) // Step 3: Enrich results - enrichedResponse <- enrichResponseWithWindowValues(baseResponse, windowCache, request) + enrichedResponse <- enrichResponseWithWindowValues( + baseResponse, + windowCache, + request, + baseContext + ) } yield enrichedResponse.copy(schema = statementSchema(request)) } @@ -2102,6 +2113,11 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC s"🔍 Executing window aggregation query:\n${aggRequest.sql}" ) + // SCHEMA-2b (B7 b, c) -- the window aggregation's own reading: its partition keys, metrics and + // top_hits values by type; a partition column paired by instant by its bucket's instant + val aggContext = + SchemaColumns.ofWindowAggregation(request, aggRequest, context, setOperationVersion) + for { // Use singleSearch to execute aggregation aggResponse <- singleSearch( @@ -2110,10 +2126,10 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC aggRequest.sqlAggregations, extractOutputFieldNames(aggRequest), aggRequest.nestedHitsMappings - ) + )(aggContext) // Parse aggregation results into cache - cache <- parseWindowAggregationsToCache(aggResponse, request) + cache <- parseWindowAggregationsToCache(aggResponse, request, aggContext) } yield cache } @@ -2144,7 +2160,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC */ private def parseWindowAggregationsToCache( response: ElasticResponse, - request: SingleSearch + request: SingleSearch, + context: ConversionContext ): ElasticResult[WindowCache] = { logger.info( @@ -2167,8 +2184,11 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC } } + // SCHEMA-2b (B8) -- the cache side's partition keys, built once + val partitionKeys = PartitionKeys.of(request, context) + val cache = aggRows.map { row => - val partitionKey = extractPartitionKey(row, request) + val partitionKey = PartitionKey(partitionKeys.of(row)) val windowValues = extractWindowValues(row, response.aggregations) val rankings = extractRankings(row, rankingWindows) partitionKey -> windowValues.copy(rankings = rankings) @@ -2228,23 +2248,25 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // BASE QUERY EXECUTION // ======================================================================== - /** Execute base query without window functions + /** Execute the base query (the statement without its window functions, `createBaseQuery`), its + * rows read in `context` (`SchemaColumns.ofWindowBase`). */ private def executeBaseQuery( - request: SingleSearch + baseQuery: SingleSearch )(implicit timestamp: Long, context: ConversionContext): ElasticResult[ElasticResponse] = { - val baseQuery = createBaseQuery(request) - logger.info(s"🔍 Executing base query without window functions ${baseQuery.sql}") // Retain `_id` on the parsed rows: the enrichment step matches each base row to its // ranking ordinals by document id. The id is stripped after enrichment. singleSearchInternal( - ElasticQuery( - baseQuery, - collection.immutable.Seq(baseQuery.sources: _*), - sql = Some(baseQuery.sql) + windowBaseQuery( + ElasticQuery( + baseQuery, + collection.immutable.Seq(baseQuery.sources: _*), + sql = Some(baseQuery.sql) + ), + context ), baseQuery.fieldAliases, baseQuery.sqlAggregations, @@ -2254,6 +2276,21 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC ) } + /** The search of a window statement's base query (`base`, as translated), its rows read in + * `baseContext` (`SchemaColumns.ofWindowBase`): every route's ONE request (one-shot, async, + * scroll), which also asks the `epoch_millis` doc value of each partition column paired by + * instant (`PartitionKeys.instantColumns`); `base` itself when none is. Built once per + * statement. + */ + private[client] def windowBaseQuery( + base: ElasticQuery, + baseContext: ConversionContext + ): ElasticQuery = { + val fields = SchemaColumns.instantFields(baseContext) + if (fields.isEmpty) base + else base.copy(query = PartitionKeys.withInstantDocValues(base.query, fields)) + } + /** Create base query by removing window functions from SELECT: `SingleSearch.withoutWindows`, the * ONE derivation the parse-time UNNEST rule judges too (`UnnestScope`), so the statement that is * judged and the statement that runs cannot diverge. @@ -2262,30 +2299,6 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC request: SingleSearch ): SingleSearch = request.withoutWindows - /** Extract partition key from aggregation row - */ - private def extractPartitionKey( - row: ListMap[String, Any], - request: SingleSearch - ): PartitionKey = { - - // Get all partition fields from window functions - val partitionFields = request.windowFunctions - .flatMap(_.partitionBy) - .map(_.aliasOrName) - .distinct - - if (partitionFields.isEmpty) { - return PartitionKey(ListMap("__global__" -> true)) - } - - val keyValues = partitionFields.flatMap { field => - row.get(field).map(field -> _) - } - - PartitionKey(ListMap(keyValues: _*)) - } - /** Extract window function values from aggregation row */ private def extractWindowValues( @@ -2307,38 +2320,80 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC private def enrichResponseWithWindowValues( response: ElasticResponse, cache: WindowCache, - request: SingleSearch - )(implicit context: ConversionContext): ElasticResult[ElasticResponse] = { + request: SingleSearch, + baseContext: ConversionContext + ): ElasticResult[ElasticResponse] = { val baseRows = response.results - val outputFields = extractOutputFieldNames(request) - - // Determine ONCE whether the document ID stays in the output — never per row - val shouldKeepDocumentId = keepsDocumentId(outputFields) - // Built once for the whole result set — never per row (#229) - val normalizeOutputRow = rowNormalizer(outputFields) + // Built once for the whole result set — never per row (#229): the base side's partition keys + // (B8) and the final projection (B7 d) + val partitionKeys = PartitionKeys.of(request, baseContext) + val project = windowRowProjection(request, baseContext) - // Enrich each row with window values, then normalize field order. The base rows carry - // their `_id` (see singleSearchInternal with retainDocumentId = true) for the ordinal - // lookup — strip it on the way out unless the document-id column is enabled or `_id` - // is selected. Only window-enriched rows ever pay this per-row strip. + // Enrich each row with window values, then project it. The base rows carry their `_id` (see + // singleSearchInternal with retainDocumentId = true) for the ordinal lookup — the projection + // drops it unless the document-id column is enabled or `_id` is selected. val enrichedRows = baseRows.map { row => - val enriched = enrichDocumentWithWindowValues(row, cache, request) - val normalized = normalizeOutputRow(enriched) - if (shouldKeepDocumentId) normalized - else normalized - ElasticConversion.DocumentIdField + project(enrichDocumentWithWindowValues(row, cache, request, partitionKeys)) } ElasticResult.success(response.copy(results = enrichedRows)) } + /** SCHEMA-2b (B7 d) -- the final projection of a window-enriched statement's rows, built once per + * statement: under its schema, STRICT over the statement as written (D9), `_id` kept only where + * the statement keeps it, so the partition columns its base rows carried are dropped after the + * join, in the projection's one rebuild; elsewhere today's normalization, then `_id` stripped -- + * and the doc values of the partition columns paired by instant, wherever the rows hold them. + */ + private[client] def windowRowProjection( + request: SingleSearch, + baseContext: ConversionContext + ): ListMap[String, Any] => ListMap[String, Any] = { + val outputFields = extractOutputFieldNames(request) + val keepId = keepsDocumentId(outputFields) + baseContext match { + case s: SchemaColumns + if s.reading.schema.isDefined && outputFields.nonEmpty && + !request.select.fieldsWithComputedAliases.exists(HitReading.isStar) => + rowNormalizer(outputFields)( + SchemaColumns( + HitReading.projection( + if (keepId) Set(ElasticConversion.DocumentIdField) else Set.empty + ) + )() + ) + case _ => + val normalize = rowNormalizer(outputFields)(baseContext) + val instants = SchemaColumns.instantRowKeys(baseContext) + if (instants.isEmpty) { + if (keepId) normalize + else row => normalize(row) - ElasticConversion.DocumentIdField + } else { + val internal = if (keepId) instants else ElasticConversion.DocumentIdField +: instants + row => normalize(row) -- internal + } + } + } + /** Enrich a single document with window values */ protected def enrichDocumentWithWindowValues( doc: ListMap[String, Any], cache: WindowCache, request: SingleSearch + ): ListMap[String, Any] = + enrichDocumentWithWindowValues(doc, cache, request, PartitionKeys.of(request, NativeContext)) + + /** [[enrichDocumentWithWindowValues]] over the base side's partition keys (SCHEMA-2b, B8), built + * once per statement. + */ + private[client] def enrichDocumentWithWindowValues( + doc: ListMap[String, Any], + cache: WindowCache, + request: SingleSearch, + partitionKeys: PartitionKeys ): ListMap[String, Any] = { if (request.windowFunctions.isEmpty) { @@ -2346,7 +2401,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC } // Build partition key from document - val partitionKey = extractPartitionKey(doc, request) + val partitionKey = PartitionKey(partitionKeys.of(doc)) val rowId = doc.get("_id").map(_.toString).getOrElse("") val rankingAliases: Seq[String] = diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index 1a24aaa1..f38298c1 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -619,12 +619,18 @@ package object client extends SerializationApi { * `doubleKeys` names the levels the statement sends as a numeric `terms` of doubles * (`SingleSearch.doubleKeyLevels`), whose non-finite keys a response writes as text * (`ElasticConversion.extractBucketKey`). + * + * `aggregates` (SCHEMA-2b, B1 / A-22) reads the statement's aggregation response by type: its + * bucket keys, metrics and top_hits values ([[AggregateReading]]). Built only by + * [[SchemaColumns.of]]; a context that does not carry it reads them as 0.24.0 did. */ private[client] final case class SchemaColumns( reading: HitReading, doubleKeys: Set[String] = Set.empty - )(val elements: Map[String, Any => Any] = Map.empty) - extends ConversionContext { + )( + val elements: Map[String, Any => Any] = Map.empty, + val aggregates: AggregateReading = AggregateReading.Empty + ) extends ConversionContext { override private[client] def elementConversions: Map[String, Any => Any] = elements } @@ -639,6 +645,15 @@ package object client extends SerializationApi { single: SingleSearch, context: ConversionContext, version: => Option[String] = None + ): ConversionContext = + read(single, context, version, _ => Set.empty) + + /** [[of]], the keys of the levels `instants` names read as their buckets' instants. */ + private def read( + single: SingleSearch, + context: ConversionContext, + version: => Option[String], + instants: SetOperationValues.DateReading => Set[String] ): ConversionContext = context match { case NativeContext => @@ -653,10 +668,76 @@ package object client extends SerializationApi { elements.keySet ), single.doubleKeyLevels + )(elements, AggregateReading.of(single, dates, instants(dates))) + case other => other + } + + /** SCHEMA-2b (B7 a) -- `context` for the rows of `base`, the base query of the window-enriched + * statement `request` (`SearchApi.createBaseQuery`): [[of]] 's reading, with each plain + * PARTITION BY column of `request` read by its type and carried through the projection + * (`HitReading.of(carried)`), so the base row holds the partition key the window values are + * joined on -- a column paired by instant (`PartitionKeys.instantColumns`, the set + * [[ofWindowAggregation]] reads too) by its `epoch_millis` doc value too + * (`HitReading.of(instants)`, which the base request asks); `context` itself when it is not + * native. + */ + def ofWindowBase( + request: SingleSearch, + base: SingleSearch, + context: ConversionContext, + version: => Option[String] + ): ConversionContext = + context match { + case NativeContext => + val dates = SetOperationValues.DateReading(version) + val elements = ElementConversions.of(base, dates) + SchemaColumns( + HitReading.of( + base, + dates, + HitReading.Default, + elements.keySet, + request.windowFunctions.flatMap(_.buckets), + PartitionKeys.instantColumns(request, dates) + ), + base.doubleKeyLevels )(elements) case other => other } + /** SCHEMA-2b -- `context` for the response of `aggRequest`, the window aggregation request of + * the window-enriched statement `request`: [[of]] 's reading, the key of each partition column + * paired by instant (`PartitionKeys.instantColumns`, the set [[ofWindowBase]] reads too) read + * as the bucket's instant; `context` itself when it is not native. + */ + def ofWindowAggregation( + request: SingleSearch, + aggRequest: SingleSearch, + context: ConversionContext, + version: => Option[String] + ): ConversionContext = + read(aggRequest, context, version, dates => PartitionKeys.instantColumns(request, dates)) + + /** The partition columns' doc values the base query of a window statement read in `context` + * asks (`PartitionKeys.withInstantDocValues`), sorted; none elsewhere. + */ + def instantFields(context: ConversionContext): Seq[String] = + context match { + case s: SchemaColumns if !s.reading.instants.isEmpty => + s.reading.instants.keySet().toArray(new Array[String](0)).toSeq.sorted + case _ => Nil + } + + /** The row keys of those doc values (`PartitionKeys.instantKey`): dropped by the final + * projection; none elsewhere. + */ + def instantRowKeys(context: ConversionContext): Seq[String] = + context match { + case s: SchemaColumns if !s.reading.instants.isEmpty => + s.reading.instants.values().toArray(new Array[String](0)).toSeq + case _ => Nil + } + /** The schema a response read in `context` carries (D13). */ def schemaOf(context: ConversionContext): Option[ResultSchema] = context match { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AggregateReadingSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AggregateReadingSpec.scala new file mode 100644 index 00000000..2c145a94 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/AggregateReadingSpec.scala @@ -0,0 +1,387 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.SingleSearch +import com.fasterxml.jackson.databind.node.ObjectNode +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.{LocalDate, LocalTime, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap + +/** SCHEMA-2b (B1-B6) -- an aggregation response read by its statement's declared types: bucket keys + * (B2), metrics exactly (B3 / B4), top_hits values (B5) and Case 4's global metrics (B6), on the + * canned pages each client form answers (B13: Elasticsearch 6 / 7's raw JSON, the typed tree of + * Elasticsearch 8 / 9 whose metrics are doubles). Network-free. + */ +class AggregateReadingSpec extends AnyFlatSpec with Matchers { + + private val version = "8.18.3" + + private val properties = + """{"b": {"type": "boolean"}, "dc": {"type": "date", "format": "yyyy/MM/dd"}, + | "c_des": {"type": "date", "format": "epoch_second"}, "big": {"type": "long"}, + | "n": {"type": "integer"}, "k": {"type": "keyword"}, "x": {"type": "double"}, + | "r": {"type": "float"}, "pt": {"type": "date"}, "tm": {"type": "date"}, + | "id": {"type": "keyword"}, + | "children": {"type": "nested", "properties": {"name": {"type": "keyword"}, + | "birthDate": {"type": "date"}}}}""".stripMargin + + private val schema = + Index( + "s2b_t", + s"""{"s2b_t": {"aliases": {}, "mappings": {"_meta": {"columns": { + | "c_des": {"data_type": "DATE"}, "tm": {"data_type": "TIME"}}}, + | "properties": $properties}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + ).schema + + private def statement(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(schema)) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + private def clientAggregations(single: SingleSearch): ListMap[String, ClientAggregation] = + ListMap(single.sqlAggregations.toSeq.map { case (k, v) => + k -> sqlAggregationToClientAggregation(v) + }: _*) + + private def contextOf(single: SingleSearch): ConversionContext = + SchemaColumns.of(single, NativeContext, Some(version)) + + /** The rows `json` answers for `sql`, read in the statement's schema context. */ + private def rows(sql: String, json: String): Seq[ListMap[String, Any]] = { + val single = statement(sql) + ElasticConversion + .parseResponse( + json, + single.fieldAliases, + clientAggregations(single), + single.select.fieldsWithComputedAliases.map(_.outputName) + )(contextOf(single)) + .get + } + + private def grouped(sql: String, level: String, buckets: String): Seq[ListMap[String, Any]] = + rows( + sql, + s"""{"hits": {"total": {"value": 9, "relation": "eq"}, "hits": []}, + | "aggregations": {"$level": {"buckets": [$buckets]}}}""".stripMargin + ) + + private def global(sql: String, aggregations: String, total: Int = 9): ListMap[String, Any] = + rows( + sql, + s"""{"hits": {"total": {"value": $total, "relation": "eq"}, "hits": []}, + | "aggregations": {$aggregations}}""".stripMargin + ) match { + case Seq(one) => one + case other => fail(s"[$sql] answered ${other.size} rows: $other") + } + + private def utc(y: Int, mo: Int, d: Int, h: Int = 0, mi: Int = 0): ZonedDateTime = + ZonedDateTime.of(y, mo, d, h, mi, 0, 0, ZoneOffset.UTC) + + private def classOf(v: Any): String = if (v == null) "null" else v.getClass.getName + + // ----------------------------------------------------------------------------------------------- + // Keys (B2) -- Table A rows 20a-20f + // ----------------------------------------------------------------------------------------------- + + behavior of "a bucket key read by its level's declared type (B2)" + + it should "read a BOOLEAN key 1 / 0 as a Boolean (20a)" in { + val rs = grouped( + "SELECT b, COUNT(*) AS c FROM s2b_t GROUP BY b", + "b", + """{"key": 1, "key_as_string": "true", "doc_count": 2}, + |{"key": 0, "key_as_string": "false", "doc_count": 1}""".stripMargin + ) + rs.map(_("b")) shouldBe Seq(java.lang.Boolean.TRUE, java.lang.Boolean.FALSE) + } + + it should "read a custom-format date key from its epoch millis, at Z (20b)" in { + val rs = grouped( + "SELECT dc, COUNT(*) AS c FROM s2b_t GROUP BY dc", + "dc", + """{"key": 1706659200000, "key_as_string": "2024/01/31", "doc_count": 1}""" + ) + rs.head("dc") shouldBe utc(2024, 1, 31) + rs.head("dc").asInstanceOf[ZonedDateTime].getZone shouldBe ZoneOffset.UTC + } + + it should "read a BIGINT key as a Long, never a 10^12 date guess (20c)" in { + val rs = grouped( + "SELECT big, COUNT(*) AS c FROM s2b_t GROUP BY big", + "big", + """{"key": 42, "doc_count": 1}, {"key": 1706697000000, "doc_count": 1}""" + ) + rs.map(_("big")) shouldBe Seq( + java.lang.Long.valueOf(42L), + java.lang.Long.valueOf(1706697000000L) + ) + rs.map(r => classOf(r("big"))).distinct shouldBe Seq("java.lang.Long") + } + + it should "read a DATE key exactly: a whole day a LocalDate, 10:30 as main reads it (20d, A-25)" in { + val rs = grouped( + "SELECT c_des, COUNT(*) AS c FROM s2b_t GROUP BY c_des", + "c_des", + """{"key": 1706697000000, "key_as_string": "1706697000", "doc_count": 1}, + |{"key": 1706659200000, "key_as_string": "1706659200", "doc_count": 1}""".stripMargin + ) + rs.head("c_des") shouldBe ElasticConversion.extractBucketKey( + mapper.readTree("""{"key": 1706697000000, "key_as_string": "1706697000"}""") + ) + rs(1)("c_des") shouldBe LocalDate.of(2024, 1, 31) + } + + it should "read an INT key as an Integer (20e) and a keyword key as its text (20f)" in { + grouped("SELECT n, COUNT(*) AS c FROM s2b_t GROUP BY n", "n", """{"key": 5, "doc_count": 1}""") + .head("n") shouldBe java.lang.Integer.valueOf(5) + grouped( + "SELECT k, COUNT(*) AS c FROM s2b_t GROUP BY k", + "k", + """{"key": "10:30", "doc_count": 1}, {"key": "2024-01-31", "doc_count": 1}""" + ).map(_("k")) shouldBe Seq("10:30", "2024-01-31") + } + + it should "read a TIME key only on 1970-01-01 and a DOUBLE key's non-finite text as the Double" in { + val times = grouped( + "SELECT tm, COUNT(*) AS c FROM s2b_t GROUP BY tm", + "tm", + """{"key": 37800000, "key_as_string": "1970-01-01T10:30:00.000Z", "doc_count": 1}, + |{"key": 1706697000000, "key_as_string": "2024-01-31T10:30:00.000Z", "doc_count": 1}""".stripMargin + ) + times.head("tm") shouldBe LocalTime.of(10, 30) + // not a time of day: today's reading + times(1)("tm") shouldBe ElasticConversion.tryParseAsDateTime("2024-01-31T10:30:00.000Z").get + grouped( + "SELECT x, COUNT(*) AS c FROM s2b_t GROUP BY x", + "x", + """{"key": 2.5, "doc_count": 1}, {"key": "Infinity", "doc_count": 1}""" + ).map(_("x")) shouldBe Seq(java.lang.Double.valueOf(2.5), java.lang.Double.POSITIVE_INFINITY) + } + + it should "keep a key no typed reader reads as main reads it (A-12)" in { + grouped( + "SELECT n, COUNT(*) AS c FROM s2b_t GROUP BY n", + "n", + """{"key": 3000000000, "doc_count": 1}""" + ).head("n") shouldBe 3000000000L + } + + // ----------------------------------------------------------------------------------------------- + // Metrics (B3 / B4) -- Table A rows 21a-21d + // ----------------------------------------------------------------------------------------------- + + behavior of "a metric read by its column's declared type, exactly (B3 / B4)" + + it should "read a temporal MAX from its epoch millis at Z, never value_as_string (21a)" in { + val row = global( + "SELECT MAX(dc) AS m FROM s2b_t", + """"m": {"value": 1.7067456E12, "value_as_string": "2024/02/01"}""" + ) + row("m") shouldBe utc(2024, 2, 1) + row("m").asInstanceOf[ZonedDateTime].getZone shouldBe ZoneOffset.UTC + } + + it should "read SUM over an INT as a Long (21b, A-20), MAX as an Integer (21c), COUNT as a Long (21d)" in { + val row = global( + "SELECT SUM(n) AS s, MAX(n) AS m, COUNT(*) AS c, COUNT(n) AS cn FROM s2b_t", + """"s": {"value": 15.0}, "m": {"value": 10.0}, "c": {"value": 3.0}, "cn": {"value": 3}""" + ) + row("s") shouldBe java.lang.Long.valueOf(15L) + row("m") shouldBe java.lang.Integer.valueOf(10) + row("c") shouldBe java.lang.Long.valueOf(3L) + row("cn") shouldBe java.lang.Long.valueOf(3L) + Seq("s", "m", "c", "cn").map(k => classOf(row(k))) shouldBe + Seq("java.lang.Long", "java.lang.Integer", "java.lang.Long", "java.lang.Long") + } + + it should "read a SUM beyond the INT range as a Long (T0-4)" in { + global("SELECT SUM(n) AS s FROM s2b_t", """"s": {"value": 6.0E9}""")("s") shouldBe + java.lang.Long.valueOf(6000000000L) + } + + it should "keep a value its class does not hold unchanged as stored, never failing" in { + // a fractional SUM (a scripted operand), a SUM beyond Long, a MAX beyond INT + global("SELECT SUM(n) AS s FROM s2b_t", """"s": {"value": 2.5}""")("s") shouldBe 2.5 + global("SELECT SUM(n) AS s FROM s2b_t", """"s": {"value": 1.0E19}""")("s") shouldBe 1.0e19 + global("SELECT MAX(n) AS m FROM s2b_t", """"m": {"value": 3.0E9}""")("m") shouldBe 3.0e9 + // a non-finite value (the typed tree of Elasticsearch 8 holds it as a double) + val single = statement("SELECT SUM(n) AS s FROM s2b_t") + val json = mapper + .readTree("""{"hits": {"total": 9, "hits": []}, "aggregations": {"s": {"value": 0.0}}}""") + json.path("aggregations").path("s").asInstanceOf[ObjectNode].put("value", Double.NaN) + val nan = ElasticConversion + .parseResponseTree(json, single.fieldAliases, clientAggregations(single), Seq("s"))( + contextOf(single) + ) + .get + .head("s") + nan.asInstanceOf[Double].isNaN shouldBe true + } + + it should "read DOUBLE / REAL / BOOLEAN metrics in their classes" in { + val row = global( + "SELECT MAX(x) AS mx, MIN(r) AS mr, MAX(b) AS mb, AVG(n) AS a FROM s2b_t", + """"mx": {"value": 2.5}, "mr": {"value": 1.5}, "mb": {"value": 1.0, "value_as_string": "true"}, + | "a": {"value": 5.0}""".stripMargin + ) + row("mx") shouldBe java.lang.Double.valueOf(2.5) + row("mr") shouldBe java.lang.Float.valueOf(1.5f) + row("mb") shouldBe java.lang.Boolean.TRUE + row("a") shouldBe java.lang.Double.valueOf(5.0) + } + + it should "answer an empty input before any reader: COUNT 0, SUM 0, MIN NULL (ES 8's 0.0 included)" in { + val row = global( + "SELECT COUNT(*) AS c, SUM(n) AS s, MIN(n) AS m FROM s2b_t", + """"c": {"value": 0.0}, "s": {"value": 0.0}, "m": {"value": 0.0}""", + total = 0 + ) + row("c") shouldBe java.lang.Long.valueOf(0L) + row("s") shouldBe java.lang.Long.valueOf(0L) + row("m") shouldBe (null: Any) + } + + it should "read a temporal bucket_script from its epoch millis (MAX(dc) + 1)" in { + val rs = grouped( + "SELECT k, MAX(dc) + 1 AS d FROM s2b_t GROUP BY k", + "k", + """{"key": "a", "doc_count": 1, "max_dc": {"value": 1.7066592E12, "value_as_string": "2024/01/31"}, + | "d": {"value": 1.7067456E12}}""".stripMargin + ) + rs.head("d") shouldBe utc(2024, 2, 1) + } + + // ----------------------------------------------------------------------------------------------- + // top_hits (B5), Case 4 (B6) + // ----------------------------------------------------------------------------------------------- + + behavior of "a top_hits value read by its aggregated column's reader (B5)" + + it should "read FIRST / LAST and ARRAY_AGG (DISTINCT) values canonically" in { + val row = global( + "SELECT FIRST(pt) AS f, ARRAY_AGG(DISTINCT pt) AS a FROM s2b_t", + """"f": {"hits": {"hits": [{"_source": {"pt": "2024-01-31"}}]}}, + |"a": {"hits": {"hits": [{"_source": {"pt": "2024-01-31"}}, + | {"_source": {"pt": "2024-01-31T00:00:00.000Z"}}, {"_source": {"pt": "2024-02-01"}}]}}""".stripMargin + ) + row("f") match { + case m: scala.collection.Map[_, _] => + m.asInstanceOf[scala.collection.Map[String, Any]]("pt") shouldBe utc(2024, 1, 31) + case other => fail(s"FIRST answered $other") + } + // the same instant spelled twice is ONE canonical value + row("a") shouldBe List(utc(2024, 1, 31), utc(2024, 2, 1)) + } + + it should "read a qualified column's top_hits value (FIRST(p.pt) FROM t p)" in { + val row = global( + "SELECT FIRST(p.pt) AS f FROM s2b_t p", + """"f": {"hits": {"hits": [{"_source": {"pt": "2024-01-31"}}]}}""" + ) + row("f") match { + case m: scala.collection.Map[_, _] => + m.asInstanceOf[scala.collection.Map[String, Any]]("pt") shouldBe utc(2024, 1, 31) + case other => fail(s"FIRST answered $other") + } + } + + it should "read a nested (UNNEST) top_hits value relative to its nested object" in { + val rs = rows( + "SELECT ARRAY_AGG(DISTINCT child.birthDate) AS birthDates FROM s2b_t p JOIN UNNEST(p.children) AS child LIMIT 10", + """{"hits": {"total": 9, "hits": []}, "aggregations": {"children": {"doc_count": 2, + | "birthDates": {"hits": {"hits": [{"_source": {"birthDate": "1999-05-09"}}, + | {"_source": {"birthDate": "2002-05-09"}}]}}}}}""".stripMargin + ) + rs.flatMap(_.get("birthDates")).head shouldBe List(utc(1999, 5, 9), utc(2002, 5, 9)) + } + + behavior of "Case 4's global metrics (B6)" + + it should "read a global metric by type under today's key, a temporal one without value_as_string as today" in { + val single = + statement("SELECT id, COUNT(*) OVER () AS c, MAX(pt) OVER () AS m FROM s2b_t LIMIT 5") + val ctx = contextOf(single) + val aggs = clientAggregations(single) + def read(m: String): ListMap[String, Any] = + ElasticConversion + .parseResponse( + s"""{"hits": {"total": 1, "hits": [{"_id": "1", "_source": {"id": "1"}}]}, + | "aggregations": {"c": {"value": 1.0}, "m": $m}}""".stripMargin, + single.fieldAliases, + aggs, + single.select.fieldsWithComputedAliases.map(_.outputName) + )(ctx) + .get + .head + val typed = read("""{"value": 1.7066592E12, "value_as_string": "2024-01-31T00:00:00.000Z"}""") + typed("c") shouldBe java.lang.Long.valueOf(1L) + typed("m") shouldBe utc(2024, 1, 31) + // ES 8's missing 0.0 must not become the epoch + read("""{"value": 0.0}""")("m") shouldBe 0.0 + } + + // ----------------------------------------------------------------------------------------------- + // The carrier (A-22) + // ----------------------------------------------------------------------------------------------- + + behavior of "the carrier (A-22)" + + it should "be built by SchemaColumns.of for a native statement with aggregations, Empty elsewhere" in { + val single = statement("SELECT k, SUM(n) AS s FROM s2b_t GROUP BY k") + SchemaColumns.of(single, NativeContext, Some(version)) match { + case s: SchemaColumns => + s.aggregates.isEmpty shouldBe false + s.aggregates.key("k") should not be null + s.aggregates.metric("s") should not be null + case other => fail(s"$other") + } + // a subquery's inner statement (B11), and a statement with no schema (D10) + SchemaColumns.of(single, EntityContext, Some(version)) shouldBe EntityContext + val unmapped = Parser("SELECT k, SUM(n) AS s FROM s2b_t GROUP BY k") match { + case Right(s: SingleSearch) => s + case other => fail(s"$other") + } + AggregateReading + .of(unmapped, SetOperationValues.DateReading(Some(version))) + .isEmpty shouldBe true + } + + it should "leave the public parsers on 0.24.0's reading" in { + val single = statement("SELECT b, MAX(n) AS m FROM s2b_t GROUP BY b") + val node = mapper.readTree( + """{"b": {"buckets": [{"key": 1, "key_as_string": "true", "doc_count": 1, "m": {"value": 10.0}}]}}""" + ) + val parsed = + ElasticConversion.parseAggregations( + node, + ListMap.empty, + single.fieldAliases, + clientAggregations(single) + ) + parsed.head("b") shouldBe ElasticConversion.extractBucketKey( + mapper.readTree("""{"key": 1, "key_as_string": "true"}""") + ) + parsed.head("m") shouldBe 10.0 + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/WindowPartitionKeySpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/WindowPartitionKeySpec.scala new file mode 100644 index 00000000..d8312b23 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/WindowPartitionKeySpec.scala @@ -0,0 +1,448 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.SingleSearch +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.time.{LocalDate, LocalTime, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap + +/** SCHEMA-2b (B7 / B8, A-23) -- both sides of a window statement's join read a partition column by + * ONE rule: the base row's value (`SchemaColumns.ofWindowBase`) and the window aggregation's key + * (`SchemaColumns.ofWindowAggregation`) give equal partition keys, with equal hashes, for every + * partition column type of fixture `s2b_w`; a DATE value that is not exactly a day pairs as on + * main and shares no key with a day; a date read as stored pairs by instant. Network-free (canned + * pages). + */ +class WindowPartitionKeySpec extends AnyFlatSpec with Matchers { + + private val version = "8.18.3" + + private val properties = + """{"id": {"type": "keyword"}, "n": {"type": "integer"}, + | "pk1": {"type": "keyword"}, "pk2": {"type": "keyword"}, "pu": {"type": "keyword"}, + | "pt": {"type": "date"}, "pb": {"type": "boolean"}, + | "pd": {"type": "date", "format": "epoch_second"}, "pe": {"type": "date"}, + | "pz": {"type": "date", "format": "yyyy-MM-dd HH:mm z"}, "pn": {"type": "integer"}, + | "tm": {"type": "date"}}""".stripMargin + + private val schema = + Index( + "s2b_w", + s"""{"s2b_w": {"aliases": {}, "mappings": {"_meta": {"columns": { + | "pd": {"data_type": "DATE"}, "pe": {"data_type": "DATE"}, "tm": {"data_type": "TIME"}}}, + | "properties": $properties}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + ).schema + + private def statement(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(schema)) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + private def windowStatement(p: String): SingleSearch = + statement( + s"SELECT id, n, $p, ROW_NUMBER() OVER (PARTITION BY $p ORDER BY id) AS rn, " + + s"SUM(n) OVER (PARTITION BY $p) AS s FROM s2b_w" + ) + + /** The window aggregation request, as `SearchApi.buildWindowAggregationRequest` derives it. */ + private def aggregationRequest(request: SingleSearch): SingleSearch = + request + .copy( + select = request.select.copy(fields = request.windowFields.map(_.update(request))), + groupBy = None, + orderBy = None, + limit = None + ) + .update() + + private def clientAggregations(single: SingleSearch): ListMap[String, ClientAggregation] = + ListMap(single.sqlAggregations.toSeq.map { case (k, v) => + k -> sqlAggregationToClientAggregation(v) + }: _*) + + private def outputs(single: SingleSearch): Seq[String] = + single.select.fieldsWithComputedAliases.map(_.outputName) + + /** The base row `_source` (and the hit's `fields`, when given) reads as, in `context`. */ + private def baseRow( + request: SingleSearch, + source: String, + context: ConversionContext = NativeContext, + fields: Option[String] = None + ): ListMap[String, Any] = { + val base = request.withoutWindows + val ctx = SchemaColumns.ofWindowBase(request, base, context, Some(version)) + ElasticConversion + .parseResponse( + s"""{"hits": {"total": 1, "hits": [{"_id": "1", "_source": $source""" + + fields.fold("")(f => s""", "fields": $f""") + "}]}}", + base.fieldAliases, + ListMap.empty, + outputs(base), + retainDocumentId = true + )(ctx) + .get + .head + } + + /** The base side: the partition key of the base row `_source` (and `fields`) reads as. */ + private def baseKey( + request: SingleSearch, + source: String, + context: ConversionContext = NativeContext, + fields: Option[String] = None + ): ListMap[String, Any] = { + val ctx = SchemaColumns.ofWindowBase(request, request.withoutWindows, context, Some(version)) + PartitionKeys.of(request, ctx).of(baseRow(request, source, context, fields)) + } + + /** The cache side: the partition key of the window aggregation's bucket `bucket` of level `p`. */ + private def cacheKey( + request: SingleSearch, + p: String, + bucket: String, + context: ConversionContext = NativeContext + ): ListMap[String, Any] = { + val aggRequest = aggregationRequest(request) + val ctx = SchemaColumns.ofWindowAggregation(request, aggRequest, context, Some(version)) + val row = ElasticConversion + .parseResponse( + s"""{"hits": {"total": 1, "hits": []}, "aggregations": {"$p": {"buckets": [$bucket]}}}""", + aggRequest.fieldAliases, + clientAggregations(aggRequest), + outputs(aggRequest) + )(ctx) + .get + .head + PartitionKeys.of(request, ctx).of(row) + } + + private def samePartition(a: ListMap[String, Any], b: ListMap[String, Any]): Unit = { + a shouldBe b + a.hashCode shouldBe b.hashCode + } + + private def utc(y: Int, mo: Int, d: Int, h: Int = 0, mi: Int = 0, ms: Int = 0): ZonedDateTime = + ZonedDateTime.of(y, mo, d, h, mi, 0, ms * 1000000, ZoneOffset.UTC) + + behavior of "a window partition key" + + it should "pair a keyword spelled as a time or a date by its text (pk1, pk2)" in { + val r1 = windowStatement("pk1") + val k1 = baseKey(r1, """{"id": "1", "n": 1, "pk1": "10:30"}""") + k1 shouldBe ListMap("pk1" -> "10:30") + samePartition( + k1, + cacheKey(r1, "pk1", """{"key": "10:30", "doc_count": 1, "s": {"value": 1.0}}""") + ) + val r2 = windowStatement("pk2") + samePartition( + baseKey(r2, """{"id": "1", "n": 1, "pk2": "2024-01-31"}"""), + cacheKey(r2, "pk2", """{"key": "2024-01-31", "doc_count": 1, "s": {"value": 1.0}}""") + ) + } + + it should "pair a BOOLEAN, an INT and a TIMESTAMP with sub-millisecond text (pb, pn, pt)" in { + val rb = windowStatement("pb") + val kb = baseKey(rb, """{"id": "1", "n": 1, "pb": true}""") + kb shouldBe ListMap("pb" -> java.lang.Boolean.TRUE) + samePartition( + kb, + cacheKey( + rb, + "pb", + """{"key": 1, "key_as_string": "true", "doc_count": 1, "s": {"value": 1.0}}""" + ) + ) + val ri = windowStatement("pn") + samePartition( + baseKey(ri, """{"id": "1", "n": 1, "pn": 7}"""), + cacheKey(ri, "pn", """{"key": 7, "doc_count": 1, "s": {"value": 1.0}}""") + ) + val rt = windowStatement("pt") + val kt = baseKey(rt, """{"id": "1", "n": 1, "pt": "2024-01-31T10:30:00.123456Z"}""") + kt shouldBe ListMap("pt" -> utc(2024, 1, 31, 10, 30, 123)) + samePartition( + kt, + cacheKey( + rt, + "pt", + """{"key": 1706697000123, "key_as_string": "2024-01-31T10:30:00.123Z", "doc_count": 1, + | "s": {"value": 1.0}}""".stripMargin + ) + ) + } + + it should "pair a DATE exactly a day as a LocalDate, and a DATE not a day as main does (pd, pe)" in { + val rd = windowStatement("pd") + val day = baseKey(rd, """{"id": "1", "n": 1, "pd": 1706659200}""") + day shouldBe ListMap("pd" -> LocalDate.of(2024, 1, 31)) + samePartition( + day, + cacheKey( + rd, + "pd", + """{"key": 1706659200000, "key_as_string": "1706659200", "doc_count": 1, "s": {"value": 1.0}}""" + ) + ) + // 10:30 and 14:00: none pairs (as on main), and neither shares the day's key + Seq(1706697000L -> "1706697000", 1706709600L -> "1706709600").foreach { case (seconds, text) => + val b = baseKey(rd, s"""{"id": "1", "n": 1, "pd": $seconds}""") + val c = cacheKey( + rd, + "pd", + s"""{"key": ${seconds * 1000}, "key_as_string": "$text", "doc_count": 1, "s": {"value": 1.0}}""" + ) + b should not be c + b should not be day + c should not be day + } + val re = windowStatement("pe") + samePartition( + baseKey(re, """{"id": "1", "n": 1, "pe": "2024-01-31T00:00:00Z"}"""), + cacheKey( + re, + "pe", + """{"key": 1706659200000, "key_as_string": "2024-01-31T00:00:00.000Z", "doc_count": 1, + | "s": {"value": 1.0}}""".stripMargin + ) + ) + // 10:30: main pairs it (equal ZonedDateTimes), and so does the branch + val inexact = baseKey(re, """{"id": "1", "n": 1, "pe": "2024-01-31T10:30:00Z"}""") + samePartition( + inexact, + cacheKey( + re, + "pe", + """{"key": 1706697000000, "key_as_string": "2024-01-31T10:30:00.000Z", "doc_count": 1, + | "s": {"value": 1.0}}""".stripMargin + ) + ) + inexact should not be ListMap("pe" -> LocalDate.of(2024, 1, 31)) + } + + it should "pair a date read as stored (pz) by instant, as Elasticsearch buckets a date" in { + // 2024-01-31 10:30 UTC = 11:30 CET (CET = UTC+1 in January) = 1706697000000 ms: one instant, + // so one bucket, whose `key_as_string` Elasticsearch 8.18.3 spells `2024-01-31 10:30 Z` (its + // own DateFormatter / DocValueFormat run on the mapping's format) -- neither stored spelling. + // Both sides pair by the instant Elasticsearch supplies: the bucket's millisecond `key` and the + // column's `epoch_millis` doc value (the same formatters print `1706697000000` for both docs). + val rz = windowStatement("pz") + val bucket = + """{"key": 1706697000000, "key_as_string": "2024-01-31 10:30 Z", "doc_count": 2, + | "s": {"value": 3.0}}""".stripMargin + val instant = Some("""{"pz": ["1706697000000"]}""") + val utcDoc = + baseKey(rz, """{"id": "1", "n": 1, "pz": "2024-01-31 10:30 UTC"}""", fields = instant) + val cetDoc = + baseKey(rz, """{"id": "2", "n": 2, "pz": "2024-01-31 11:30 CET"}""", fields = instant) + utcDoc shouldBe ListMap("pz" -> "1706697000000") + samePartition(utcDoc, cacheKey(rz, "pz", bucket)) + samePartition(cetDoc, cacheKey(rz, "pz", bucket)) + // another instant is another partition + val w3 = baseKey( + rz, + """{"id": "3", "n": 4, "pz": "2024-02-01 00:00 UTC"}""", + fields = Some("""{"pz": ["1706745600000"]}""") + ) + w3 should not be utcDoc + samePartition( + w3, + cacheKey( + rz, + "pz", + """{"key": 1706745600000, "key_as_string": "2024-02-01 00:00 Z", "doc_count": 1, + | "s": {"value": 4.0}}""".stripMargin + ) + ) + // the displayed value stays the stored text; the instant has its own row key + val row = baseRow(rz, """{"id": "2", "n": 2, "pz": "2024-01-31 11:30 CET"}""", fields = instant) + row("pz") shouldBe "2024-01-31 11:30 CET" + row(PartitionKeys.instantKey("pz")) shouldBe "1706697000000" + // a document without the doc value (no `pz`) pairs with no bucket, as on main + baseKey(rz, """{"id": "4", "n": 8}""") shouldBe ListMap.empty + } + + it should "pair a TIME of day (tm)" in { + val rm = windowStatement("tm") + val time = baseKey(rm, """{"id": "1", "n": 1, "tm": "1970-01-01T10:30:00Z"}""") + time shouldBe ListMap("tm" -> LocalTime.of(10, 30)) + samePartition( + time, + cacheKey( + rm, + "tm", + """{"key": 37800000, "key_as_string": "1970-01-01T10:30:00.000Z", "doc_count": 1, + | "s": {"value": 1.0}}""".stripMargin + ) + ) + } + + it should "keep today's reading on both sides under EntityContext (a mode-W subquery)" in { + val rb = windowStatement("pb") + val base = baseKey(rb, """{"id": "1", "n": 1, "pb": true}""", EntityContext) + val cache = cacheKey( + rb, + "pb", + """{"key": 1, "key_as_string": "true", "doc_count": 1, "s": {"value": 1.0}}""", + EntityContext + ) + base shouldBe ListMap("pb" -> true) + // the key as 0.24.0 read it: `key_as_string` first + cache shouldBe ListMap("pb" -> "true") + } + + it should "carry an unselected partition column through the base projection when it is fetched" in { + val request = statement( + "SELECT UPPER(id) AS u, SUM(n) OVER (PARTITION BY pu) AS s FROM s2b_w" + ) + baseKey(request, """{"id": "a", "n": 1, "pu": "x"}""") shouldBe ListMap("pu" -> "x") + } + + behavior of "the final projection of a window statement (B7 d)" + + private val client = new NopeClientApi { + override protected def logger: Logger = LoggerFactory.getLogger(classOf[WindowPartitionKeySpec]) + } + + it should "drop the carried partition columns and `_id` in its one rebuild" in { + val request = statement("SELECT id, SUM(n) OVER (PARTITION BY pu) AS s FROM s2b_w") + val base = request.withoutWindows + val ctx = SchemaColumns.ofWindowBase(request, base, NativeContext, Some(version)) + val project = client.windowRowProjection(request, ctx) + project(ListMap("id" -> "a", "_id" -> "1", "pu" -> "x", "s" -> 3L)) shouldBe + ListMap("id" -> "a", "s" -> 3L) + // under EntityContext: today's normalization, then `_id` stripped + val legacy = client.windowRowProjection(request, EntityContext) + legacy(ListMap("id" -> "a", "_id" -> "1", "pu" -> "x", "s" -> 3L)) shouldBe + ListMap("id" -> "a", "s" -> 3L, "pu" -> "x") + } + + it should "drop the doc value of a partition column paired by instant, strict or under `*`" in { + val instant = PartitionKeys.instantKey("pz") + val strict = statement("SELECT id, SUM(n) OVER (PARTITION BY pz) AS s FROM s2b_w") + val strictCtx = + SchemaColumns.ofWindowBase(strict, strict.withoutWindows, NativeContext, Some(version)) + client.windowRowProjection(strict, strictCtx)( + ListMap("id" -> "a", "_id" -> "1", "pz" -> "2024-01-31 10:30 UTC", instant -> "1", "s" -> 3L) + ) shouldBe ListMap("id" -> "a", "s" -> 3L) + val star = statement("SELECT *, SUM(n) OVER (PARTITION BY pz) AS s FROM s2b_w") + val starCtx = + SchemaColumns.ofWindowBase(star, star.withoutWindows, NativeContext, Some(version)) + val projected = client.windowRowProjection(star, starCtx)( + ListMap("id" -> "a", "_id" -> "1", "pz" -> "2024-01-31 10:30 UTC", instant -> "1", "s" -> 3L) + ) + projected("pz") shouldBe "2024-01-31 10:30 UTC" + projected.keySet should not contain instant + } + + behavior of "the base query of a window statement" + + private val body = """{"query": {"match_all": {}}, "_source": {"includes": ["id"]}}""" + + private def baseQuery(request: SingleSearch, context: ConversionContext): String = { + val base = request.withoutWindows + val ctx = SchemaColumns.ofWindowBase(request, base, context, Some(version)) + client + .windowBaseQuery(ElasticQuery(body, Seq("s2b_w")), ctx) + .query + } + + it should "ask the epoch_millis doc value of a partition column paired by instant only" in { + val json = new com.fasterxml.jackson.databind.ObjectMapper() + json.readTree(baseQuery(windowStatement("pz"), NativeContext)) shouldBe json.readTree( + """{"query": {"match_all": {}}, "_source": {"includes": ["id"]}, + | "docvalue_fields": [{"field": "pz", "format": "epoch_millis"}]}""".stripMargin + ) + // every other partition kind keeps its request, byte for byte + Seq("pk1", "pt", "pb", "pd", "pe", "pn", "tm").foreach { p => + withClue(s"$p ") { + baseQuery(windowStatement(p), NativeContext) shouldBe + """{"query": {"match_all": {}}, "_source": {"includes": ["id"]}}""" + } + } + // a mode-W subquery keeps today's reading on both sides, and its request + baseQuery(windowStatement("pz"), EntityContext) shouldBe + """{"query": {"match_all": {}}, "_source": {"includes": ["id"]}}""" + // a body that already asks doc values keeps them + PartitionKeys.withInstantDocValues( + """{"docvalue_fields": [{"field": "x"}]}""", + Seq("pz") + ) shouldBe """{"docvalue_fields":[{"field":"x"},{"field":"pz","format":"epoch_millis"}]}""" + } + + it should "pair by instant on both sides through ONE function (instantColumns)" in { + val dates = SetOperationValues.DateReading(Some(version)) + PartitionKeys.instantColumns(windowStatement("pz"), dates) shouldBe Set("pz") + Seq("pk1", "pk2", "pt", "pb", "pd", "pe", "pn", "tm").foreach { p => + withClue(s"$p ") { PartitionKeys.instantColumns(windowStatement(p), dates) shouldBe empty } + } + } + + it should "give both sides one rule when a computed column takes the partition column's name" in { + val dates = SetOperationValues.DateReading(Some(version)) + val bucket = + """{"key": 1706697000000, "key_as_string": "2024-01-31 10:30 Z", "doc_count": 2, + | "s": {"value": 3.0}}""".stripMargin + val source = """{"id": "1", "n": 1, "pk1": "a", "pz": "2024-01-31 10:30 UTC"}""" + def instantsOf(request: SingleSearch): Set[String] = { + val ctx = + SchemaColumns.ofWindowBase(request, request.withoutWindows, NativeContext, Some(version)) + ctx match { + case s: SchemaColumns => s.reading.instants.keySet().toArray(new Array[String](0)).toSet + case other => fail(s"expected SchemaColumns, got $other") + } + } + // the item's reader reads the hit's `fields` entry `pz`, so a doc value asked there would be + // read as the item's value: neither side pairs by instant, the bucket keeps its + // `key_as_string`. The canned `fields` entry below stands for that reader; Elasticsearch + // 8.18.3 sends none for this shape (the item keeps the window's bucket, no script field), + // and the column reads NULL, as on 0.24.0 + Seq( + "CONCAT(pz, '') AS pz" -> "2024-01-31T10:30Z", + "UPPER(pk1) AS pz" -> "A" + ).foreach { case (item, computed) => + withClue(s"$item ") { + val r = statement(s"SELECT id, n, $item, SUM(n) OVER (PARTITION BY pz) AS s FROM s2b_w") + PartitionKeys.instantColumns(r, dates) shouldBe empty + instantsOf(r) shouldBe empty + baseQuery(r, NativeContext) shouldBe body + baseKey(r, source, fields = Some(s"""{"pz": ["$computed"]}""")) shouldBe + ListMap("pz" -> computed) + cacheKey(r, "pz", bucket) shouldBe ListMap("pz" -> "2024-01-31 10:30 Z") + } + } + // the plain column keeps pairing by instant, on both sides + val plain = statement("SELECT id, n, pz, SUM(n) OVER (PARTITION BY pz) AS s FROM s2b_w") + PartitionKeys.instantColumns(plain, dates) shouldBe Set("pz") + instantsOf(plain) shouldBe Set("pz") + val json = new com.fasterxml.jackson.databind.ObjectMapper() + json.readTree(baseQuery(plain, NativeContext)) shouldBe json.readTree( + """{"query": {"match_all": {}}, "_source": {"includes": ["id"]}, + | "docvalue_fields": [{"field": "pz", "format": "epoch_millis"}]}""".stripMargin + ) + val key = baseKey(plain, source, fields = Some("""{"pz": ["1706697000000"]}""")) + key shouldBe ListMap("pz" -> "1706697000000") + samePartition(key, cacheKey(plain, "pz", bucket)) + } +} diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md index f0ca70c4..57c8cebb 100644 --- a/documentation/client/gateway.md +++ b/documentation/client/gateway.md @@ -238,7 +238,33 @@ ignored (`ignore_malformed`) is NULL, and the other values of the same array kee integral column reads a fractional number or a numeric text as Elasticsearch indexed it, truncated toward zero. A value a typed column cannot read is answered as stored: a value Elasticsearch did not index (a number outside its type's range, a `long` text with surrounding spaces) is never narrowed. -Aggregation values and window-enriched statements keep their reading in this release. + +Aggregation values follow the same table: + +- A GROUP BY key is read by its column's type: a `boolean` key is a `Boolean`, a date key the + `java.time` value of its epoch milliseconds at UTC (whatever the mapping format), a `long` key a + `Long`, a keyword key its text (a keyword spelling a date or a time stays text). A `DATE` key that + is not exactly a day, or a `TIME` key that is not a time of day of 1970-01-01, keeps its stored + reading. +- A metric is read by its column's type, exactly: `COUNT` is a `Long` on every client, `MIN` / + `MAX` of an integer an `Integer` (a `Long` for a `long`), of a date its `java.time` value at UTC, + of a boolean a `Boolean`. A value the class cannot hold unchanged (a sum beyond `Long`, a + fractional result over an integer column) is answered as Elasticsearch computed it, never + rounded. +- `SUM` is declared `BIGINT` over an integer column of any width and `DOUBLE` over a `REAL` or a + `DOUBLE` column, as DuckDB widens a sum: `SELECT SUM(n)` over an `integer` field answers a + `Long`, a `CREATE TABLE ... AS SELECT SUM(n) ...` maps the column `long`. +- A `FIRST_VALUE` / `LAST_VALUE` / `ARRAY_AGG` value is read by its column's type (an + `ARRAY_AGG(DISTINCT d)` keeps one value per instant). +- A window-enriched statement (`... OVER (PARTITION BY p ...)`) reads its rows by type and + partitions on the same values: a `BOOLEAN`, `TIMESTAMP`, `BIGINT` or whole-day `DATE` partition + column is answered, and the rows hold only the SELECT's columns. A partition column the SELECT + does not project is read only when the whole document is fetched, as with `SELECT *`. + A `date` partition column (not `date_nanos`) read as its stored text (a format with a zone name) + is matched by its instant, as Elasticsearch buckets it, whether the SELECT projects it under its + own name or not at all; a projected one still shows its stored text. Projected under another + name (`pz AS pz2`), or shadowed by a computed column of the SELECT that takes its name, it is + not matched that way. --- diff --git a/documentation/sql/functions_aggregate.md b/documentation/sql/functions_aggregate.md index 9fbdd095..69e2fd0d 100644 --- a/documentation/sql/functions_aggregate.md +++ b/documentation/sql/functions_aggregate.md @@ -37,7 +37,9 @@ Aggregate functions perform calculations on sets of rows and return a single res **The type of an aggregate column** (*since `0.24.0`*) is the type the aggregate produces — what `DESCRIBE`, a `CREATE TABLE … AS SELECT` mapping, a materialized view and the JDBC / Arrow metadata declare: `COUNT` is `BIGINT`; `AVG`, the `STDDEV` / `VARIANCE` family and the percentiles are -`DOUBLE`; `SUM` keeps the summed column's numeric type; `MIN`, `MAX`, `FIRST_VALUE` and `LAST_VALUE` +`DOUBLE`; `SUM` is `BIGINT` over an integer column of any width and `DOUBLE` over a `REAL` or a +`DOUBLE` column (*since `0.25.0`*: it kept the summed column's type, which overflowed); `MIN`, `MAX`, +`FIRST_VALUE` and `LAST_VALUE` keep the type of the expression they aggregate (`MAX(YEAR(d))` is `BIGINT`, `MAX(CAST(ts AS DATE))` a `DATE`). The values are unchanged. `MIN` / `MAX` over `ROUND(x, n)` or a `DATEDIFF` report `NUMERIC`, which a `CREATE TABLE … AS SELECT` cannot map: it fails at index creation @@ -220,7 +222,9 @@ SUM(expr) OVER (PARTITION BY partition_expr, ...) --(since v0.14.0) ignored. **Output:** -- `NUMERIC` - Sum of values (same type as input, or promoted) +- `BIGINT` over an integer column (`TINYINT` to `BIGINT`), `DOUBLE` over a `REAL` or a `DOUBLE` + column (*since `0.25.0`*). A sum beyond the `BIGINT` range, or a fractional sum over an integer + expression, is answered as the double Elasticsearch computed. **NULL Handling:** - NULL values are ignored diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala index 5d7bce9a..cb355d3a 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala @@ -69,9 +69,12 @@ class JavaClientSchemaConsumerSpec extends AnyFlatSpecLike with ElasticDockerTes | "n": {"type": "integer"}, "bin": {"type": "binary"}, "k": {"type": "keyword"} |}}""".stripMargin + // SCHEMA-2b: `z` sums two INTs of 2,000,000,000 -- beyond the INT range (A-20) private val docs: Seq[(String, String)] = Seq( "c1" -> """{"id":"c1","ts":"2024-01-31T10:30:00","c_des":1706697000,"c_c1":"2024/01/31","n":1,"bin":"aGVsbG8=","k":"a"}""", - "c2" -> """{"id":"c2","ts":"2024-02-01T00:00:00Z","c_des":1706745600,"c_c1":"2024/02/01","n":2,"k":"b"}""" + "c2" -> """{"id":"c2","ts":"2024-02-01T00:00:00Z","c_des":1706745600,"c_c1":"2024/02/01","n":2,"k":"b"}""", + "c3" -> """{"id":"c3","n":2000000000,"k":"z"}""", + "c4" -> """{"id":"c4","n":2000000000,"k":"z"}""" ) override def beforeAll(): Unit = { @@ -202,4 +205,81 @@ class JavaClientSchemaConsumerSpec extends AnyFlatSpecLike with ElasticDockerTes } count.map(_.asInstanceOf[Number].longValue()) shouldBe Some(docs.size.toLong) } + + // --------------------------------------------------------------------------------------------- + // SCHEMA-2b -- the consumers' forms over aggregation rows + // --------------------------------------------------------------------------------------------- + + "Arrow" should "read aggregation rows through both match shapes, a SUM beyond INT a Long (A-20)" in { + val sql = s"SELECT k, SUM(n) AS s, COUNT(*) AS c, MAX(n) AS m FROM $table GROUP BY k" + val result = run(sql) + val (rows, schema) = rowsAndSchema(result) + extractRows(result).map(_.toMap) shouldBe rows.map(_.toMap) + val names = schema.getOrElse(fail("no schema")).names + names shouldBe Seq("k", "s", "c", "m") + rows.foreach(row => names.foreach(n => row.contains(n) shouldBe true)) + val z = rows.find(_("k") == "z").getOrElse(fail(s"no group z in $rows")) + z("s") shouldBe java.lang.Long.valueOf(4000000000L) + z("c") shouldBe java.lang.Long.valueOf(2L) + z("m") shouldBe Integer.valueOf(2000000000) + } + + it should "read an aggregation leg row through its leg readers unchanged (D12)" in { + val sql = + s"SELECT k, SUM(n) AS s FROM $table GROUP BY k UNION SELECT k, SUM(n) AS s FROM $table GROUP BY k" + val multiple = Parser(sql) match { + case Right(m: MultiSearch) => + m.copy(requests = m.requests.map(r => r.update(Some(client.loadSchema(table).get)))) + case other => fail(s"$other") + } + val legs = SetOperationValues + .legSchemas(multiple.requests, multiple.resolvedOperators, Some("8.18.3")) + .fold(refusal => fail(refusal), identity) + multiple.requests.zip(legs).foreach { case (leg, schema) => + val rows = extractRows(run(leg.sql)) + rows should not be empty + val names = leg.select.fieldsWithComputedAliases.map(_.outputName) + schema.foreach { s => + rows.foreach { row => + names.zip(s.slots).foreach { + case (name, Some(slot)) => + val cell = row.getOrElse(name, null) + slot.read(cell) shouldBe cell + case _ => + } + } + } + } + } + + "The JDBC driver" should "read aggregation rows by its match" in { + val rows = run(s"SELECT k, SUM(n) AS s FROM $table GROUP BY k") match { + case QueryRows(rows, _, _) => rows + case QueryStructured(response, _) => response.results + case QueryStream(stream, _, _) => + Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) + case other => fail(s"$other") + } + rows.map(r => ValueCoercion.coerceToString(r("k")) -> r("s")).toMap shouldBe Map( + "a" -> java.lang.Long.valueOf(1L), + "b" -> java.lang.Long.valueOf(2L), + "z" -> java.lang.Long.valueOf(4000000000L) + ) + } + + "The extensions" should "fold a GROUP BY count(*) row as they do" in { + val counts = run(s"SELECT k, count(*) as nb FROM $table GROUP BY k") match { + case QueryRows(rows, _, _) => rows + case QueryStructured(response, _) => response.results + case QueryStream(stream, _, _) => + Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) + case other => fail(s"$other") + } + // `MaterializedViewExtension.extractCount` keeps an Int / a Long / a String + counts.map(_("nb")).foreach { + case _: java.lang.Long | _: java.lang.Integer | _: String => + case other => fail(s"count(*) answered ${other.getClass.getName}") + } + counts.map(_("nb").asInstanceOf[Number].longValue()).sum shouldBe docs.size.toLong + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index e6c6faea..5f8e6e06 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -56,9 +56,12 @@ package object aggregate { * * - `COUNT` counts: BIGINT, whatever it counts. * - `AVG` averages: DOUBLE. - * - `SUM` adds: the summed column's numeric type (whole stays whole, fractional stays - * fractional); a sum of anything else is the DOUBLE Elasticsearch answers -- a sum of - * dates is not a date. + * - `SUM` adds, as DuckDB widens a sum: a `BIGINT` over any whole column (`TINYINT` to + * `BIGINT`) -- DuckDB's `HUGEINT`, of which a `BIGINT` holds every whole sum Elasticsearch + * computes below 2^63 --, a `DOUBLE` over a `REAL` or a `DOUBLE`; a sum of anything else + * is the DOUBLE Elasticsearch answers -- a sum of dates is not a date. The column's own + * width overflowed (`SUM(n)` of three integers of 2,000,000,000 is not an `INT`), and a + * `REAL` sum is computed by Elasticsearch as a double. * - `MIN`, `MAX`, `FIRST_VALUE`, `LAST_VALUE` return one of the values they aggregate: the * REPORTED type of the aggregated chain, so a DATE stays a DATE and `MAX(CAST(ts AS * DATE))` is a DATE. @@ -78,11 +81,9 @@ package object aggregate { case AVG | _: AvgAgg => SQLTypes.Double case SUM | _: SumAgg => aggregated match { - case whole @ (SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | - SQLTypes.BigInt) => - whole - case fractional @ (SQLTypes.Real | SQLTypes.Double) => fractional - case _ => SQLTypes.Double + case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | SQLTypes.BigInt => + SQLTypes.BigInt + case _ => SQLTypes.Double } case MIN | MAX | _: MinAgg | _: MaxAgg | _: FirstValue | _: LastValue => aggregated case other => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index b37c5317..59432bde 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -4080,8 +4080,10 @@ package object query { * `DATE` although Painless reads a `TIMESTAMP`), with DuckDB's reading of a whole number * literal -- an `INTEGER` when it fits one, as `SELECT 1` is. */ - private def declaredOperandType(field: Field): SQLType = { - val id = field.identifier + private def declaredOperandType(field: Field): SQLType = + declaredOperandType(field.identifier) + + private def declaredOperandType(id: Identifier): SQLType = { if (id.name.isEmpty) id.functions match { case (l: LongValue) :: Nil if l.value.isValidInt => SQLTypes.Int @@ -4141,15 +4143,27 @@ package object query { def at(t: SetOperationType): Operand = copy(valueType = t) } - private def operandOf(branch: Int, field: Field): Operand = { - val declaredType = declaredOperandType(field) - val column = field.identifier match { - case id: GenericIdentifier if id.functions.isEmpty => id.col - case _ => None + private def operandOf(branch: Int, field: Field): Operand = + Operand(branch, field, declaredOperandType(field), identifierType(field.identifier)) + + /** The type of ONE identifier as a set operation types a projection item of it: its declared + * type ([[declaredOperandType]]) at every depth, a bare column's date format included + * ([[positionType]]). + */ + private def identifierType(id: Identifier): SetOperationType = { + val column = id match { + case g: GenericIdentifier if g.functions.isEmpty => g.col + case _ => None } - Operand(branch, field, declaredType, positionType(declaredType, column)) + positionType(declaredOperandType(id), column) } + /** SCHEMA-2b (B2) -- the type of the key of the aggregation level `b` (a GROUP BY level, a + * window's PARTITION BY): the type a projection item of its identifier has ([[fieldType]]), so + * the key of a selected level and its column are typed by ONE rule (#292). + */ + private[elastic] def bucketType(b: Bucket): SetOperationType = identifierType(b.identifier) + /** SCHEMA-2 (D1) -- the type of ONE projection item, as a set operation types its position * ([[positionType]]): the result schema of a statement and the typing of its set-operation * positions are this one rule (#292), never two. diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala index a9dfe2ee..d5ed8392 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/AggregateReportedTypeSpec.scala @@ -108,32 +108,14 @@ class AggregateReportedTypeSpec extends AnyFlatSpec with Matchers { /** DuckDB refuses the input: it is not part of the type rule, so the cell is left out. */ private case object Refused extends Cell - private val wholeWidth: Map[SQLType, Int] = - Map(SQLTypes.TinyInt -> 1, SQLTypes.SmallInt -> 2, SQLTypes.Int -> 3, SQLTypes.BigInt -> 4) - - /** A whole SUM stays whole, at least as wide as the summed column. */ - private def wholeAtLeast(column: SQLType): SQLType => Unit = produced => - withClue(s"a SUM of $column is $produced: ") { - wholeWidth.get(produced).exists(_ >= wholeWidth(column)) shouldBe true - } - - /** A fractional SUM stays fractional. */ - private val fractional: SQLType => Unit = produced => - withClue(s"a SUM of a fractional column is $produced: ") { - Set[SQLType](SQLTypes.Real, SQLTypes.Double) should contain(produced) - } - - // 🔴 Open: core keeps the summed column's width, DuckDB widens a whole sum to HUGEINT and a REAL - // sum to DOUBLE. Which one core follows is still to be decided, so no cell pins a SUM's width. - private val sumWidthIsOpen = - "the width of a SUM is an open decision (core keeps the column's, DuckDB widens)" - private val count: Kind => Cell = _ => Pinned(SQLTypes.BigInt) + // SCHEMA-2b (A-20): a SUM widens as DuckDB widens it -- a whole sum is DuckDB's HUGEINT, of which + // core's BIGINT holds every whole sum Elasticsearch computes below 2^63, and a REAL sum a DOUBLE private val sum: Kind => Cell = k => k.family match { - case WholeNumber => Agreed(sumWidthIsOpen, wholeAtLeast(k.sqlType)) - case FractionalNumber => Agreed(sumWidthIsOpen, fractional) + case WholeNumber => Pinned(SQLTypes.BigInt) + case FractionalNumber => Pinned(SQLTypes.Double) // DuckDB counts the trues as a HUGEINT; core declares the DOUBLE Elasticsearch's sum answers case Bool => Agreed("SUM of a BOOLEAN: DuckDB HUGEINT, core DOUBLE") case _ => Refused @@ -305,11 +287,8 @@ class AggregateReportedTypeSpec extends AnyFlatSpec with Matchers { } private val windows: List[(String, Cell)] = List( - "SELECT COUNT(n) OVER (PARTITION BY g) AS a FROM t" -> Pinned(SQLTypes.BigInt), - "SELECT SUM(i8) OVER (PARTITION BY g) AS a FROM t" -> Agreed( - sumWidthIsOpen, - wholeAtLeast(SQLTypes.TinyInt) - ), + "SELECT COUNT(n) OVER (PARTITION BY g) AS a FROM t" -> Pinned(SQLTypes.BigInt), + "SELECT SUM(i8) OVER (PARTITION BY g) AS a FROM t" -> Pinned(SQLTypes.BigInt), "SELECT AVG(n) OVER (PARTITION BY g) AS a FROM t" -> Pinned(SQLTypes.Double), "SELECT MAX(n) OVER (PARTITION BY g) AS a FROM t" -> Pinned(SQLTypes.Int), "SELECT FIRST_VALUE(d) OVER (PARTITION BY g ORDER BY n) AS a FROM t" -> Pinned(SQLTypes.Date), @@ -373,9 +352,10 @@ class AggregateReportedTypeSpec extends AnyFlatSpec with Matchers { } } - // Not pinned to a width (open decision, see `sumWidthIsOpen`): what holds either way is that a - // whole SUM stays whole and at least as wide as its column, and a fractional SUM stays fractional. - it should "keep a SUM whole over a whole column and fractional over a fractional one" in { + // SCHEMA-2b (A-20): a whole SUM is a BIGINT whatever its column's width -- three INTs of + // 2,000,000,000 sum beyond an INT -- and a fractional SUM a DOUBLE, a REAL one included + // (Elasticsearch's `sum` is a double); a CTAS maps them `long` and `double` + it should "declare a SUM BIGINT over a whole column and DOUBLE over a fractional one" in { for (name <- List("i8", "i16", "n", "l", "r", "x")) assertCell(s"SELECT SUM($name) AS a FROM t", sum(kind(name))) } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala index 92716aa6..ba566924 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala @@ -45,8 +45,8 @@ import org.scalatest.matchers.should.Matchers * * What an operand IS for these rules: * - `NOW()` is a DATETIME, which these rules read as an instant, like a TIMESTAMP; - * - an aggregate is the metric it produces: `COUNT` a BIGINT, `AVG` a DOUBLE, `SUM` its column's - * number, `MIN` / `MAX` what they aggregate; + * - an aggregate is the metric it produces: `COUNT` a BIGINT, `AVG` a DOUBLE, `SUM` a BIGINT + * over a whole column and a DOUBLE over a fractional one, `MIN` / `MAX` what they aggregate; * - `CASE`, `COALESCE`, `GREATEST` and `LEAST` are the least common supertype of their branches * or arguments, `NULLIF` its first argument -- `GREATEST` / `LEAST` over dates included, a * DATE when every argument is one, a TIMESTAMP when they mix; @@ -129,7 +129,8 @@ class DateArithmeticTypeSpec extends AnyFlatSpec with Matchers { Operand("GREATEST(n, 2)", Whole, "BIGINT"), Operand("LEAST(x, 1.5)", Fractional, "DOUBLE"), Operand("AVG(n)", Fractional, "DOUBLE", aggregate = true, ddl = false), - Operand("SUM(n)", Whole, "INT", aggregate = true, ddl = false), + // SCHEMA-2b (A-20): a SUM over a whole column is a BIGINT + Operand("SUM(n)", Whole, "BIGINT", aggregate = true, ddl = false), Operand("COUNT(d)", Whole, "BIGINT", aggregate = true, ddl = false), Operand("CAST(NULL AS DATE)", DateKind, "DATE"), Operand("CAST(NULL AS INT)", Whole, "INT"), diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala index 10969df5..a819671b 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala @@ -309,7 +309,8 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven ( "arithmetic (per group)", "SELECT g, SUM(n) + '1.5' AS v FROM t GROUP BY g", - "output 'INT' is not compatible with input 'VARCHAR'" + // SUM over a whole column is declared BIGINT + "output 'BIGINT' is not compatible with input 'VARCHAR'" ) ) ) { (site, sql, refusal) => diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 0501454e..cc1a24a8 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -1132,8 +1132,9 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M case Some(m: Map[_, _]) => // Elasticsearch 7.14+ returns an object for first/last aggregations on date fields m.asInstanceOf[Map[String, Any]].get("birthDate") match { + // SCHEMA-2b (B5): a top_hits value read by its column's type, a TIMESTAMP at Z case Some(t: Temporal) => - t should ===(LocalDate.parse("1967-11-21")) + t should ===(LocalDate.parse("1967-11-21").atStartOfDay(ZoneOffset.UTC)) case Some(d: Double) => d should ===( LocalDate @@ -1166,8 +1167,9 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M case Some(m: Map[_, _]) => // Elasticsearch 7.14+ returns an object for first/last aggregations on date fields m.asInstanceOf[Map[String, Any]].get("birthDate") match { + // SCHEMA-2b (B5): a top_hits value read by its column's type, a TIMESTAMP at Z case Some(t: Temporal) => - t should ===(LocalDate.parse("1969-05-09")) + t should ===(LocalDate.parse("1969-05-09").atStartOfDay(ZoneOffset.UTC)) case Some(d: Double) => d should ===( LocalDate @@ -1216,7 +1218,8 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } .map(_.toString) birthDates.nonEmpty shouldBe true - birthDates should contain allOf ("1999-05-09", "2002-05-09") // LocalDate instances sorted ASC + // SCHEMA-2b (B5): each element read by its column's type -- a TIMESTAMP at Z, sorted ASC + birthDates should contain allOf ("1999-05-09T00:00Z", "2002-05-09T00:00Z") case Failure(f) => fail(f.getMessage) } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala index 32c5078c..d2419aeb 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala @@ -55,6 +55,17 @@ import scala.util.Try * reload (AC6, Elasticsearch 7+); * - "describe": `describe(sql)` = `run(sql)`'s schema (AC1). * + * SCHEMA-2b -- aggregation values and window statements by type, on fixtures `s2b_t` / `s2b_w`: + * + * - "aggregation rows": Table A (AC3); + * - "aggregation classes": every non-null value of an aggregation column has D2's class on every + * aggregation route (AC2): one-shot search, `searchAsync`, the `scroll()` API, the `_msearch` + * legs of a UNION ALL; + * - "window oracle": every partition column of `s2b_w` x {one-shot, async, scroll} x {rn, s, f} + * answers the fixture's arithmetic (AC4); + * - "describe aggregates": `describe(sql)` = `run(sql)`'s schema over aggregate, top_hits, Case + * 4 and window statements (AC1). + * * PUBLIC API ONLY but `describe` and the schema: the same fixture runs as the T0 baseline probe on * a main clone. */ @@ -119,15 +130,71 @@ trait SchemaReadingSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M client.refresh(index) } + // SCHEMA-2b fixtures: Table A (`s2b_t`) and the window oracle (`s2b_w`) + protected val aggTable = "s2b_t" + protected val windowTable = "s2b_w" + + private val aggMapping: String = + """{"_meta": {"columns": {"c_des": {"data_type": "DATE"}}}, + | "properties": { + | "id": {"type": "keyword"}, "b": {"type": "boolean"}, + | "dc": {"type": "date", "format": "yyyy/MM/dd"}, + | "c_des": {"type": "date", "format": "epoch_second"}, + | "big": {"type": "long"}, "n": {"type": "integer"}, "k": {"type": "keyword"}, + | "x": {"type": "double"}, "r": {"type": "float"} + |}}""".stripMargin + + /** Table A: `big` {42, 1706697000000}, `n` sum 15 max 10, `k` "10:30"; `a3` holds only a + * whole-day `c_des` (A-25). + */ + private val aggDocs: Seq[(String, String)] = Seq( + "a1" -> """{"id":"a1","b":true,"dc":"2024/01/31","c_des":1706697000,"big":42,"n":5,"k":"10:30","x":1.5,"r":0.5}""", + "a2" -> """{"id":"a2","b":true,"dc":"2024/02/01","big":1706697000000,"n":10,"k":"10:30","x":2.5,"r":1.5}""", + "a3" -> """{"id":"a3","c_des":1706659200}""" + ) + + /** `pz` (a zone-name format, read as stored) exists from Elasticsearch 7 on: 6.8's Joda-Time + * cannot parse a zone name, so it would refuse the documents. + */ + protected lazy val zoneNames: Boolean = major >= 7 + + // `pn` is the spec's `pi` (PI is a reserved word) + private def windowMapping: String = + """{"_meta": {"columns": {"pd": {"data_type": "DATE"}, "pe": {"data_type": "DATE"}}}, + | "properties": { + | "id": {"type": "keyword"}, "n": {"type": "integer"}, + | "pk1": {"type": "keyword"}, "pk2": {"type": "keyword"}, "pu": {"type": "keyword"}, + | "pt": {"type": "date"}, "pb": {"type": "boolean"}, + | "pd": {"type": "date", "format": "epoch_second"}, "pe": {"type": "date"},""".stripMargin + + (if (zoneNames) """ "pz": {"type": "date", "format": "yyyy-MM-dd HH:mm z"},""" else "") + + """ "pn": {"type": "integer"}}}""" + + private def windowDocs: Seq[(String, String)] = { + def pz(text: String): String = if (zoneNames) s""","pz":"$text"""" else "" + Seq( + "w1" -> s"""{"id":"w1","n":1,"pk1":"10:30","pk2":"2024-01-31","pu":"x","pt":"2024-01-31T10:30:00.123456Z","pb":true,"pd":1706659200,"pe":"2024-01-31T00:00:00Z"${pz( + "2024-01-31 10:30 UTC" + )},"pn":7}""", + "w2" -> s"""{"id":"w2","n":2,"pk1":"10:30","pk2":"2024-01-31","pu":"x","pt":"2024-01-31T10:30:00.123Z","pb":true,"pd":1706697000,"pe":"2024-01-31T10:30:00Z"${pz( + "2024-01-31 11:30 CET" + )},"pn":7}""", + "w3" -> s"""{"id":"w3","n":4,"pk1":"11:00","pk2":"2024-02-01","pu":"y","pt":"2024-02-01T00:00:00Z","pb":false,"pd":1706709600,"pe":"2024-01-31T00:00:00Z"${pz( + "2024-02-01 00:00 UTC" + )},"pn":8}""" + ) + } + override def beforeAll(): Unit = { super.beforeAll() - Seq(table, q4, q4d).foreach(drop) + Seq(table, q4, q4d, aggTable, windowTable).foreach(drop) load(table, mapping, docs) + load(aggTable, aggMapping, aggDocs) + load(windowTable, windowMapping, windowDocs) () } override def afterAll(): Unit = { - Seq(table, q4, q4d).foreach(drop) + Seq(table, q4, q4d, aggTable, windowTable).foreach(drop) super.afterAll() } @@ -385,4 +452,241 @@ trait SchemaReadingSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } } } + + // ------------------------------------------------------------------------------------------- + // SCHEMA-2b -- Table A (AC3) + // ------------------------------------------------------------------------------------------- + + private def byKey(sql: String, key: String): Map[Any, ListMap[String, Any]] = + rowsOf(sql).map(r => r.getOrElse(key, null) -> r).toMap + + "Table A" should "give the aggregation rows' answers (AC3)" in { + // 20a + byKey(s"SELECT b, COUNT(*) AS c FROM $aggTable GROUP BY b", "b").keySet shouldBe + Set[Any](java.lang.Boolean.TRUE) + // 20b: a custom-format date key from its epoch millis, at Z + val dc = byKey(s"SELECT dc, COUNT(*) AS c FROM $aggTable GROUP BY dc", "dc").keySet + dc shouldBe Set[Any](utc(2024, 1, 31), utc(2024, 2, 1)) + dc.foreach(v => v.asInstanceOf[ZonedDateTime].getZone shouldBe ZoneOffset.UTC) + // 20c: BIGINT always a Long + val big = byKey(s"SELECT big, COUNT(*) AS c FROM $aggTable GROUP BY big", "big").keySet + big shouldBe Set[Any](java.lang.Long.valueOf(42L), java.lang.Long.valueOf(1706697000000L)) + big.map(_.getClass.getName) shouldBe Set("java.lang.Long") + // 20d (A-25): 10:30 keeps main's reading (a counted residual), the whole day is a LocalDate + val cDes = byKey(s"SELECT c_des, COUNT(*) AS c FROM $aggTable GROUP BY c_des", "c_des").keySet + cDes should contain(LocalDate.of(2024, 1, 31)) + cDes.filterNot(_ == LocalDate.of(2024, 1, 31)).map(_.getClass.getName) shouldBe + Set("java.lang.String") + // 20e / 20f + byKey(s"SELECT n, COUNT(*) AS c FROM $aggTable GROUP BY n", "n").keySet shouldBe + Set[Any](Integer.valueOf(5), Integer.valueOf(10)) + byKey(s"SELECT k, COUNT(*) AS c FROM $aggTable GROUP BY k", "k").keySet shouldBe Set[Any]( + "10:30" + ) + // 21a-21d + val metrics = one( + s"SELECT MAX(dc) AS d, SUM(n) AS s, MAX(n) AS m, COUNT(*) AS c, COUNT(n) AS cn FROM $aggTable" + ) + metrics("d") shouldBe utc(2024, 2, 1) + metrics("d").asInstanceOf[ZonedDateTime].getZone shouldBe ZoneOffset.UTC + metrics("s") shouldBe java.lang.Long.valueOf(15L) + metrics("m") shouldBe Integer.valueOf(10) + metrics("c") shouldBe java.lang.Long.valueOf(3L) + metrics("cn") shouldBe java.lang.Long.valueOf(2L) + } + + // ------------------------------------------------------------------------------------------- + // SCHEMA-2b -- aggregation classes (AC2) + // ------------------------------------------------------------------------------------------- + + /** The aggregation routes (AC2, B13): one-shot `search`, `run` (`searchAsync`), the `scroll()` + * API (classic scroll), and the `_msearch` legs of a UNION ALL (aggregation legs never take the + * per-leg route). + */ + protected def onAggregationRoute( + route: String, + sql: String + ): (Seq[ListMap[String, Any]], Option[ResultSchema]) = + route match { + case "search" => + client.search(SelectStatement(sql))(NativeContext) match { + case ElasticSuccess(r) => (r.results, r.schema) + case ElasticFailure(error) => fail(s"[$route] $sql: ${error.message}") + } + case "searchAsync" => answer(runOk(sql)) + case "scroll" => + val (schema, source) = client.scrollWithSchema(SelectStatement(sql))(system, NativeContext) + (Await.result(source.map(_._1).runWith(Sink.seq), 60.seconds), schema) + case "msearch" => answer(runOk(s"$sql UNION ALL $sql")) + } + + protected val aggregationRoutes: Seq[String] = Seq("search", "searchAsync", "scroll", "msearch") + + "Every aggregation route" should "answer every value of a typed column in D2's class (AC2)" in { + // residuals excluded by construction: no inexact DATE key, no ARRAY_AGG column (A-21), no + // statistics map + Seq( + s"SELECT b, COUNT(*) AS c FROM $aggTable GROUP BY b", + s"SELECT dc, COUNT(*) AS c, MAX(n) AS m FROM $aggTable GROUP BY dc", + s"SELECT big, SUM(n) AS s FROM $aggTable GROUP BY big", + s"SELECT n, MIN(x) AS mx, MAX(r) AS mr FROM $aggTable GROUP BY n", + s"SELECT k, COUNT(n) AS c, SUM(n) AS s, AVG(n) AS a, MAX(dc) AS d, MAX(b) AS mb FROM $aggTable GROUP BY k", + s"SELECT k, b, COUNT(*) AS c FROM $aggTable GROUP BY k, b", + s"SELECT COUNT(*) AS c, SUM(n) AS s, MAX(n) AS m, MIN(big) AS mb, MAX(dc) AS d FROM $aggTable", + s"SELECT COUNT(*) AS c, SUM(n) AS s, MIN(n) AS m FROM $aggTable WHERE id = 'none'", + s"SELECT k, SUM(n) AS s FROM $aggTable GROUP BY k HAVING SUM(n) > 1", + s"SELECT k, MAX(n) - MIN(n) AS d, SUM(n) + 1 AS s1 FROM $aggTable GROUP BY k", + // Case 4: hits beside global metrics + s"SELECT id, COUNT(*) OVER () AS c, MAX(pt) OVER () AS m FROM $windowTable LIMIT 5" + ).foreach { sql => + aggregationRoutes.foreach { route => + withClue(s"[$route] [$sql] ") { + val (rows, schema) = onAggregationRoute(route, sql) + rows should not be empty + classMisses(rows, schema.getOrElse(fail("no schema"))) shouldBe empty + } + } + } + } + + it should "read top_hits elements canonically (ARRAY_AGG DISTINCT over a date)" in { + val row = one(s"SELECT ARRAY_AGG(DISTINCT dc) AS a FROM $aggTable") + row("a") match { + case s: Seq[_] => + // a3 holds no `dc`: its element is NULL, as SQL keeps it -- DuckDB 1.5.5 `SELECT + // array_agg(DISTINCT x) FROM (VALUES (DATE '2024-01-31'), (NULL), (DATE '2024-02-01'), + // (DATE '2024-01-31')) t(x)` = [NULL, 2024-01-31, 2024-02-01]; 0.24.0 answers that NULL too + s.toSet shouldBe Set[Any](utc(2024, 1, 31), utc(2024, 2, 1), null) + s.filter(_ != null).foreach(_ shouldBe a[ZonedDateTime]) + case other => fail(s"ARRAY_AGG answered $other") + } + } + + // ------------------------------------------------------------------------------------------- + // SCHEMA-2b -- the window oracle (AC4) + // ------------------------------------------------------------------------------------------- + + private def windowSql(p: String, selected: Boolean = true, head: String = "id, n"): String = + s"SELECT $head${if (selected) s", $p" else ""}, " + + s"ROW_NUMBER() OVER (PARTITION BY $p ORDER BY id) AS rn, " + + s"SUM(n) OVER (PARTITION BY $p) AS s, " + + s"FIRST_VALUE(n) OVER (PARTITION BY $p ORDER BY id) AS f FROM $windowTable" + + /** The window routes: one-shot sync (LIMIT 100), async (`run`, LIMIT 100), scroll (no LIMIT). */ + private def onWindowRoute(route: String, sql: String): Seq[ListMap[String, Any]] = + route match { + case "search" => + client.search(SelectStatement(s"$sql LIMIT 100"))(NativeContext) match { + case ElasticSuccess(r) => r.results + case ElasticFailure(error) => fail(s"[$route] $sql: ${error.message}") + } + case "searchAsync" => answer(runOk(s"$sql LIMIT 100"))._1 + case "scroll" => answer(runOk(sql))._1 + } + + /** The fixture's arithmetic: id -> (rn, s, f), `null` where the partition is not answered. */ + private type Oracle = Map[String, (Any, Any, Any)] + + private def answered(rn: Long, s: Long, f: Int): (Any, Any, Any) = + (java.lang.Long.valueOf(rn), java.lang.Long.valueOf(s), Integer.valueOf(f)) + + private val none: (Any, Any, Any) = (null, null, null) + + // {w1, w2} share a partition and w3 has its own: n 1 + 2 = 3, and 4 + private val pairs: Oracle = + Map("w1" -> answered(1, 3, 1), "w2" -> answered(2, 3, 1), "w3" -> answered(1, 4, 4)) + + private def oracles: Seq[(String, Oracle)] = Seq( + "pk1" -> pairs, + "pk2" -> pairs, + // w1 / w2 hold the same millisecond (w1's micros truncated as Elasticsearch indexes them) + "pt" -> pairs, + "pb" -> pairs, + "pn" -> pairs, + // only w1 is exactly a day (A-23): w2 (10:30) and w3 (14:00) answer as main does (NULL) + "pd" -> Map("w1" -> answered(1, 1, 1), "w2" -> none, "w3" -> none), + // w1 / w3 at 00:00 share the day; w2 at 10:30 pairs as on main (equal instants) + "pe" -> Map("w1" -> answered(1, 5, 1), "w2" -> answered(1, 2, 2), "w3" -> answered(2, 5, 1)) + ) ++ ( + // read as stored, partitioned by instant, as Elasticsearch buckets a date: w1 2024-01-31 + // 10:30 UTC = w2 11:30 CET (CET = UTC+1 in January) = 1706697000000 ms, one bucket {w1, w2} + // (n 1 + 2); w3 2024-02-01 00:00 UTC alone (n 4) + if (zoneNames) Seq("pz" -> pairs) else Nil + ) + + private def assertOracle(route: String, sql: String, oracle: Oracle, columns: Seq[String]): Unit = + withClue(s"[$route] [$sql] ") { + val rows = onWindowRoute(route, sql) + rows.map(_.keys.toSeq).distinct shouldBe Seq(columns) + val byId = rows.map(r => r("id").toString -> r).toMap + byId.keySet shouldBe oracle.keySet + oracle.foreach { case (id, (rn, s, f)) => + withClue(s"$id ") { + val row = byId(id) + (row("rn"), row("s"), row("f")) shouldBe ((rn, s, f)) + } + } + } + + "The window oracle" should "answer every partition column by the fixture's arithmetic (AC4)" in { + for { + (p, oracle) <- oracles + route <- Seq("search", "searchAsync", "scroll") + } assertOracle(route, windowSql(p), oracle, Seq("id", "n", p, "rn", "s", "f")) + } + + it should "answer an unselected partition column only where the whole document is fetched (B10)" in { + Seq("search", "searchAsync", "scroll").foreach { route => + // its `_source` is not fetched: NULL, as on main (recorded, not fixed) + assertOracle( + route, + windowSql("pu", selected = false), + Map("w1" -> none, "w2" -> none, "w3" -> none), + Seq("id", "n", "rn", "s", "f") + ) + } + Seq("search", "searchAsync", "scroll").foreach { route => + // a computed column alone fetches the whole `_source`: answered, the carried column dropped + withClue(s"[$route] UPPER(id) ") { + val rows = onWindowRoute(route, windowSql("pu", selected = false, head = "UPPER(id) AS u")) + rows.map(_.keys.toSeq).distinct shouldBe Seq(Seq("u", "rn", "s", "f")) + rows.map(r => r("u").toString -> r("s")).toMap shouldBe Map( + "W1" -> java.lang.Long.valueOf(3L), + "W2" -> java.lang.Long.valueOf(3L), + "W3" -> java.lang.Long.valueOf(4L) + ) + } + // `*` fetches it too + withClue(s"[$route] * ") { + val rows = onWindowRoute(route, windowSql("pu", selected = false, head = "*")) + rows.map(r => r("id").toString -> r("s")).toMap shouldBe Map( + "w1" -> java.lang.Long.valueOf(3L), + "w2" -> java.lang.Long.valueOf(3L), + "w3" -> java.lang.Long.valueOf(4L) + ) + } + } + } + + // ------------------------------------------------------------------------------------------- + // SCHEMA-2b -- describe = run's schema (AC1) + // ------------------------------------------------------------------------------------------- + + "describe" should "answer run's schema over aggregate, top_hits, Case 4 and window statements (AC1)" in { + Seq( + s"SELECT b, COUNT(*) AS c FROM $aggTable GROUP BY b", + s"SELECT dc, SUM(n) AS s, MAX(n) AS m, MAX(dc) AS d FROM $aggTable GROUP BY dc", + s"SELECT COUNT(*) AS c, SUM(n) AS s, SUM(x) AS sx, SUM(r) AS sr FROM $aggTable", + s"SELECT k, MAX(n) - MIN(n) AS d, SUM(n) + 1 AS s1 FROM $aggTable GROUP BY k", + s"SELECT k, FIRST(dc) AS f FROM $aggTable GROUP BY k", + s"SELECT id, COUNT(*) OVER () AS c, MAX(pt) OVER () AS m FROM $windowTable LIMIT 5", + s"${windowSql("pb")} LIMIT 100", + windowSql("pt") + ).foreach { sql => + withClue(s"[$sql] ") { + val described: ElasticResult[Option[ResultSchema]] = client.describe(sql) + described shouldBe ElasticSuccess(answer(runOk(sql))._2) + } + } + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala index a8327d34..f0fc4c9c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala @@ -555,12 +555,15 @@ trait UnnestFunctionRefusalExecutionSpec "a window over UNNEST columns beside a parent function" should "keep its rows" in { // The window's argument is evaluated per element of its nested aggregation, and the parent // function reads the parent, where it is visible: the rule accepts both. Over ARITHMETIC of the - // UNNEST columns -- a window over a BARE one answers NULL (recorded, not this rule's). + // UNNEST columns -- a window over a BARE one answers NULL (recorded, not this rule's). `i.n` is + // a `long`, so SUM(i.n * 1) is an integer sum: BIGINT, a Long (A-20), as an integer SUM is an + // integer type elsewhere -- DuckDB 1.5.5 `SELECT typeof(sum(i)) FROM (VALUES (4), (6)) t(i)` = + // HUGEINT; the value is unchanged. val expected = parents .flatMap(p => p.items.map { it => val name = p.name.toUpperCase(_root_.java.util.Locale.ROOT) - s"${p.id}|${it.k}|$name|${p.items.map(_.n).sum.toDouble}" + s"${p.id}|${it.k}|$name|${p.items.map(_.n).sum}" } ) .sorted From 2cb97ee558dff2cbeabf56bd628daebf6dfeb53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 31 Oct 2026 22:14:43 +0100 Subject: [PATCH 2/3] docs: core pages describe the measured typed results and window partitions (SCHEMA-2b) GROUP BY key classes, aggregate types (UNION ALL, type conversion, SUM over no row), set-operation dates, window partitions, row projection, and the JDBC / Arrow column types, rewritten to what SCHEMA-2 and SCHEMA-2b measure. Story SCHEMA-2b. Reviewed: docs r0:must-fix, r1:must-fix, r2:must-fix (one clause, fixed as prescribed). Co-Authored-By: Claude Opus 5.5 --- documentation/client/arrow_flight_sql.md | 15 ++-- documentation/client/gateway.md | 17 ++-- documentation/client/jdbc.md | 20 +++-- documentation/sql/dql_statements.md | 98 +++++++++++++++--------- documentation/sql/functions_aggregate.md | 7 +- documentation/sql/known_limitations.md | 39 ++++++++-- documentation/sql/type_conversion.md | 12 ++- 7 files changed, 139 insertions(+), 69 deletions(-) diff --git a/documentation/client/arrow_flight_sql.md b/documentation/client/arrow_flight_sql.md index 021258ba..76c9e7dd 100644 --- a/documentation/client/arrow_flight_sql.md +++ b/documentation/client/arrow_flight_sql.md @@ -170,11 +170,16 @@ elastic.credentials { | `ARRAY` | `nested` | `List` | The table is the declared mapping. A plain query's Arrow schema is inferred from the values of its -rows (a `BIGINT` column holding small whole numbers arrives as `Int(32)`); a set operation's -branches are typed from the declared types above when the statement is planned, and a JOIN result -carries DuckDB's types. A column the reader cannot answer — a dictionary-encoded column, an Arrow -type it has no reader for — fails naming the column instead of reading `NULL` (*since -arrow-extensions `0.3.4`*). +rows; the engine answers every value it reads by type in the class of its column's declared type +(*since engine `0.25.0`*), so the schema follows the table: a `BIGINT` column arrives as `Int(64)` +whatever its values, and a computed column the engine declares `DOUBLE` (`x * 2` over a `DOUBLE` +column) as `Float(DOUBLE)` — in the advertised schema and in the stream alike. A value the engine +answers as stored keeps its own class (see [Value classes](gateway.md#value-classes)), and a column +the engine types only as `NUMERIC` or `ANY` can carry whole and decimal numbers alike: there, the +column takes the type of its first value. A set operation's branches are typed from the declared +types above when the statement is planned, and a JOIN result carries DuckDB's types. A column the +reader cannot answer — a dictionary-encoded column, an Arrow type it has no reader for — fails +naming the column instead of reading `NULL` (*since arrow-extensions `0.3.4`*). --- diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md index 57c8cebb..b517c107 100644 --- a/documentation/client/gateway.md +++ b/documentation/client/gateway.md @@ -244,8 +244,9 @@ Aggregation values follow the same table: - A GROUP BY key is read by its column's type: a `boolean` key is a `Boolean`, a date key the `java.time` value of its epoch milliseconds at UTC (whatever the mapping format), a `long` key a `Long`, a keyword key its text (a keyword spelling a date or a time stays text). A `DATE` key that - is not exactly a day, or a `TIME` key that is not a time of day of 1970-01-01, keeps its stored - reading. + is not exactly a day, or a `TIME` key that is not a time of day of 1970-01-01, keeps the reading + `0.24.0` gave it (a `DATE` key: a timestamp at UTC under the default format, its text in the + column's format under a format other than ISO). - A metric is read by its column's type, exactly: `COUNT` is a `Long` on every client, `MIN` / `MAX` of an integer an `Integer` (a `Long` for a `long`), of a date its `java.time` value at UTC, of a boolean a `Boolean`. A value the class cannot hold unchanged (a sum beyond `Long`, a @@ -258,13 +259,15 @@ Aggregation values follow the same table: `ARRAY_AGG(DISTINCT d)` keeps one value per instant). - A window-enriched statement (`... OVER (PARTITION BY p ...)`) reads its rows by type and partitions on the same values: a `BOOLEAN`, `TIMESTAMP`, `BIGINT` or whole-day `DATE` partition - column is answered, and the rows hold only the SELECT's columns. A partition column the SELECT - does not project is read only when the whole document is fetched, as with `SELECT *`. + column is answered, and the rows of a statement without `*` hold only the SELECT's columns. A + partition column the SELECT does not project is read only when the whole document is fetched, + as with `SELECT *`: otherwise every window of the statement reads NULL, and so does the column + each one aggregates. A `date` partition column (not `date_nanos`) read as its stored text (a format with a zone name) is matched by its instant, as Elasticsearch buckets it, whether the SELECT projects it under its - own name or not at all; a projected one still shows its stored text. Projected under another - name (`pz AS pz2`), or shadowed by a computed column of the SELECT that takes its name, it is - not matched that way. + own name or not at all, unless another window's partition column is not projected; a projected + one still shows its stored text. Projected under another name (`pz AS pz2`), or shadowed by a + computed column of the SELECT that takes its name, it is not matched that way. --- diff --git a/documentation/client/jdbc.md b/documentation/client/jdbc.md index 0fb9c837..fb8d8283 100644 --- a/documentation/client/jdbc.md +++ b/documentation/client/jdbc.md @@ -261,13 +261,19 @@ The JDBC driver supports the full SQL Gateway syntax: - **SHOW/DESCRIBE** — Tables, pipelines, watchers, enrich policies **Column types.** A plain query's `ResultSetMetaData` is inferred from the first non-null value of -each column, not from the type the column declares: a `BIGINT` column holding small whole numbers -is reported as `INTEGER`, and `getLong` still reads it as a long. A set operation's columns follow -the declared types. On a JOIN result (*since arrow-extensions `0.3.4`*) the columns carry DuckDB's -types, an integer getter throws `SQLException` on a value outside its type's range instead of -answering a wrapped value, `getBigDecimal` of a `BIGINT` is exact, and a column the reader has no -mapping for (a dictionary-encoded column, for instance) fails naming the column instead of reading -`NULL`. +each column. The engine answers every value it reads by type in the class of its column's declared +type (*since engine `0.25.0`*), so the inferred type is the declared one: a `BIGINT` column is +reported as `BIGINT`, its small values included, and `getObject` answers a `Long` on every row; a +computed column the engine declares `DOUBLE` (`x * 2` over a `DOUBLE` column) is reported as +`DOUBLE` and answers a `Double` on every row. A `TINYINT` or `SMALLINT` column, whose values are +`Integer`s, is reported as `INTEGER`. A value the engine answers as stored keeps its own class (see +[Value classes](gateway.md#value-classes)), and a column the engine types only as `NUMERIC` or `ANY` +can carry whole and decimal numbers alike: there, the column is reported as the type of its first +value. A set operation's columns follow the declared types. On a JOIN result (*since +arrow-extensions `0.3.4`*) the columns carry DuckDB's types, an integer getter throws `SQLException` +on a value outside its type's range instead of answering a wrapped value, `getBigDecimal` of a +`BIGINT` is exact, and a column the reader has no mapping for (a dictionary-encoded column, for +instance) fails naming the column instead of reading `NULL`. --- diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index a1b93709..4c0f4403 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -89,10 +89,9 @@ ORDER BY id ASC; - Aliases (`AS full_name`, `AS city`) are returned as column names. - An un-aliased nested field is returned under its dotted name: `SELECT id, profile.city FROM dql_users` returns the columns `id` and `profile.city`. The parent - object `profile` is normally left out of these rows. A row can still carry it, for example when - the `SELECT` list also names `profile`, contains `*`, or reads another field of `profile` under - an alias (`profile.followers AS followers`). Rows read into a typed entity (`searchAs`) keep the - parent object. + object `profile` is in a row only when the `SELECT` list names `profile` or contains `*` (*since + `0.25.0`*): reading another field of `profile`, under an alias or not (`profile.followers AS + followers`), does not add it. Rows read into a typed entity (`searchAs`) keep the parent object. --- @@ -840,8 +839,10 @@ position holds ONE class: a `DOUBLE` a `Double`, a `BIGINT` a `Long`, a `TIMESTA bytes, a `VARCHAR` a string, a list a list of its element's type, an object a map of its fields in order — which is the type JDBC and Arrow report, since they read it from the first value. A position whose branches already have one type is returned as each branch reads it, nothing -converted: a branch over an aggregate keeps the form an aggregate answers, a `Double`, so `SELECT -MAX(n) ... UNION ALL SELECT n ...` answers `10.0` beside `3` and `5`. +converted, and a branch over an aggregate answers the class of the aggregate's type (*since +`0.25.0`*: a `Double` before). So, `n` being an `INT` column, `SELECT MAX(n) AS k FROM t UNION ALL +SELECT n AS k FROM t WHERE n < 10` is an `INT` column answering the `Integer`s `10`, `3` and `5`, +while `SUM(n)`, a `BIGINT`, in place of `MAX(n)` makes a `BIGINT` column of `Long`s. Each value is first read as the type its field is declared with, as Elasticsearch indexed it, and only then converted: a value Elasticsearch accepted as text is the value its queries see. In an @@ -862,10 +863,10 @@ second on Elasticsearch 6; `"2024/01/31"` in a `yyyy/MM/dd` field is a date, not format `"2024"` and `2024` are the year 2024. A value computed by the statement (a function, a `CAST`) has no format: there, digits are epoch milliseconds and an ISO text is its instant. A date whose branches already agree -- a column, or a field of an object or of a list of objects at any -depth -- is read the same way, in every branch, as soon as one of its branches is a field with a -`format` of its own (`epoch_second`, `yyyy/MM/dd`, ...), every other value of the row as read, while -a date position whose branches all have the default format, a multi-valued value and a date core -cannot read (a zone name, say) are returned as stored. +depth -- is read the same way, in every branch, with its field's format, the default format +included (*since `0.25.0`*: a date whose branches all had the default format, and a multi-valued +date, were returned as stored); a multi-valued date is the list of its values, each read so, and a +date core cannot read (below) is returned as stored. A value is read only where its reading is the same on every Java version core runs on (8, 11, 17 and 21) and equal to the instant the Elasticsearch major indexed, measured value by value on @@ -880,37 +881,47 @@ Elasticsearch 8 indexes `2024-01-31` as `2023-12-31`, the first day of the week- week, and Elasticsearch 7 as `2024-01-01`; a year with no month and an hour (`2024T10` in a `strict_date_optional_time` field) is `1970-01-01 10:00:00`. -A value its field's format does not read fails the statement, naming the value, the field and the -format, and so does a value core cannot read without guessing, rather than be read as a date it -may not be: +A date field whose format core cannot read every value of for sure is read as its text as stored, +decided from the format before any row is read (*since `0.25.0`*: a position that converts failed): +the column is a `VARCHAR`, its other branches' values spelled as text (below), and at a field of an +object the value is returned as stored. Such a format is one with: - a pattern letter, a width or a combination of letters that no measurement covers — two zone - letters included (an offset beside a zone id, Joda-Time's `ZZ ZZZ`) — and a name of + letters included (an offset beside a zone id, Joda-Time's `ZZ ZZZ`) — or a name of Elasticsearch 6.8's `8`-prefixed mode; -- a zone id or a zone name other than `UTC`, `GMT` and `UT`, in every format, the default - included: a region id (`Europe/Paris` in a `VV` or a Joda-Time `ZZZ` letter, in the default - format, or in brackets after an offset, `+05:00[Europe/Paris]`) and an abbreviation or a long - name (`CET`, `PST`, `Pacific Time`). Its offset comes from zone data, and the zone data core's - Java version holds is not the one Elasticsearch indexed the value with: `2023-07-01T12:00:00 - America/Mexico_City` is `17:00:00Z` on Elasticsearch 8.18.3 and `18:00:00Z` with older zone data, - and `IST` is UTC on Elasticsearch 8 and Israel time on Elasticsearch 7; -- a value whose reading would depend on the Java version core runs on: a localized offset (`O`, +- a zone letter that reads a zone id or a zone name: a region id (`Europe/Paris` in a `VV` or a + Joda-Time `ZZZ` letter, or in brackets after an offset, `+05:00[Europe/Paris]`) or an + abbreviation or a long name (`CET`, `PST`, `Pacific Time`). Its offset comes from zone data, and + the zone data core's Java version holds is not the one Elasticsearch indexed the value with: + `2023-07-01T12:00:00 America/Mexico_City` is `17:00:00Z` on Elasticsearch 8.18.3 and `18:00:00Z` + with older zone data, and `IST` is UTC on Elasticsearch 8 and Israel time on Elasticsearch 7; +- a letter whose reading would depend on the Java version core runs on: a localized offset (`O`, `OOOO`, `ZZZZ`: `GMT+1`), which Java 8 parses differently from later versions; a fraction right after digits of a variable width (`yyyyMMddHHmmssSSS`); and, in Elasticsearch 6.8's `8`-prefixed - mode, an AM/PM with no hour (`8yyyy a`), which Java 16 and later set to 06:00 or 18:00, and a week - past the last of its week-based year (`2020-53-7` in a `8YYYY-ww-e` field), which Java 21 - rejects; + mode, an AM/PM with no hour (`8yyyy a`), which Java 16 and later set to 06:00 or 18:00. + +A value Elasticsearch did not index — one its field's format does not read, kept in the document by +`ignore_malformed` — is `NULL` (*since `0.25.0`*: a position that converts failed). A value core +cannot read in a format it reads otherwise fails the statement at a position that converts, naming +the value, the field and the format, rather than be read as a date it may not be, and is returned as +stored where the branches agree: + +- a zone id or a zone name other than `UTC`, `GMT` and `UT` in a format with no zone letter, the + default format included (`2025-07-01T12:00:00America/Asuncion`); +- in Elasticsearch 6.8's `8`-prefixed mode, a week past the last of its week-based year + (`2020-53-7` in a `8YYYY-ww-e` field), which Java 21 rejects; - a value the response already gives as a date or a time, without the offset or the digits its format reads (`10:30:00Z` in a `time` field); - a fractional JSON number (`1706697000.0`) whose digits, which Elasticsearch read, the number no longer pins to the millisecond. ```sql --- ts is a TIMESTAMP field of format 'yyyy/MM/dd' holding '31-01-2024', s a KEYWORD +-- s is a KEYWORD; ts is a date field SELECT ts AS k FROM t UNION ALL SELECT s AS k FROM t; --- Error: Conversion Error: Could not read '31-01-2024' as a TIMESTAMP when casting from source column k: the date format 'yyyy/MM/dd' of field ts does not read it as Elasticsearch 8 does --- a zone name --- Error: Conversion Error: Could not read '2024-01-31T10:30:00 CET' as a TIMESTAMP when casting from source column k: core cannot read it with the date format 'yyyy-MM-dd'T'HH:mm:ss z' of field ts as Elasticsearch 8 does: the zone 'CET' is a region id or a zone name, whose offset comes from the zone data of the JVM reading it, not Elasticsearch's: core reads only Z, an offset, UTC, GMT and UT +-- ts of format 'yyyy-MM-dd'T'HH:mm:ss z' holding '2024-01-31T10:30:00 CET': one VARCHAR column ('2024-01-31T10:30:00 CET', ..., 'a', ...) +-- ts of format 'yyyy/MM/dd' with ignore_malformed holding '31-01-2024': NULL, beside 'a', ... +-- ts of the default format holding '2025-07-01T12:00:00America/Asuncion': +-- Error: Conversion Error: Could not read '2025-07-01T12:00:00America/Asuncion' as a TIMESTAMP when casting from source column k: core cannot read it with the date format 'strict_date_optional_time||epoch_millis' of field ts as Elasticsearch 8 does: the zone 'America/Asuncion' is a region id or a zone name, whose offset comes from the zone data of the JVM reading it, not Elasticsearch's: core reads only Z, an offset, UTC, GMT and UT ``` The readings were measured on Elasticsearch 6.8.23, 7.17.29 and 8.18.3. The other minor versions @@ -1476,18 +1487,31 @@ ORDER BY COUNT(*) DESC; `WEEKDAY(created_at)`, `DATE_FORMAT(created_at, '%Y-%m')`, `ISNULL(city)`, `UPPER(city)`) is a `terms` level on the function's script: one group per value of the function. - The key of such a function reads back as the type it reports: a number for a function that - answers one (`YEAR(created_at)` answers `2024`, `n + 0` a number), ordered as a number - (`ORDER BY DAY(created_at) DESC` answers `31, 20, 11, 5, 3`), beside a `HAVING` condition on the - key too, and text otherwise. A conditional + The key of such a function reads back as the type it reports, in that type's class: a number for a + function that answers one (`YEAR(created_at)` answers the `BIGINT` `2024`, a `Long`; `n + 0` a + number), ordered as a number (`ORDER BY DAY(created_at) DESC` answers `31, 20, 11, 5, 3`), beside + a `HAVING` condition on the key too; a `BOOLEAN` for a predicate (`ISNULL(city)` answers `true` + and `false`); a date for a function that answers one (`LAST_DAY(created_at)` answers the `DATE` + `2024-01-31`, `DATE_TRUNC(created_at, MONTH)` the `TIMESTAMP` `2024-01-01T00:00Z`); and text for a + function that answers text (`UPPER(city)`, `DATE_FORMAT(created_at, '%Y-%m')`) (*since `0.25.0`*: + a `BOOLEAN` key read back as text, and so could a date key, depending on the function and the + column's format). A conditional key (`COALESCE(n, 2)`, `GREATEST`, `LEAST`, a `CASE`) is grouped as a number only when every one of its arguments is that kind of number: over a `scaled_float`, `ip` or `geo_point` column, or a column no mapping types, it keeps text keys, so no two values are merged. A key over a field the index does not map reads that field as `NULL`, and its rows form no group (a `GROUP BY` level forms - no `NULL` group). A bare - `DATE` column's key reads back as a timestamp at midnight UTC. On Elasticsearch 6, a key that - formats a date column as stored (`DATE_FORMAT(created_at, …)`, `DATETIME_FORMAT(created_at, …)`) - is refused: truncate the column first, `DATE_FORMAT(DATE_TRUNC(created_at, DAY), '%Y-%m')` or + no `NULL` group). A bare column's key reads back as its column's type: a `BOOLEAN` key as `true` + or `false`, a `BIGINT` key as a `Long`, an `INT` key as an `Integer`, a `TIMESTAMP` key as its + instant at UTC, and a `DATE` key as its day, `2024-01-31` (*since `0.25.0`*: a `BOOLEAN` key read + back as text, an `INT` key as a `Long`, a `BIGINT` key holding a value the size of epoch + milliseconds as a timestamp, a date key of a custom format other than ISO as text, and a `DATE` + key of the default format as a timestamp at midnight UTC). A key of a `DATE` column that + is not a whole UTC day, the column holding a time of day, keeps the reading it had before + `0.25.0`: a timestamp at UTC under the default format (`2024-01-31T10:30Z`), its text in the + column's format under a format other than ISO (`1706697000` in an `epoch_second` column). On + Elasticsearch 6, a key that formats a date column as stored + (`DATE_FORMAT(created_at, …)`, `DATETIME_FORMAT(created_at, …)`) is refused: truncate the column + first, `DATE_FORMAT(DATE_TRUNC(created_at, DAY), '%Y-%m')` or `DATETIME_FORMAT(DATE_TRUNC(created_at, SECOND), …)`. - `HAVING` filters groups based on aggregate conditions. - Translated to Elasticsearch aggregations. diff --git a/documentation/sql/functions_aggregate.md b/documentation/sql/functions_aggregate.md index 69e2fd0d..a9ca7f9f 100644 --- a/documentation/sql/functions_aggregate.md +++ b/documentation/sql/functions_aggregate.md @@ -228,9 +228,10 @@ SUM(expr) OVER (PARTITION BY partition_expr, ...) --(since v0.14.0) **NULL Handling:** - NULL values are ignored -- If all values are NULL, or over no row, returns `0.0`: the engine keeps Elasticsearch's own sum - of no value, where ANSI SQL answers NULL. `AVG`, `MIN`, `MAX`, the `STDDEV` / `VARIANCE` family - and the percentiles answer NULL there, and `COUNT` answers `0`. +- If all values are NULL, or over no row, returns zero — `0` (a `BIGINT`) over an integer column, + `0.0` over a `REAL` or a `DOUBLE` one (*since `0.25.0`*: `0.0` over every column): the engine + keeps Elasticsearch's own sum of no value, where ANSI SQL answers NULL. `AVG`, `MIN`, `MAX`, the + `STDDEV` / `VARIANCE` family and the percentiles answer NULL there, and `COUNT` answers `0`. **Examples:** diff --git a/documentation/sql/known_limitations.md b/documentation/sql/known_limitations.md index 615e03d5..652343a7 100644 --- a/documentation/sql/known_limitations.md +++ b/documentation/sql/known_limitations.md @@ -620,9 +620,28 @@ not give yet: ## Windows: partitions that read NULL -A window answers NULL on every row when its `PARTITION BY` column is one the `SELECT` list does not -return (`MIN(n) OVER (PARTITION BY g)` without `g` in the `SELECT`), or a `BOOLEAN` column. Return the -partition column in the `SELECT` list. +A window answers NULL on every row when a `PARTITION BY` column is one the `SELECT` list does not +return (`MIN(n) OVER (PARTITION BY g)` without `g` in the `SELECT`) and the `SELECT` list contains +no `*`, which fetches the whole document: every window of the statement then reads NULL, and so does +the column each one aggregates (`n`) when the `SELECT` list returns it. Return the partition columns +in the `SELECT` list. A `date` partition column whose format has a zone name, read as its stored +text, is the exception: it is matched by its instant whether the `SELECT` list returns it or not, +unless another window's partition column is not returned (*since `0.25.0`*: it read NULL). A +returned `BOOLEAN`, `TIMESTAMP`, `BIGINT` or whole-day `DATE` partition column is answered +(*since `0.25.0`*: a `BOOLEAN`, `TIMESTAMP` or whole-day `DATE` one read NULL, and so did a `BIGINT` +one holding values the size of epoch milliseconds). + +These partitions still read NULL on every row, the window and the column it aggregates alike: + +- a zone-name `date` partition column returned under another name (`pz AS pz2`); +- a partition column whose name a computed item of the `SELECT` list takes (`UPPER(g) AS g`), that + item reading NULL too; +- a `date_nanos` partition column whose format has a zone name; +- a partition column an index pattern, a list of indices or an alias maps as a zone-name `date` in + one index and as a `keyword` in another. + +And a row with no value of the partition column, or with several, reads NULL for the window and for +the column it aggregates, whatever the other rows answer. ## GROUP BY: the NULL group, and keys over UNNEST @@ -666,11 +685,15 @@ partition column in the `SELECT` list. text and stored value round differently then differs from DuckDB: `0.35` rounds to `0.4` in the view, where DuckDB rounds the `REAL` `0.35` (`0.3499999940…`) to `0.3`. A plain query over the same column answers as DuckDB. Declare the column `DOUBLE` to avoid it. -- **JDBC and Arrow Flight read a plain query's column types from its values**, not from the type the - column declares: a `BIGINT` column holding small whole numbers is reported as an `INTEGER` - (`Int32`), and a computed column whose rows carry whole and decimal numbers alike can mix - `Long` and `Double` values. A set operation (`UNION ALL` and the others) is typed from the - declared types. +- **JDBC and Arrow Flight read a plain query's column types from its values.** The values the engine + reads by type carry the class of their column's declared type (*since `0.25.0`*), so a `BIGINT` + column is reported as a `BIGINT` (`Int(64)`) whatever its values, and a computed column the engine + declares `DOUBLE` holds a `Double` on every row. A value the engine answers as stored keeps its + own class (see [Value classes](../client/gateway.md#value-classes)), and a column the engine types + only as `NUMERIC` or `ANY` can carry whole and decimal numbers alike: there, the column is + reported as the type of its first value. JDBC reports a `TINYINT` or `SMALLINT` column, whose + values are `Integer`s, as an `INTEGER`. A set operation (`UNION ALL` and the others) is typed from + the declared types. ## Coming in the upcoming release (Quarter 1 2027) diff --git a/documentation/sql/type_conversion.md b/documentation/sql/type_conversion.md index 888149d7..130862c0 100644 --- a/documentation/sql/type_conversion.md +++ b/documentation/sql/type_conversion.md @@ -88,8 +88,8 @@ SELECT CAST(MAX(salary) AS BIGINT) AS m FROM emp GROUP BY dept; All four spellings behave the same way — `::`, `CAST`, `TRY_CAST` and `CONVERT`. -Cast the aggregate's **input** instead. This is equivalent for every aggregate whose result type -follows its argument (`MIN`, `MAX`, `SUM`, `AVG`, the `STDDEV` / `VARIANCE` family, the percentiles): +Cast the aggregate's **input** instead. This is equivalent for `MIN` and `MAX`, whose result type +follows their argument: ```sql -- Both supported @@ -97,6 +97,14 @@ SELECT MAX(salary::BIGINT) AS m FROM emp GROUP BY dept; SELECT MAX(CAST(salary AS BIGINT)) AS m FROM emp GROUP BY dept; ``` +`COUNT`, `SUM`, `AVG`, the `STDDEV` / `VARIANCE` family and the percentiles produce a type of their +own (see [Aggregate functions](functions_aggregate.md#overview)): `COUNT` is a `BIGINT`; `AVG`, the +`STDDEV` / `VARIANCE` family and the percentiles are `DOUBLE`, whatever their input; `SUM` is a +`BIGINT` over an input of any whole-number type and a `DOUBLE` over a `REAL` or a `DOUBLE` (*since +`0.25.0`*: it kept the summed column's type, which overflowed), so `SUM(CAST(x AS INT))` is a +`BIGINT`, not an `INT`. For these, casting the input converts each value before it is aggregated, +which is not the same as converting the result. + Casting is unrestricted everywhere an aggregate is not involved — the SELECT list, `WHERE`, `ORDER BY`, and inside another function: From dcd6622662d64e38c08452c645dd936dba0e308e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 1 Nov 2026 00:08:41 +0100 Subject: [PATCH 3/3] docs: SUM over no value is 0 over an integer column (SCHEMA-2b) The no-matching-document paragraph still said SUM over no value is 0.0 on every column; it now follows the aggregate page: 0 over an integer column, 0.0 over a REAL or DOUBLE one. Story SCHEMA-2b. Reviewed: docs r3:clean. Co-Authored-By: Claude Opus 5.5 --- documentation/sql/dql_statements.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index 4c0f4403..a51050cf 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -1530,10 +1530,11 @@ ORDER BY COUNT(*) DESC; - Over **no matching document**, with no `GROUP BY`: a `MAX`, a `MIN` or an `AVG` is NULL, and so is a calculation that reads one; `COUNT(*)` is `0` (`COUNT(*) * 2` is `0`, and `HAVING COUNT(*) = 0` keeps the one row). A calculation over `COUNT(column)`, - `COUNT(DISTINCT column)` or `SUM(column)` is evaluated too — `COUNT(n) * 2` is `0`, `SUM(x) * 2` - is `0.0`, `SUM` over no value being `0.0` — on Elasticsearch 7.17 and later. Elasticsearch 6.8 - cannot evaluate it (it refuses the `keep_values` gap policy), so such a calculation is NULL - there. + `COUNT(DISTINCT column)` or `SUM(column)` is evaluated too — `COUNT(n) * 2` is `0`, and `SUM` + over no value is zero, `0` over an integer column and `0.0` over a `REAL` or a `DOUBLE` one, so + `SUM(n) + 1` is `1` over an integer `n` (*since `0.25.0`*: `0.0` over every column) — on + Elasticsearch 7.17 and later. Elasticsearch 6.8 cannot evaluate it (it refuses the `keep_values` + gap policy), so such a calculation is NULL there. - `HAVING` may reference a `SELECT` aggregate by its alias (`COUNT(*) AS cnt ... HAVING cnt > 1`), including the alias of an arithmetic expression over aggregates (`... AS price_range ... HAVING price_range > 10`); `BETWEEN`, `IN` and `NOT` apply to aggregates as to columns.