From 644811d1b661a619e779342b606bce0b840aa754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 30 Oct 2026 13:12:02 +0100 Subject: [PATCH 1/3] feat(core): declared ResultSchema and hit values read by type (SCHEMA-2) Story SCHEMA-2. Reviewed: r0:must-fix, r1:must-fix. Co-Authored-By: Claude Opus 5.5 --- build.sbt | 2 +- .../client/ElasticClientDelegator.scala | 15 + .../elastic/client/ElasticConversion.scala | 282 +++++-- .../elastic/client/GatewayApi.scala | 87 +- .../elastic/client/HitReading.scala | 793 ++++++++++++++++++ .../elastic/client/ResultSchema.scala | 136 +++ .../elastic/client/ScrollApi.scala | 36 +- .../elastic/client/SearchApi.scala | 162 +++- .../elastic/client/SetOperationValues.scala | 146 +++- .../elastic/client/SubqueryResolver.scala | 51 +- .../client/extensions/CoreDqlExtension.scala | 15 +- .../client/metrics/MetricsElasticClient.scala | 17 +- .../softnetwork/elastic/client/package.scala | 79 +- .../elastic/client/repl/Repl.scala | 4 +- .../client/repl/StreamingReplExecutor.scala | 14 +- .../elastic/client/result/CsvFormatter.scala | 14 +- .../elastic/client/result/JsonFormatter.scala | 18 +- .../client/result/ResultRenderer.scala | 24 +- .../elastic/client/result/package.scala | 24 +- .../client/DateDialectVersionSpec.scala | 5 +- .../client/DeclaredTextColumnSpec.scala | 23 +- .../elastic/client/DescribeSpec.scala | 277 ++++++ .../client/FromlessSelectGatewaySpec.scala | 4 +- .../elastic/client/HitReadingSpec.scala | 467 +++++++++++ .../client/NonFiniteBucketKeySpec.scala | 2 +- .../elastic/client/ResultSchemaSpec.scala | 212 +++++ .../client/SchemaMissInvalidationSpec.scala | 30 + .../elastic/client/SchemaPageFixture.scala | 110 +++ .../elastic/client/ScrollSettingsSpec.scala | 18 + .../client/SetOperationLegReadersSpec.scala | 8 +- .../client/SetOperationLegSchemasSpec.scala | 42 +- .../client/SetOperationMappedDatesSpec.scala | 40 +- .../client/SetOperationValuesSpec.scala | 13 +- .../client/ShowTablesTableTypeSpec.scala | 4 +- .../client/TemporalLiteralSearchSpec.scala | 24 +- .../elastic/client/UnionAllRoutingSpec.scala | 18 + .../client/UnnestElementConversionSpec.scala | 4 +- .../extensions/CoreDqlExtensionSpec.scala | 144 ++++ documentation/client/gateway.md | 65 ++ documentation/client/search.md | 10 +- .../client/JestClientSchemaReadingSpec.scala | 19 + .../client/JestScrollPageFailureSpec.scala | 22 + ...RestHighLevelClientSchemaReadingSpec.scala | 19 + ...HighLevelClientScrollPageFailureSpec.scala | 21 + ...RestHighLevelClientSchemaReadingSpec.scala | 19 + ...HighLevelClientScrollPageFailureSpec.scala | 13 + .../client/java/JavaClientConversion.scala | 16 +- .../arrow/JavaClientDottedColumnSpec.scala | 4 +- .../arrow/JavaClientSchemaConsumerSpec.scala | 205 +++++ .../client/JavaClientSchemaReadingSpec.scala | 19 + .../java/JavaClientConversionSpec.scala | 27 + .../client/java/JavaClientConversion.scala | 16 +- .../client/JavaClientSchemaReadingSpec.scala | 19 + .../java/JavaClientConversionSpec.scala | 27 + .../elastic/sql/query/package.scala | 18 + .../elastic/sql/type/ValueCoercion.scala | 7 +- .../client/AggregateWithoutGroupBySpec.scala | 4 +- .../client/ClientErrorStatusSpec.scala | 8 +- .../elastic/client/DateArithmeticSpec.scala | 23 +- .../client/DateFunctionExecutionSpec.scala | 12 +- .../elastic/client/DottedColumnSpec.scala | 31 +- .../client/GatewayApiIntegrationSpec.scala | 95 ++- .../client/GatewayIntegrationTestKit.scala | 4 +- .../client/GroupByCompletenessSpec.scala | 26 +- .../GroupedAggregateSemanticsSpec.scala | 13 +- .../client/NonDocMappingTypeSpec.scala | 4 +- .../client/PredicateFunctionResultSpec.scala | 8 +- .../SchemaMissAfterCreateTableSpec.scala | 4 +- .../elastic/client/SchemaReadingSpec.scala | 388 +++++++++ .../elastic/client/SetOperationTypeSpec.scala | 70 +- .../elastic/client/TemporalLiteralSpec.scala | 4 +- .../client/TypesPredicatesFunctionsSpec.scala | 4 +- .../UnnestFunctionRefusalExecutionSpec.scala | 4 +- .../UnnestInnerHitsSizeExecutionSpec.scala | 4 +- .../elastic/client/WindowFunctionSpec.scala | 4 +- .../repl/ReplGatewayIntegrationSpec.scala | 16 +- .../client/repl/ReplIntegrationTestKit.scala | 2 +- 77 files changed, 4160 insertions(+), 478 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/SchemaPageFixture.scala create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSchemaReadingSpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala diff --git a/build.sbt b/build.sbt index 00faa08a9..810fdba79 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.24.0" +ThisBuild / version := "0.25.0-SNAPSHOT" ThisBuild / scalaVersion := scala213 diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 567ab58e7..5e0d62d79 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -1529,6 +1529,14 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { context: ConversionContext ): Source[(ListMap[String, Any], ScrollMetrics), NotUsed] = delegate.scroll(statement, config) + /** SCHEMA-2 (D13) -- the delegate's: its schema cache, its reload memo. */ + override private[client] def scrollWithSchema(statement: SearchStatement, config: ScrollConfig)( + implicit + system: ActorSystem, + context: ConversionContext + ): (Option[ResultSchema], Source[(ListMap[String, Any], ScrollMetrics), NotUsed]) = + delegate.scrollWithSchema(statement, config) + /** Scroll and convert results into typed entities from an SQL query. * * @note @@ -2005,6 +2013,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): Future[ElasticResult[QueryResult]] = delegate.run(statement) + /** SCHEMA-2 (D14) -- the delegate's: its extensions, its schema cache, its reload memo. */ + override def describe(sql: String): ElasticResult[Option[ResultSchema]] = + delegate.describe(sql) + + override def describe(statement: query.Statement): ElasticResult[Option[ResultSchema]] = + delegate.describe(statement) + // ==================== Extensions (delegate) ==================== /** #163 — the delegator inherits ExtensionApi's lazy vals, so `this.extensionRegistry` would be a diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index 31ead76d4..1aa2eaa1a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -433,8 +433,8 @@ trait ElasticConversion { // the levels the statement sends as a numeric `terms` of doubles (`SingleSearch.doubleKeyLevels`) val doubleKeys: Set[String] = context match { - case TextColumns(_, keys) => keys - case _ => Set.empty + case s: SchemaColumns => s.doubleKeys + case _ => Set.empty } val rows = (hitsNode, aggsNode) match { @@ -670,19 +670,23 @@ trait ElasticConversion { )(implicit context: ConversionContext): Seq[ListMap[String, Any]] = { // One decision per parse — the per-row hot path does no `_id` work when it is not kept val withDocumentId = retainDocumentId || keepsDocumentId(requestedFields) - // ...and none for the columns declared VARCHAR unless the statement declares some - val text: Set[String] = context match { - case TextColumns(columns, _) => columns - case _ => Set.empty + context match { + // SCHEMA-2 (D4) -- every value read by its column's declared type, where it is first read + case s: SchemaColumns => parseHitsByPlan(hits, fieldAliases, withDocumentId, s.reading) + case _ => parseHitsAsRead(hits, fieldAliases, withDocumentId) } + } + + /** The hits of a statement read in a context that types none of its values (an entity, a + * document, a window-enriched statement): every value as `jsonNodeToAny` reads it. + */ + private def parseHitsAsRead( + hits: List[JsonNode], + fieldAliases: ListMap[String, String], + withDocumentId: Boolean + ): Seq[ListMap[String, Any]] = hits.map { hit => - var source = - if (text.isEmpty) extractSource(hit, fieldAliases) - else - Option(hit.get("_source")) - .filter(_.isObject) - .map(jsonNodeToMap(_, fieldAliases, text)) - .getOrElse(ListMap.empty) + var source = extractSource(hit, fieldAliases) fieldAliases.foreach(entry => { if (!source.contains(entry._2)) { findKeyValue(entry._1, source) match { @@ -697,13 +701,133 @@ trait ElasticConversion { val fieldsNode = Option(hit.path("fields")) .filter(!_.isMissingNode) val fields = fieldsNode - .map(n => - if (text.isEmpty) jsonNodeToMap(n, fieldAliases) - else jsonNodeToMap(n, fieldAliases, text) - ) + .map(n => jsonNodeToMap(n, fieldAliases)) .getOrElse(ListMap.empty) source ++ metadata ++ innerHits ++ fields } + + /** SCHEMA-2 (D4, D6-D8) -- the hits of a statement read through its plan ([[HitReading]]): each + * `_source` key the projection touches, each computed column's `fields` entry and each UNNEST + * element field by its column's reader, before any re-read as a temporal; every other key as it + * always was. A date column the hit's `_ignored` names follows Elasticsearch's verdict, per + * value: a cell of one value is NULL, an array keeps the elements its format reads and answers + * the others NULL ([[HitReading.onIgnored]]). A several-element cell a scalar column refuses + * ([[MultiValued.Fail]]) fails the statement, naming the document. + */ + private def parseHitsByPlan( + hits: List[JsonNode], + fieldAliases: ListMap[String, String], + withDocumentId: Boolean, + reading: HitReading + ): Seq[ListMap[String, Any]] = + hits.map { hit => + try { + val ignored: java.util.Set[String] = + if (reading.readsIgnored) ignoredOf(hit) else null + var source = hit.get("_source") match { + case node if node != null && node.isObject => + readSource(node, fieldAliases, reading, ignored) + case _ => ListMap.empty[String, Any] + } + fieldAliases.foreach(entry => { + if (!source.contains(entry._2)) { + findKeyValue(entry._1, source) match { + case Some(value) => source += (entry._2 -> value) + case None => + } + } + }) + val metadata = + if (withDocumentId) extractHitId(hit) else ListMap.empty[String, Any] + val innerHits = extractInnerHits(hit, fieldAliases, reading.inner) + val fieldsNode = hit.get("fields") + val fields = + if (fieldsNode != null && fieldsNode.isObject) + readScripted(fieldsNode, fieldAliases, reading) + else ListMap.empty[String, Any] + source ++ metadata ++ innerHits ++ fields + } catch { + case cell: HitReading.MultiValuedCell => + throw new IllegalArgumentException( + HitReading.multiValuedMessage(cell, Option(hit.get("_id")).map(_.asText())) + ) + } + } + + /** The paths a hit's `_ignored` names (D8): `null` when it names none. */ + private def ignoredOf(hit: JsonNode): java.util.Set[String] = { + val node = hit.get("_ignored") + if (node == null || !node.isArray || node.size() == 0) null + else { + val paths = new java.util.HashSet[String]() + val it = node.elements() + while (it.hasNext) paths.add(it.next().asText()) + paths + } + } + + /** A hit's `_source` through the plan (D4): a key it holds by its reader, a flat dotted key under + * `*` folded into its mapped parent (a non-null flat value wins, A-3), an unmapped key under `*` + * dropped, any other key as read. + */ + private def readSource( + node: JsonNode, + fieldAliases: ListMap[String, String], + reading: HitReading, + ignored: java.util.Set[String] + ): ListMap[String, Any] = { + val entries = ListMap.newBuilder[String, Any] + var folds: ListBuffer[(String, String, JsonNode)] = null + val it = node.properties().iterator() + while (it.hasNext) { + val e = it.next() + val key = e.getKey + val reader = reading.source.get(key) + if (reader ne null) { + val onIgnored = + if ((ignored ne null) && ignored.contains(key)) reading.onIgnored(key) else null + val value = if (onIgnored ne null) onIgnored(e.getValue) else reader(e.getValue) + entries += fieldAliases.getOrElse(key, key) -> value + } else { + val parent = reading.foldsUnder(key) + if (parent ne null) { + if (folds eq null) folds = ListBuffer.empty + folds += ((parent, key, e.getValue)) + } else if (!reading.star) + entries += fieldAliases.getOrElse(key, key) -> jsonNodeToAny(e.getValue, fieldAliases) + } + } + val row = entries.result() + if (folds eq null) row + else + folds.foldLeft(row) { case (acc, (parent, key, value)) => + reading.flatReaders(key)(value) match { + case null => acc + case v => + val path = key.substring(parent.length + 1).split("\\.").toList + acc + (parent -> ElasticConversion.withPath(acc.getOrElse(parent, null), path, v)) + } + } + } + + /** A hit's `fields` through the plan: a computed column's entry by its reader (one value, not + * Elasticsearch's array), any other entry as read. + */ + private def readScripted( + node: JsonNode, + fieldAliases: ListMap[String, String], + reading: HitReading + ): ListMap[String, Any] = { + val entries = ListMap.newBuilder[String, Any] + val it = node.properties().iterator() + while (it.hasNext) { + val e = it.next() + val reader = reading.scripted.get(e.getKey) + entries += fieldAliases.getOrElse(e.getKey, e.getKey) -> ( + if (reader ne null) reader(e.getValue) else jsonNodeToAny(e.getValue, fieldAliases) + ) + } + entries.result() } /** Normalize a row to ensure all requested fields are present in the original SQL SELECT order. @@ -860,6 +984,15 @@ trait ElasticConversion { case EntityContext => false case _ => true } + // SCHEMA-2 (D9) -- a hit-row statement read by its plan is projected STRICTLY: an entry no + // column declares is dropped, but `_id` and the nested-hit keys (`HitReading.keep`). Decided + // once per stream; `null` (every extra kept, as before) for every other context. + val strictKeep: java.util.Set[String] = context match { + case s: SchemaColumns if s.reading.strict => + new java.util.HashSet[String](java.util.Arrays.asList(s.reading.keep.toSeq: _*)) + case _ => null + } + val strict = strictKeep ne null // a converted value changes the row, so a converting projection never hands a row back as is val rebuilds = renames || (convs ne null) if (fieldIndex.size() != len) { @@ -873,7 +1006,18 @@ trait ElasticConversion { // hand-written duplicate projection reaches. if (!renames) { val requestedSet = sourceFields.toSet - if (convs eq null) row => normalizeRowOrdered(row, sourceFields, requestedSet) + if (strict) + row => + normalizeRowOrdered(row, sourceFields, requestedSet) + .filter(entry => requestedSet.contains(entry._1) || strictKeep.contains(entry._1)) + .map { entry => + if (convs eq null) entry + else { + val j = sourceFields.indexOf(entry._1) + if (j >= 0 && (convs(j) ne null)) entry._1 -> convs(j)(entry._2) else entry + } + } + else if (convs eq null) row => normalizeRowOrdered(row, sourceFields, requestedSet) else { // the column a repeated name keeps is its FIRST position, and so is its conversion val convertByName = new java.util.HashMap[String, Any => Any](len * 2) @@ -906,7 +1050,10 @@ trait ElasticConversion { j += 1 } row.foreach { entry => - if (!sourceSet.contains(entry._1) && !targetNames.contains(entry._1)) + if ( + !sourceSet.contains(entry._1) && !targetNames.contains(entry._1) && + (!strict || strictKeep.contains(entry._1)) + ) builder += entry } builder.result() @@ -944,12 +1091,23 @@ trait ElasticConversion { seen(idx.intValue) = true } else if (renames && targetNames.contains(entry._1)) { // a stray entry wearing a result column's name — dropped, see `targetNames` - } else { + } else if (!strict || strictKeep.contains(entry._1)) { if (extras eq null) extras = new ListBuffer[(String, Any)] extras += entry } } } + // SCHEMA-2 (D9) -- under the strict projection, the entries after an in-order match are + // read here, OFF the per-entry path above (#354's +3.6 %): an undeclared one drops it + var dropped = false + if (strict && !rebuilds && passthrough) + while (it.hasNext) { + val entry = it.next() + if (strictKeep.contains(entry._1)) { + if (extras eq null) extras = new ListBuffer[(String, Any)] + extras += entry + } else dropped = true + } // Under a rename -- or a conversion -- the loop may have stopped on `passthrough` with // entries still to come; none of them can be a source name (a row map's keys are unique // and all `len` of them were just consumed), so they are extras -- subject, under a @@ -957,14 +1115,17 @@ trait ElasticConversion { if (rebuilds) { while (it.hasNext) { val entry = it.next() - if (!renames || !targetNames.contains(entry._1)) { + if ( + (!renames || !targetNames.contains(entry._1)) && + (!strict || strictKeep.contains(entry._1)) + ) { if (extras eq null) extras = new ListBuffer[(String, Any)] extras += entry } } } // An in-order strict prefix needs no rebuild either when missing fields are skipped - if (!rebuilds && (passthrough || (inOrder && !nullFillMissing))) row + if (!rebuilds && !dropped && (passthrough || (inOrder && !nullFillMissing))) row else { val builder = ListMap.newBuilder[String, Any] var j = 0 @@ -1080,12 +1241,15 @@ trait ElasticConversion { /** Extract inner_hits from a hit (for nested or parent-child queries) */ private def extractInnerHits( hit: JsonNode, - fieldAliases: ListMap[String, String] + fieldAliases: ListMap[String, String], + readers: Map[String, java.util.HashMap[String, JsonNode => Any]] = Map.empty ): ListMap[String, Any] = { Option(hit.get("inner_hits")) match { case Some(n: ObjectNode) => var entries: Seq[(String, Any)] = Seq.empty n.forEachEntry { case (innerHitName, innerHitData) => + // SCHEMA-2 (D4) -- an UNNEST element's fields by their columns' readers + val byField = readers.getOrElse(innerHitName, null) // Extract the hits array from inner_hits val innerHitsList = Option(innerHitData.path("hits").path("hits")) .filter(_.isArray) @@ -1097,12 +1261,14 @@ trait ElasticConversion { // Inner-hit `_id` is never needed internally: only expose it when the // document-id column is enabled (nested rows are out of reach of the // top-level egress strip). - val source = extractSource(innerHit, fieldAliases) + val source = + if (byField eq null) extractSource(innerHit, fieldAliases) + else readElement(innerHit, fieldAliases, byField) val metadata = if (includeDocumentId) extractHitId(innerHit) else ListMap.empty[String, Any] // Recursively handle nested inner_hits if present - val nestedInnerHits = extractInnerHits(innerHit, fieldAliases) + val nestedInnerHits = extractInnerHits(innerHit, fieldAliases, readers) source ++ metadata ++ nestedInnerHits } @@ -1117,6 +1283,27 @@ trait ElasticConversion { } } + /** An inner hit's `_source`, its element fields by their readers, every other key as read. */ + private def readElement( + innerHit: JsonNode, + fieldAliases: ListMap[String, String], + byField: java.util.HashMap[String, JsonNode => Any] + ): ListMap[String, Any] = + innerHit.get("_source") match { + case node if node != null && node.isObject => + val entries = ListMap.newBuilder[String, Any] + val it = node.properties().iterator() + while (it.hasNext) { + val e = it.next() + val reader = byField.get(e.getKey) + entries += fieldAliases.getOrElse(e.getKey, e.getKey) -> ( + if (reader ne null) reader(e.getValue) else jsonNodeToAny(e.getValue, fieldAliases) + ) + } + entries.result() + case _ => ListMap.empty + } + /** Parse recursively aggregations from Elasticsearch response with parent context */ /** @param docCount @@ -1920,36 +2107,6 @@ trait ElasticConversion { } } - /** [[jsonNodeToMap]], with the top-level entries named in `text` -- the columns a statement - * declares VARCHAR ([[TextColumns]]) -- read as their text, never re-read as a temporal. - */ - private[client] def jsonNodeToMap( - node: JsonNode, - fieldAliases: ListMap[String, String], - text: Set[String] - ): ListMap[String, Any] = - node match { - case n: ObjectNode => - var entries: Seq[(String, Any)] = Seq.empty - n.forEachEntry { (name, e) => - val column = fieldAliases.getOrElse(name, name) - val value = - if (text.contains(column) || text.contains(name)) jsonNodeToText(e, fieldAliases) - else jsonNodeToAny(e, fieldAliases) - entries ++= Seq(column -> value) - } - ListMap(entries: _*) - case _ => - ListMap.empty - } - - /** [[jsonNodeToAny]] with no temporal inference: a text is its own value. */ - private def jsonNodeToText(node: JsonNode, fieldAliases: ListMap[String, String]): Any = - if (node != null && node.isTextual) node.asText() - else if (node != null && node.isArray) - node.elements().asScala.map(jsonNodeToText(_, fieldAliases)).toList - else jsonNodeToAny(node, fieldAliases) - /** Convert a JsonNode to Any (primitive types, List, Map) */ def jsonNodeToAny(node: JsonNode, fieldAliases: ListMap[String, String]): Any = { @@ -1987,6 +2144,21 @@ object ElasticConversion extends ElasticConversion { */ val DocumentIdField: String = "_id" + /** `current` (a map, or nothing) with `value` placed at `path` (SCHEMA-2, A-3: a flat dotted key + * folded into its parent). + */ + private[client] def withPath(current: Any, path: List[String], value: Any): Any = + path match { + case Nil => value + case head :: tail => + current match { + case m: Map[_, _] => + val map = m.asInstanceOf[Map[String, Any]] + map + (head -> withPath(map.getOrElse(head, null), tail, value)) + case _ => ListMap(head -> withPath(null, tail, value)) + } + } + /** The texts a response writes a non-finite double key as (`extractBucketKey`). */ private[client] val NonFiniteKeys: Set[String] = Set("Infinity", "-Infinity", "NaN") diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index c1e6b9094..c646145e0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -35,6 +35,7 @@ import app.softnetwork.elastic.client.result.{ } import app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.client.extensions.CoreDqlExtension import app.softnetwork.elastic.sql.query.{ AlterTable, AlterTableSetting, @@ -157,8 +158,10 @@ class SearchExecutor(api: ScrollApi with SearchApi, logger: Logger) } } else { logger.info(s"▶ Executing scroll search on index ${single.from.tables.mkString(",")}") + // SCHEMA-2 (D13) -- the stream carries the statement's schema, known before any row + val (schema, source) = api.scrollWithSchema(single) Future.successful( - ElasticSuccess(QueryStream(api.scroll(single))) + ElasticSuccess(QueryStream(source, schema = schema)) ) } @@ -2026,7 +2029,12 @@ class FromlessSelectExecutor( case ElasticSuccess(row) => val offset = statement.limit.flatMap(_.offset).map(_.offset).getOrElse(0) val max = statement.limit.map(_.limit).getOrElse(1) - ElasticSuccess(QueryRows(Seq(row).drop(offset).take(max))) + ElasticSuccess( + QueryRows( + Seq(row).drop(offset).take(max), + schema = Some(ResultSchemas.of(statement)) + ) + ) case ElasticFailure(error) => ElasticFailure(error) } } @@ -2272,6 +2280,81 @@ trait GatewayApi extends IndicesApi with ElasticClientHelpers { } } + /** SCHEMA-2 (D14, A-19) -- the result schema `run(sql)` would carry, running NOTHING: no search, + * no subquery; a mapping may be read (the attach, and an explicit path's reload, D10). ONE query + * -- a SELECT, a set operation, a FROM-less SELECT -- else a 400. A statement another extension + * than core's handles answers `None` (its layer types it); a statement `run` refuses before + * executing anything is refused with `run`'s status and message, through the SAME resolve body + * (`resolveWithSchema(_, executeSubqueries = false)`). A refusal that needs a subquery's values + * or Elasticsearch surfaces at `run` only. `None` where no schema is known. + */ + def describe(sql: String): ElasticResult[Option[ResultSchema]] = + GatewayApi.splitStatements(sql) match { + case statement :: Nil => + Parser(statement) match { + case Right(parsed) => describe(parsed) + case Left(l) => + ElasticResult.failure( + ElasticError( + message = GatewayApi.parseRejectionMessage(statement, l.msg), + cause = l.cause, + statusCode = Some(400), + operation = Some("describe") + ) + ) + } + case _ => describeRefusal("DESCRIBE answers exactly one query") + } + + /** [[describe]] over a parsed statement. */ + def describe(statement: Statement): ElasticResult[Option[ResultSchema]] = + statement match { + case _: SelectStatement | _: SingleSearch | _: MultiSearch | _: FromlessSelect => + extensionRegistry.findHandler(statement) match { + case Some(extension) if !extension.isInstanceOf[CoreDqlExtension] => + ElasticResult.success(None) + case _ => describeQuery(statement) + } + case _ => + describeRefusal( + "DESCRIBE answers a query only: a SELECT, a set operation or a FROM-less SELECT" + ) + } + + /** [[describe]] over a query core's own handler runs. */ + private def describeQuery(statement: Statement): ElasticResult[Option[ResultSchema]] = + statement match { + case select: SelectStatement => + select.statement match { + case Some(single: SingleSearch) => describeQuery(single.copy(score = select.score)) + case Some(multiple: MultiSearch) => describeQuery(multiple) + case _ => + describeRefusal( + select.parseError.getOrElse( + s"SELECT statement could not be translated into a search query: $statement" + ) + ) + } + case single: SingleSearch => + resolveWithSchema(single, executeSubqueries = false).map(statementSchema) + case multiple: MultiSearch => + resolveWithSchema(multiple, executeSubqueries = false).map(unionAllSchema) + case fromless: FromlessSelect => + fromless.validate() match { + case Left(reason) => describeRefusal(reason) + case Right(_) => ElasticResult.success(Some(ResultSchemas.of(fromless))) + } + case _ => + describeRefusal( + "DESCRIBE answers a query only: a SELECT, a set operation or a FROM-less SELECT" + ) + } + + private def describeRefusal(message: String): ElasticResult[Option[ResultSchema]] = + ElasticResult.failure( + ElasticError(message = message, statusCode = Some(400), operation = Some("describe")) + ) + private def dispatch( statement: Statement )(implicit system: ActorSystem, ec: ExecutionContext): Future[ElasticResult[QueryResult]] = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala new file mode 100644 index 000000000..99d3ced51 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala @@ -0,0 +1,793 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.query.{ + Field, + GroupedRowOrder, + MultiSearch, + SingleSearch, + UnnestScope +} +import app.softnetwork.elastic.sql.query.MultiSearch.SetOperationType +import com.fasterxml.jackson.databind.JsonNode + +import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap +import scala.collection.mutable.ListBuffer + +/** SCHEMA-2 (D7) -- what a SEVERAL-element cell in a SCALAR column answers. */ +private[client] sealed trait MultiValued + +private[client] object MultiValued { + + /** The cell is a `List` of its elements, each read by the column's type (R-22). */ + case object AsRead extends MultiValued + + /** The statement fails, naming the column, the document and the `ALTER` that declares it a list. + */ + case object Fail extends MultiValued +} + +/** SCHEMA-2 (D4-D9) -- how ONE statement reads the values of its hits: by its columns' declared + * types, where each value is first read (a Jackson node), before any re-read as a temporal. + * + * - `source`: the top-level `_source` keys the projection touches -- a bare column, a dotted + * column's parent and its flat spelling (`"obj.a"`), an alias source, every mapped column for + * `*` -- each with the reader of its type; + * - `scripted`: a computed column's hit `fields` entry, by output name (one-element array + * unwrapped); + * - `inner`: an UNNEST inner hit's element fields, by inner-hit name then field; + * - `ignoredDates`: the `_source` keys read as dates by their format, each with the reader of + * its cell where the hit's `_ignored` names it (D8): Elasticsearch's verdict is per VALUE, so + * a cell of one value is NULL and an array keeps the elements the format reads, the others + * NULL; + * - `strict` / `keep`: the projection drops every undeclared key, but `keep` (D9); + * - `star`: a bare `SELECT *` -- unmapped top-level keys dropped, a flat dotted key folded into + * its mapped parent (A-3). + * + * A key the plan does not hold is read as it always was (`ElasticConversion.jsonNodeToAny`). Built + * ONCE per statement: per row, only the readers run. + */ +private[client] final class HitReading( + val schema: Option[ResultSchema], + private[client] val source: java.util.HashMap[String, JsonNode => Any], + private[client] val scripted: java.util.HashMap[String, JsonNode => Any], + private[client] val inner: Map[String, java.util.HashMap[String, JsonNode => Any]], + private[client] val ignoredDates: java.util.HashMap[String, JsonNode => Any], + val strict: Boolean, + val keep: Set[String], + private[client] val star: Boolean, + private[client] val flatReaders: String => JsonNode => Any +) { + + /** Whether a hit's `_ignored` must be read (D8): the plan reads some `_source` key as a date. */ + private[client] val readsIgnored: Boolean = !ignoredDates.isEmpty + + /** The reader of `key`'s cell when the hit's `_ignored` names it (D8); `null` for a key that is + * not read as a date, which keeps its own reader (a keyword keeps its stored text). + */ + private[client] def onIgnored(key: String): JsonNode => Any = ignoredDates.get(key) + + /** Whether a (non-plan) top-level key is folded into its mapped parent under `*` (A-3). */ + private[client] def foldsUnder(key: String): String = { + val dot = key.indexOf('.') + if (star && dot > 0) { + val parent = key.substring(0, dot) + if (source.containsKey(parent)) parent else null + } else null + } +} + +private[client] object HitReading { + + /** SCHEMA-6 flips it to [[MultiValued.Fail]], in the same core release (D7). */ + private[client] val Default: MultiValued = MultiValued.AsRead + + /** A bare `*` item. */ + private[client] def isStar(field: Field): Boolean = + field.identifier.identifierName == "*" && field.identifier.functions.isEmpty + + /** Thrown by a scalar reader under [[MultiValued.Fail]]; the hit loop names the document. */ + private[client] final class MultiValuedCell(val column: String, val table: String) + extends RuntimeException(column, null, false, false) + + /** The message a several-element cell fails with under [[MultiValued.Fail]] (D7). */ + private[client] def multiValuedMessage(cell: MultiValuedCell, id: Option[String]): String = + s"Column ${cell.column} holds several values${id.fold("")(i => s" in document $i")}, but is " + + s"declared as one value: declare it a list with ALTER TABLE ${cell.table} ALTER COLUMN " + + s"${cell.column} TYPE ARRAY<...>" + + def of( + single: SingleSearch, + dates: SetOperationValues.DateReading, + multiValued: MultiValued = Default, + elements: Set[String] = Set.empty + ): HitReading = { + val schema = ResultSchemas.of(single, dates) + val table = single.sources.headOption.getOrElse("") + val aliases = single.fieldAliases + val readers = new Readers(aliases, dates, multiValued, table) + val fields = single.select.fieldsWithComputedAliases + val star = fields.exists(isStar) + val source = new java.util.HashMap[String, JsonNode => Any]() + val scripted = new java.util.HashMap[String, JsonNode => Any]() + val ignoredDates = new java.util.HashMap[String, JsonNode => Any]() + // the dotted paths the projection selects, relative to their parent (D4) + val selectedPaths: Map[String, Seq[String]] = fields + .filterNot(f => isStar(f) || f.identifier.nested || f.identifier.functions.nonEmpty) + .map(_.identifier.name) + .filter(n => n.indexOf('.') > 0) + .groupBy(n => n.substring(0, n.indexOf('.'))) + .map { case (parent, paths) => parent -> paths.map(_.substring(parent.length + 1)) } + // a parent read whole (`SELECT obj` or `*`) keeps the undeclared keys spelling a selected path + def keepUnder(parent: String): Set[String] = + selectedPaths + .getOrElse(parent, Nil) + .flatMap { p => + val segments = p.split("\\.") + (1 to segments.length).map(i => segments.take(i).mkString(".")) + } + .toSet + def put(key: String, t: SetOperationType, keepKeys: Set[String]): Unit = { + val (reader, onIgnored) = readers.column(key, t, keepKeys) + source.put(key, reader) + onIgnored.foreach(ignoredDates.put(key, _)) + } + single.schema.foreach { table => + if (star) + table.columns.foreach { c => + if (!source.containsKey(c.name)) + put(c.name, MultiSearch.columnType(c), keepUnder(c.name)) + } + } + // explicit columns: a bare column (or a dotted column's parent and its flat spelling) by its + // type; a computed column's `fields` entry by output name + val paths = + scala.collection.mutable.LinkedHashMap.empty[String, ListBuffer[(String, SetOperationType)]] + fields.filterNot(isStar).foreach { f => + val id = f.identifier + if (id.functions.nonEmpty || f.isScriptField) { + val name = f.outputName + val t = MultiSearch.fieldType(f) + if (name.nonEmpty && !scripted.containsKey(name)) { + if (single.schema.isDefined) scripted.put(name, readers.scripted(name, t)) + // no schema (D10): rows as today -- a column declared VARCHAR is its text, never re-read + // as a temporal, Elasticsearch's array kept + else if (t.sqlType == SQLTypes.Varchar) scripted.put(name, readers.textAsRead) + } + } else if (!id.nested && id.name.nonEmpty && single.schema.isDefined) { + val name = id.name + val t = MultiSearch.fieldType(f) + val dot = name.indexOf('.') + if (dot < 0) { + if (!source.containsKey(name)) put(name, t, keepUnder(name)) + } else { + val parent = name.substring(0, dot) + paths.getOrElseUpdate(parent, ListBuffer.empty) += (name.substring(dot + 1) -> t) + // SCHEMA-12 A-3's flat spelling, read first by `flatValue` + if (!source.containsKey(name)) { + val (reader, _) = readers.column(name, t, Set.empty) + source.put(name, reader) + } + } + } + } + // a dotted column's parent not read whole: its selected paths by their types, every other key + // as read + paths.foreach { case (parent, under) => + if (!source.containsKey(parent)) source.put(parent, readers.paths(parent, under.toList)) + } + // UNNEST element fields, by inner hit; a column with an element conversion keeps its raw value + val inner: Map[String, java.util.HashMap[String, JsonNode => Any]] = + if (single.schema.isEmpty) Map.empty + else + (single.nestedHitsMappings ++ UnnestScope.rowExplosion(single)).flatMap { + case (hit, mappings) => + val byField = new java.util.HashMap[String, JsonNode => Any]() + mappings.foreach { case (subField, output) => + if (subField.indexOf('.') < 0 && !elements.contains(output)) + fields.find(_.outputName == output).foreach { f => + val t = MultiSearch.fieldType(f) + if (readers.typed(t)) + byField.put(subField, readers.column(output, t, Set.empty)._1) + } + } + if (byField.isEmpty) None else Some(hit -> byField) + } + val strict = + schema.isDefined && !single.windowRowQuery && single.sqlAggregations.isEmpty && + GroupedRowOrder.of(single).isEmpty && !fields.exists(isStar) + // `_id` when it is kept, and every inner hit the row explosion reads AFTER the scroll's + // normalizer -- an UNNEST its WHERE filters per element included (`UnnestScope.rowExplosion`) + val keep: Set[String] = Set(ElasticConversion.DocumentIdField) ++ + single.nestedHitsMappings.keySet ++ UnnestScope.rowExplosion(single).keySet + // a flat dotted key folded under `*` (rare): its reader built once per path, on first use + val flatCache = new java.util.concurrent.ConcurrentHashMap[String, JsonNode => Any]() + val flatReaders: String => JsonNode => Any = path => { + val cached = flatCache.get(path) + if (cached ne null) cached + else { + val reader = single.schema.flatMap(_.find(path)) match { + case Some(c) => readers.column(path, MultiSearch.columnType(c), Set.empty)._1 + case None => readers.asRead + } + flatCache.putIfAbsent(path, reader) + reader + } + } + new HitReading( + schema, + source, + scripted, + inner, + ignoredDates, + strict, + keep, + star && schema.isDefined && fields.forall(isStar), + flatReaders + ) + } + + /** The readers of one statement (D5), built once: `JsonNode => Any`, a node never `null` (the + * caller hands `null` / a JSON `null` to none of them but the scalar wrapper). + */ + private[client] final class Readers( + aliases: ListMap[String, String], + dates: SetOperationValues.DateReading, + multiValued: MultiValued, + table: String + ) { + + /** Today's reading, the plan's answer for a key it does not type (ANY, NUMERIC, TEMPORAL) and + * for a value a typed reader cannot read (D6, `Unreadable.KeepAsStored`). + */ + val asRead: JsonNode => Any = node => ElasticConversion.jsonNodeToAny(node, aliases) + + /** Today's reading with no temporal inference: a text is its own value, an array a list. */ + val textAsRead: JsonNode => Any = node => + if (node == null || node.isNull) null + else if (node.isTextual) node.asText() + else if (node.isArray) { + val values = ListBuffer.empty[Any] + val it = node.elements() + while (it.hasNext) values += textAsRead(it.next()) + values.toList + } else asRead(node) + + /** Whether a type is read by a typed reader: anything but a type read as it comes. */ + def typed(t: SetOperationType): Boolean = + t.sqlType match { + case SQLTypes.Any | SQLTypes.Numeric | SQLTypes.Temporal | SQLTypes.Null => false + case SQLTypes.Struct => t.fields.nonEmpty + case _ => true + } + + /** A `_source` column of type `t`, and -- for a date read by its format -- the reader of its + * cell when the hit's `_ignored` names it (D8): Elasticsearch refused some value of it, and + * its verdict is per VALUE. A cell of one value is NULL (the value refused); an array keeps + * each element the format reads -- a value Elasticsearch indexed -- and answers the others + * NULL. + */ + def column( + name: String, + t: SetOperationType, + keepKeys: Set[String] + ): (JsonNode => Any, Option[JsonNode => Any]) = + t.sqlType match { + case SQLTypes.Array(_) => + (list(name, t.element.getOrElse(SetOperationType(SQLTypes.Any))), None) + case SQLTypes.Struct if t.fields.nonEmpty => (scalar(name, struct(name, t, keepKeys)), None) + case SQLTypes.GeoPoint => (geo(name), None) + case _ if !typed(t) => (asRead, None) + case scalarType => + val (one, ignoredOne) = scalarOne(name, scalarType, t.format) + ( + scalar(name, one), + ignoredOne.map(element => + (node: JsonNode) => + if (node != null && node.isArray) scalarValue(name, element, node) else null + ) + ) + } + + /** A computed column's `fields` entry: Elasticsearch's array unwrapped (#418 part 1). */ + def scripted(name: String, t: SetOperationType): JsonNode => Any = + t.sqlType match { + case SQLTypes.Array(_) => column(name, t, Set.empty)._1 + case _ if !typed(t) => scalar(name, asRead) + case _ => column(name, t, Set.empty)._1 + } + + /** A dotted column's parent read for its selected paths only: each by its type, every other key + * as read, in source order (D4). + */ + def paths(name: String, under: List[(String, SetOperationType)]): JsonNode => Any = { + val leaves = new java.util.HashMap[String, JsonNode => Any]() + val nested = + scala.collection.mutable.LinkedHashMap.empty[String, ListBuffer[(String, SetOperationType)]] + under.foreach { case (p, t) => + val dot = p.indexOf('.') + if (dot < 0) leaves.put(p, column(s"$name.$p", t, Set.empty)._1) + else + nested.getOrElseUpdate(p.substring(0, dot), ListBuffer.empty) += (p.substring( + dot + 1 + ) -> t) + // the flat spelling of a deeper path under this parent (`obj: {"x.y": 10}`) + if (dot >= 0 && !leaves.containsKey(p)) leaves.put(p, column(s"$name.$p", t, Set.empty)._1) + } + nested.foreach { case (k, sub) => + if (!leaves.containsKey(k)) leaves.put(k, paths(s"$name.$k", sub.toList)) + } + node => + if (node.isObject) { + val entries = ListMap.newBuilder[String, Any] + val it = node.properties().iterator() + while (it.hasNext) { + val e = it.next() + val reader = leaves.get(e.getKey) + entries += aliases.getOrElse(e.getKey, e.getKey) -> ( + if (reader ne null) reader(e.getValue) else asRead(e.getValue) + ) + } + entries.result() + } else asRead(node) + } + + /** ONE value of a scalar column: `[v]` is `v` -- `[[v]]` too --, `[]` NULL; several values per + * [[MultiValued]]. An array of one holding several values (`[[a, b]]`) is as stored (D6): its + * element is no value of the column, and unwrapping it would drop the nesting it was stored + * with. + */ + def scalar(name: String, one: JsonNode => Any): JsonNode => Any = + node => scalarValue(name, one, node) + + private def scalarValue(name: String, one: JsonNode => Any, node: JsonNode): Any = + if (node == null || node.isNull) null + else if (node.isArray) { + val size = node.size() + if (size == 0) null + else if (size == 1) { + val only = node.get(0) + if (only != null && only.isArray) { + if (only.size() == 1) scalarValue(name, one, only) + else if (only.size() > 1 && multiValued == MultiValued.Fail) + throw new MultiValuedCell(name, table) + else asRead(node) + } else element(one)(only) + } else + multiValued match { + case MultiValued.AsRead => + val values = ListBuffer.empty[Any] + val it = node.elements() + while (it.hasNext) values += element(one)(it.next()) + values.toList + case MultiValued.Fail => throw new MultiValuedCell(name, table) + } + } else one(node) + + private def element(one: JsonNode => Any)(node: JsonNode): Any = + if (node == null || node.isNull) null else one(node) + + /** An `ARRAY`: a `List` of its element reads, a scalar a list of one. */ + def list(name: String, element: SetOperationType): JsonNode => Any = { + val one: JsonNode => Any = element.sqlType match { + case SQLTypes.Struct if element.fields.nonEmpty => struct(name, element, Set.empty) + case SQLTypes.GeoPoint => geoOne(name) + case SQLTypes.Array(_) => + list(name, element.element.getOrElse(SetOperationType(SQLTypes.Any))) + case _ if !typed(element) => asRead + case scalarType => scalarOne(name, scalarType, element.format)._1 + } + node => + if (node == null || node.isNull) null + else if (node.isArray) { + val values = ListBuffer.empty[Any] + val it = node.elements() + while (it.hasNext) { + val e = it.next() + values += (if (e == null || e.isNull) null else one(e)) + } + values.toList + } else List(one(node)) + } + + /** A `STRUCT` with declared fields: those fields by name, each by its type, in source order; an + * undeclared key dropped unless it spells a selected path (`keepKeys`), then as read. + */ + def struct(name: String, t: SetOperationType, keepKeys: Set[String]): JsonNode => Any = { + val fieldReaders = new java.util.HashMap[String, JsonNode => Any]() + t.fields.foreach { f => + val under = keepKeys.collect { + case k if k.startsWith(f.name + ".") => k.substring(f.name.length + 1) + } + fieldReaders.put(f.name, column(s"$name.${f.name}", f.valueType, under)._1) + } + node => + if (node.isObject) { + val entries = ListMap.newBuilder[String, Any] + val it = node.properties().iterator() + while (it.hasNext) { + val e = it.next() + val reader = fieldReaders.get(e.getKey) + if (reader ne null) entries += e.getKey -> reader(e.getValue) + else if (keepKeys.contains(e.getKey)) entries += e.getKey -> asRead(e.getValue) + } + entries.result() + } else asRead(node) + } + + /** A `GEO_POINT` column: `{lat, lon}` (`SetOperationValues.geoPoint`) on a temporal-free value; + * `[lon, lat]` is ONE point, several points per [[MultiValued]]; a value that is not a point + * as read (D6). + */ + def geo(name: String): JsonNode => Any = { + val one = geoOne(name) + node => + if (node == null || node.isNull) null + else if (node.isArray && !coordinates(node)) { + val size = node.size() + if (size == 0) null + else if (size == 1) element(one)(node.get(0)) + else + multiValued match { + case MultiValued.AsRead => + try { + val values = ListBuffer.empty[Any] + val it = node.elements() + while (it.hasNext) { + val e = it.next() + values += (if (e == null || e.isNull) null else point(name, e)) + } + values.toList + } catch { case _: IllegalArgumentException => asRead(node) } + case MultiValued.Fail => throw new MultiValuedCell(name, table) + } + } else one(node) + } + + private def geoOne(name: String): JsonNode => Any = + node => + try point(name, node) + catch { case _: IllegalArgumentException => asRead(node) } + + private def point(name: String, node: JsonNode): Any = + SetOperationValues.geoPoint(HitReading.plain(node), name) + + /** `[lon, lat]` or `[lon, lat, z]`: one point, not a list (R-34). */ + private def coordinates(node: JsonNode): Boolean = { + val size = node.size() + (size == 2 || size == 3) && { + var all = true + val it = node.elements() + while (all && it.hasNext) all = it.next().isNumber + all + } + } + + /** ONE non-null, non-array value of a scalar type, and -- for a date read by its format -- the + * same read answering NULL where the format does not read the value (D8). + */ + def scalarOne( + name: String, + sqlType: app.softnetwork.elastic.sql.`type`.SQLType, + format: Option[String] + ): (JsonNode => Any, Option[JsonNode => Any]) = + sqlType match { + case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int => + val (min, max) = sqlType match { + case SQLTypes.TinyInt => (Byte.MinValue.toLong, Byte.MaxValue.toLong) + case SQLTypes.SmallInt => (Short.MinValue.toLong, Short.MaxValue.toLong) + case _ => (Int.MinValue.toLong, Int.MaxValue.toLong) + } + val range = new WholeRange(min, max, truncatedFirst = false) + ( + node => + if (node.isInt) { + val v = node.intValue() + if (v >= min && v <= max) Integer.valueOf(v) else asRead(node) + } else + range.read(node) match { + case l: java.lang.Long => Integer.valueOf(l.intValue()) + case null => null + case _ => asRead(node) + }, + None + ) + case SQLTypes.BigInt => + val range = new WholeRange(Long.MinValue, Long.MaxValue, truncatedFirst = true) + ( + node => + if (node.isInt || node.isLong) java.lang.Long.valueOf(node.longValue()) + else + range.read(node) match { + case l: java.lang.Long => l + case null => null + case _ => asRead(node) + }, + None + ) + case SQLTypes.Double => + ( + node => + if (node.isNumber) java.lang.Double.valueOf(node.doubleValue()) + else if (node.isTextual) + SetOperationValues.double(node.asText()) match { + case d: java.lang.Double => d + case null => null + case _ => asRead(node) + } + else asRead(node), + None + ) + case SQLTypes.Real => + ( + node => + if (node.isNumber) java.lang.Float.valueOf(node.floatValue()) + else if (node.isTextual) + SetOperationValues.real(node.asText()) match { + case f: java.lang.Float => f + case null => null + case _ => asRead(node) + } + else asRead(node), + None + ) + case SQLTypes.Boolean => + ( + node => + if (node.isBoolean) java.lang.Boolean.valueOf(node.booleanValue()) + else if (node.isTextual) + SetOperationValues.boolean(node.asText()) match { + case b: java.lang.Boolean => b + case _ => asRead(node) + } + else asRead(node), + None + ) + case SQLTypes.Varchar | SQLTypes.Keyword | SQLTypes.Text | SQLTypes.Char => + (node => if (node.isValueNode) node.asText() else asRead(node), None) + case SQLTypes.VarBinary => + ( + node => + if (node.isTextual) + try java.util.Base64.getDecoder.decode(node.asText()) + catch { case _: IllegalArgumentException => asRead(node) } + else asRead(node), + None + ) + case SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time | SQLTypes.DateTime => + val kind = if (sqlType == SQLTypes.DateTime) SQLTypes.Timestamp else sqlType + format match { + case Some(spec) => + DateRoute.of(kind, format, dates) match { + case DateRoute.AsStored(_) => + (node => if (node.isValueNode) node.asText() else asRead(node), None) + case DateRoute.Typed => + val read = byFormat(kind, spec) + ( + node => { + val value = read(node) + if (value != null) value else asRead(node) + }, + Some(read) + ) + } + case None => (computedDate(kind), None) + } + case _ => (asRead, None) + } + + /** A date field read by its mapping format, as the Elasticsearch major indexed it (D5): a text + * by `MappedDateFormat.read` -- Elasticsearch's default format through [[DefaultDates]], which + * hands every shape it does not read to it --, a number by its digits; `null` for a value the + * format does not read, which the column answers as read (D6) -- or NULL where the hit's + * `_ignored` names it (D8). + */ + private def byFormat( + kind: app.softnetwork.elastic.sql.`type`.SQLType, + spec: String + ): JsonNode => Any = { + val format = MappedDateFormat(spec, dates.dialect.get, dates.clock) + val fast = spec == app.softnetwork.elastic.sql.query.TemporalLiterals.DefaultDateFormat + val result: Instant => Any = kind match { + case SQLTypes.Date => _.atZone(ZoneOffset.UTC).toLocalDate + case SQLTypes.Time => _.atZone(ZoneOffset.UTC).toLocalTime + case _ => _.atZone(ZoneOffset.UTC) + } + val fastResult: ZonedDateTime => Any = kind match { + case SQLTypes.Date => _.toLocalDate + case SQLTypes.Time => _.toLocalTime + case _ => z => z + } + node => { + val z = if (fast && node.isTextual) DefaultDates.read(node.asText()) else null + if (z ne null) fastResult(z) + else { + val outcome = + if (node.isTextual) format.read(node.asText()) + else if (node.isNumber) format.readValue(node.numberValue()) + else MappedDateFormat.Rejected + outcome match { + case MappedDateFormat.Read(instant) => result(instant) + case _ => null + } + } + } + } + + /** A computed date (no field, so no format): `SetOperationValues`' `timestamp` / `date` over + * today's reading, a `TIMESTAMP` at UTC; a value they do not read as read. + */ + private def computedDate(kind: app.softnetwork.elastic.sql.`type`.SQLType): JsonNode => Any = + kind match { + case SQLTypes.Date => + node => + SetOperationValues.date(asRead(node)) match { + case d: LocalDate => d + case _ => asRead(node) + } + case SQLTypes.Time => + node => + asRead(node) match { + case t: LocalTime => t + case z: ZonedDateTime => z.withZoneSameInstant(ZoneOffset.UTC).toLocalTime + case l: LocalDateTime => l.toLocalTime + case other => other + } + case _ => + node => { + val read = asRead(node) + SetOperationValues.timestamp(read) match { + case z: ZonedDateTime => + if (z.getZone == ZoneOffset.UTC) z else z.withZoneSameInstant(ZoneOffset.UTC) + case _ => read + } + } + } + } + + /** What [[WholeRange.read]] answers for a value that is no value of its type. */ + private object OutOfType + + /** SCHEMA-2 (D6) -- an integral type's values, `[min, max]`, read as Elasticsearch reads a value + * into that type: a whole number as it is, a decimal or a numeric text truncated toward zero + * (`coerce`, T row 5), a blank text NULL. A value outside the range is no value of the type -- + * Elasticsearch refused it (`ignore_malformed`) and holds it in `_source` only -- so `read` + * answers [[OutOfType]] and the column keeps it as stored (`Unreadable.KeepAsStored`): narrowing + * it would wrap it. The range is checked as Elasticsearch checks it: `long` on the truncated + * value (`truncatedFirst`, `Numbers.toLong`), `integer` / `short` / `byte` on the value itself. + */ + private[client] final class WholeRange(min: Long, max: Long, truncatedFirst: Boolean) { + + private[this] val low = java.math.BigDecimal.valueOf(min) + private[this] val high = java.math.BigDecimal.valueOf(max) + + /** A `java.lang.Long` within the range, `null` for a blank text, else [[OutOfType]]. */ + def read(node: JsonNode): Any = + if (node.isIntegralNumber) { + if (node.canConvertToLong) whole(node.longValue()) else OutOfType + } else if (node.isBigDecimal) decimal(node.decimalValue()) + else if (node.isNumber) { + val d = node.doubleValue() + if (java.lang.Double.isNaN(d) || java.lang.Double.isInfinite(d)) OutOfType + else decimal(java.math.BigDecimal.valueOf(d)) + } else if (node.isTextual) { + val text = node.asText().trim + if (text.isEmpty) null + else + try decimal(new java.math.BigDecimal(text)) + catch { case _: NumberFormatException => OutOfType } + } else OutOfType + + private def whole(value: Long): Any = + if (value >= min && value <= max) java.lang.Long.valueOf(value) else OutOfType + + private def decimal(d: java.math.BigDecimal): Any = { + // the digits before the point, read off the scale: `1e1000000000` is never materialised + val integerDigits = d.precision() - d.scale() + if (d.signum() == 0 || integerDigits <= 0) java.lang.Long.valueOf(0L) + else if (integerDigits > 19) OutOfType + else { + val truncated = d.setScale(0, java.math.RoundingMode.DOWN) + val ranged = if (truncatedFirst) truncated else d + if (ranged.compareTo(low) >= 0 && ranged.compareTo(high) <= 0) + java.lang.Long.valueOf(truncated.longValue()) + else OutOfType + } + } + } + + /** A node as a plain value, with no temporal inference: what `geoPoint` reads. */ + private[client] def plain(node: JsonNode): Any = + if (node == null || node.isNull) null + else if (node.isTextual) node.asText() + else if (node.isNumber) node.numberValue() + else if (node.isBoolean) node.booleanValue() + else if (node.isArray) { + val values = ListBuffer.empty[Any] + val it = node.elements() + while (it.hasNext) values += plain(it.next()) + values.toList + } else if (node.isObject) { + val entries = ListMap.newBuilder[String, Any] + val it = node.properties().iterator() + while (it.hasNext) { + val e = it.next() + entries += e.getKey -> plain(e.getValue) + } + entries.result() + } else node.asText() + + /** SCHEMA-2 (D5) -- Elasticsearch's default `strict_date_optional_time||epoch_millis` for the + * shapes it is written in almost always -- `uuuu-MM-dd`, `uuuu-MM-ddTHH:mm:ss`, a fraction of 1 + * to 3 digits, an optional `Z` -- at UTC, by hand; `null` for any other shape, which the caller + * reads with `MappedDateFormat.read` (R-5: no second date rule; `HitReadingSpec` proves the two + * equal over the corpus). + */ + private[client] object DefaultDates { + + private def digit(c: Char): Int = if (c >= '0' && c <= '9') c - '0' else -1 + + private def number(s: String, from: Int, to: Int): Int = { + var n = 0 + var i = from + while (i < to) { + val d = digit(s.charAt(i)) + if (d < 0) return -1 + n = n * 10 + d + i += 1 + } + n + } + + def read(s: String): ZonedDateTime = { + val len = s.length + if (len < 10 || s.charAt(4) != '-' || s.charAt(7) != '-') return null + val year = number(s, 0, 4) + val month = number(s, 5, 7) + val day = number(s, 8, 10) + if (year < 0 || month < 1 || month > 12 || day < 1 || day > 31) return null + var hour = 0 + var minute = 0 + var second = 0 + var nanos = 0 + var end = len + if (len > 10) { + if (s.charAt(len - 1) == 'Z') end = len - 1 + if (end < 19 || s.charAt(10) != 'T' || s.charAt(13) != ':' || s.charAt(16) != ':') + return null + hour = number(s, 11, 13) + minute = number(s, 14, 16) + second = number(s, 17, 19) + if (hour < 0 || hour > 23 || minute < 0 || minute > 59 || second < 0 || second > 59) + return null + if (end > 19) { + val digits = end - 20 + if (s.charAt(19) != '.' || digits < 1 || digits > 3) return null + val fraction = number(s, 20, end) + if (fraction < 0) return null + nanos = + fraction * (if (digits == 1) 100000000 else if (digits == 2) 10000000 else 1000000) + } + } + try ZonedDateTime.of(year, month, day, hour, minute, second, nanos, ZoneOffset.UTC) + catch { case _: java.time.DateTimeException => null } + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala b/core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala new file mode 100644 index 000000000..4857e68ed --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala @@ -0,0 +1,136 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.query.{FromlessSelect, MultiSearch, SingleSearch} +import app.softnetwork.elastic.sql.query.MultiSearch.{SetOperationColumn, SetOperationType} + +/** ONE column of a statement's result, declared BEFORE any row is read. + * + * @param name + * the name the column takes in every row (`Field.outputName`; a struct's field name; empty for a + * list's element) + * @param sqlType + * its type at this depth: `KEYWORD`, `TEXT` and `CHAR` are `VARCHAR`, a `DATETIME` a + * `TIMESTAMP`; a date field whose format core cannot read for sure is `VARCHAR` (its text as + * stored); `ANY` where it is not known, `NUMERIC` / `TEMPORAL` for a number / a temporal of + * unknown width + * @param nullable + * always `true` in this release + * @param fields + * a `STRUCT`'s or a `GEO_POINT`'s fields, by name; empty for any other type and for a `STRUCT` + * whose fields are not known + * @param element + * an `ARRAY`'s element; `None` for any other type + */ +final case class ResultColumn( + name: String, + sqlType: SQLType, + nullable: Boolean = true, + fields: Seq[ResultColumn] = Nil, + element: Option[ResultColumn] = None +) + +/** The columns of a statement's result, in the order a consumer presents them (the SELECT order; + * `SELECT *` = the mapping's columns in their table order). Rows keep their documents' key order: + * a consumer reads a value BY NAME. + */ +final case class ResultSchema(columns: Seq[ResultColumn]) { + + /** The column names, in schema order. */ + def names: Seq[String] = columns.map(_.name) +} + +/** THE builder of a statement's [[ResultSchema]] (SCHEMA-2, D1): from the type each projection item + * has as a set-operation position (`MultiSearch.fieldType` / `columnType`, #292) and the route of + * each date field ([[DateRoute.of]]: a date read as stored is `VARCHAR`). Built once per + * statement. + */ +private[client] object ResultSchemas { + + /** `None` where no schema is known: no mapping attached to the statement (a client that is not an + * `IndicesApi`, an es8/es9 alias, a mapping that could not be read). + */ + def of(single: SingleSearch, dates: SetOperationValues.DateReading): Option[ResultSchema] = + single.schema.map { table => + ResultSchema(single.select.fieldsWithComputedAliases.flatMap { field => + if (HitReading.isStar(field)) + table.columns.map(c => columnOf(c.name, MultiSearch.columnType(c), dates)) + else Seq(columnOf(field.outputName, MultiSearch.fieldType(field), dates)) + }) + } + + /** A set operation's schema (D12): its columns typed ONCE ([[SetOperationValues.routedColumns]], + * the typing core's converters read too), named by the first branch. `None` where the first + * branch is opaque (`SELECT *`), where no branch has a schema, or where the statement is + * refused. + */ + def of(multiple: MultiSearch, dates: SetOperationValues.DateReading): Option[ResultSchema] = + multiple.requests.headOption.flatMap { first => + if (multiple.requests.forall(_.schema.isEmpty)) None + else + MultiSearch.projectionTypes(first).flatMap { declared => + SetOperationValues + .routedColumns(multiple.requests, multiple.resolvedOperators, dates) + .toOption + .filter(_.size == declared.size) + .map { columns => + ResultSchema(declared.zip(columns).map { case ((field, _), column) => + resultColumnOf(field.outputName, column) + }) + } + } + } + + /** A FROM-less statement's schema: its items' declared types, under `columnNames`. */ + def of(fromless: FromlessSelect): ResultSchema = + ResultSchema(fromless.columnNames.zip(fromless.select.fields).map { case (name, field) => + columnOf(name, MultiSearch.fieldType(field), SetOperationValues.DateReading.unknown) + }) + + private def resultColumnOf(name: String, column: SetOperationColumn): ResultColumn = + columnOf(name, column.result, SetOperationValues.DateReading.unknown) + + /** One position, at every depth; a date position [[DateRoute.of]] answers as stored is `VARCHAR`. + */ + private[client] def columnOf( + name: String, + t: SetOperationType, + dates: SetOperationValues.DateReading + ): ResultColumn = + t.sqlType match { + case SQLTypes.Struct | SQLTypes.GeoPoint => + ResultColumn( + name, + t.sqlType, + fields = t.fields.map(f => columnOf(f.name, f.valueType, dates)) + ) + case SQLTypes.Array(_) => + t.element match { + case Some(e) => + val element = columnOf("", e, dates) + ResultColumn(name, SQLTypes.Array(element.sqlType), element = Some(element)) + case None => ResultColumn(name, t.sqlType) + } + case scalar => + DateRoute.of(scalar, t.format, dates) match { + case DateRoute.Typed => ResultColumn(name, scalar) + case DateRoute.AsStored(_) => ResultColumn(name, SQLTypes.Varchar) + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index be626d834..314252189 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -271,23 +271,37 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { )(implicit system: ActorSystem, context: ConversionContext - ): Source[(ListMap[String, Any], ScrollMetrics), NotUsed] = { + ): Source[(ListMap[String, Any], ScrollMetrics), NotUsed] = + scrollWithSchema(statement, config)._2 + + /** SCHEMA-2 (D13) -- [[scroll]] with the statement's result schema, declared before any row: the + * resolve is synchronous, so the schema is known when the source is returned. ONE pipeline: + * [[scroll]] is its source. A window-enriched statement's schema is the statement's as written, + * never its base query's; `None` where no schema is known. + */ + private[client] def scrollWithSchema( + statement: SearchStatement, + config: ScrollConfig = defaultScrollConfig + )(implicit + system: ActorSystem, + context: ConversionContext + ): (Option[ResultSchema], Source[(ListMap[String, Any], ScrollMetrics), NotUsed]) = { implicit def timestamp: Long = System.currentTimeMillis() statement match { // Select statement case select: SelectStatement => select.statement match { case Some(single: SingleSearch) => - scroll(single.copy(score = select.score), config) + scrollWithSchema(single.copy(score = select.score), config) case Some(multiple: MultiSearch) => - scroll(multiple, config) + scrollWithSchema(multiple, config) case None => // Issue #389 / F7 -- carry the parser's OWN reason. A whole family of SEMANTIC refusals // reaches this branch, each naming a clause and a remedy; the generic sentence threw // every one of them away. - Source.failed( + None -> Source.failed( new IllegalArgumentException( (statement match { case s: SelectStatement => s.parseError @@ -324,14 +338,14 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { _.isAggregation ) || single.scriptFields.nonEmpty) ) - return scrollWithWindowEnrichment(single, config0) + return statementSchema(single) -> scrollWithWindowEnrichment(single, config0) val requestedFields = extractOutputFieldNames(single) - // the columns `single` declares VARCHAR are read as their text (`TextColumns`), and each + // each value of `single` read by its declared type (`SchemaColumns`), and each // conversion of a bare UNNEST element column by its reader, its dates as the cluster's // major reads them (`ElementConversions`) val textContext: ConversionContext = - TextColumns.of( + SchemaColumns.of( single, context, this match { @@ -365,7 +379,7 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { // The rows of a GROUP BY, in the order, past the offset and to the count the statement // asks for (`ElasticConversion.orderGroupedRows`). They are the rows of ONE aggregation // response, so collecting them holds nothing the response did not already. - GroupedRowOrder.of(single) match { + SchemaColumns.schemaOf(textContext) -> (GroupedRowOrder.of(single) match { case Some(order) => rows .fold(Vector.empty[(ListMap[String, Any], ScrollMetrics)])(_ :+ _) @@ -378,16 +392,16 @@ trait ScrollApi extends ElasticClientHelpers with SchemaCacheTtlApi { .toList ) case None => rows - } + }) // Multi search case _: MultiSearch => - Source.failed( + None -> Source.failed( new UnsupportedOperationException("Scrolling is not supported for multi-search queries") ) case _ => - Source.failed( + None -> Source.failed( new IllegalArgumentException("Scrolling is only supported for SELECT statements") ) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 7eafd9cb7..a2651ad50 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -142,7 +142,18 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC * (AD-17). The only self-consistent predicate is "this statement emits Painless over a column", * i.e. `shouldBeScripted` across the clauses. */ - private[client] def resolveWithSchema(single: SingleSearch): ElasticResult[SingleSearch] = { + private[client] def resolveWithSchema(single: SingleSearch): ElasticResult[SingleSearch] = + resolveWithSchema(single, executeSubqueries = true) + + /** [[resolveWithSchema]] 's ONE body (SCHEMA-2, D14): `executeSubqueries = false` -- what + * `GatewayApi.describe` runs -- replaces phase one's execution by its pre-execution checks + * (`SubqueryResolver.resolve(_, _, executes = false)`), every other step -- the attach, the + * explicit path's reload (D10), every refusal -- unchanged. + */ + private[client] def resolveWithSchema( + single: SingleSearch, + executeSubqueries: Boolean + ): ElasticResult[SingleSearch] = { // Story 22.1 (epic 22 AD-5) — THE seam. Every direct-API path crosses it BEFORE routing // (`search`, `searchAsync`, `searchWithInnerHits`, `ScrollApi.scroll`, and `IndicesApi`'s // by-query DELETE/UPDATE bodies through `resolveDmlWithSchema`), and none of them passes @@ -172,7 +183,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC val phaseOne: SingleSearch = if (!single.hasWhereSubqueries) single else - SubqueryResolver.resolve(single, this) match { + SubqueryResolver.resolve(single, this, executes = executeSubqueries) match { case ElasticSuccess(rewritten) => rewritten case ElasticFailure(error) => return ElasticResult.failure(error) } @@ -191,8 +202,11 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC lookupSchema(sources) match { case Some(lookup) => lookup match { - case Success(ElasticSuccess(schema)) => + case Success(ElasticSuccess(cached)) => schemaMisses.remove(source) + // SCHEMA-2 (D10) -- an explicitly selected path the cached mapping lacks reloads it + // ONCE per TTL (memoized), so a field mapped since is read by its type + val schema = reloadedFor(phaseOne, sources, cached) TemporalLiterals(phaseOne, schema) match { case Right(literalsResolved) => // #306 -- ATTACH the schema to the AST. `GenericIdentifier.baseType` is @@ -292,6 +306,85 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC case Right(_) => ElasticResult.success(statement) } + /** SCHEMA-2 (D10) -- the (source, path) pairs whose mapping was reloaded for an explicitly + * selected path it lacked, and when: another statement selecting that path within the source's + * TTL reads no mapping. + * + * Keys are caller-supplied names, so the memo is bounded the way [[schemaMisses]] is: above + * [[schemaPathReloadPurgeThreshold]] entries an insert also drops the expired ones (each by its + * source's TTL), and if it is STILL above four times the threshold (a client selecting thousands + * of distinct unmapped names inside one TTL) it keeps only the insert's own entries -- the worst + * case is then one mapping read per statement selecting such a name, never unbounded memory. + */ + private[this] lazy val schemaPathReloads = + new java.util.concurrent.ConcurrentHashMap[(String, String), java.lang.Long]() + + private val schemaPathReloadPurgeThreshold = 256 + + /** Current size of the D10 reload memo (tests). */ + private[client] def schemaPathReloadCount: Int = schemaPathReloads.size() + + /** Remember that `source`'s mapping was reloaded at `now` for `paths` (D10), bounded. */ + private def rememberPathReloads(source: String, paths: Seq[String], now: Long): Unit = { + paths.foreach(p => schemaPathReloads.put((source, p), java.lang.Long.valueOf(now))) + if (schemaPathReloads.size() > schemaPathReloadPurgeThreshold) { + schemaPathReloads + .entrySet() + .removeIf((e: java.util.Map.Entry[(String, String), java.lang.Long]) => + now - e.getValue >= schemaCacheTtlMsFor(e.getKey._1) + ) + if (schemaPathReloads.size() > schemaPathReloadPurgeThreshold * 4) { + schemaPathReloads.clear() + paths.foreach(p => schemaPathReloads.put((source, p), java.lang.Long.valueOf(now))) + } + } + } + + /** The paths `statement` selects explicitly (a bare column, not `*`, not a function, not an + * UNNEST element, not a metadata field). + */ + private def explicitPaths(statement: SingleSearch): Seq[String] = + statement.select.fields.collect { + case f + if f.identifier.functions.isEmpty && !f.identifier.nested && + f.identifier.name.nonEmpty && f.identifier.name != "*" && + !f.identifier.name.startsWith("_") => + f.identifier.name + }.distinct + + /** `schema`, or -- when `statement` selects a path it lacks on ONE concrete source, not reloaded + * for that path within the source's TTL -- the source's mapping read again + * (`IndicesApi.invalidateSchema`, then [[lookupSchema]]); `schema` when the reload fails. + */ + private def reloadedFor(statement: SingleSearch, sources: Seq[String], schema: Schema): Schema = + this match { + case indices: IndicesApi + if sources.size == 1 && !sources.head.contains("*") && !sources.head.contains(",") => + val source = sources.head + val missing = explicitPaths(statement).filter(p => schema.find(p).isEmpty) + if (missing.isEmpty) schema + else { + val now = System.currentTimeMillis() + val ttl = schemaCacheTtlMsFor(source) + val fresh = missing.filter { p => + Option(schemaPathReloads.get((source, p))).forall(at => now - at >= ttl) + } + if (fresh.isEmpty) schema + else { + rememberPathReloads(source, fresh, now) + logger.debug( + s"Mapping of '$source' reloaded for ${fresh.mkString(", ")}, absent from its cached schema" + ) + indices.invalidateSchema(source) + lookupSchema(sources) match { + case Some(Success(ElasticSuccess(reloaded))) => reloaded + case _ => schema + } + } + } + case _ => schema + } + /** THE schema lookup for a statement's sources, for every consumer -- the attach * ([[resolveWithSchema]]) and the scope of a subquery ([[resolvedSchema]]) alike (#292): ONE * concrete index reads its own mapping ([[IndicesApi.loadSchema]]); an index PATTERN, a comma @@ -519,7 +612,14 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC private[client] def forgetAllSchemaMisses(): Unit = schemaMisses.clear() /** [[resolveWithSchema]] over every request of a `UNION ALL`; the first rejection wins. */ - private[client] def resolveWithSchema(multiple: MultiSearch): ElasticResult[MultiSearch] = { + private[client] def resolveWithSchema(multiple: MultiSearch): ElasticResult[MultiSearch] = + resolveWithSchema(multiple, executeSubqueries = true) + + /** [[resolveWithSchema]] over every request, each through the single statement's ONE body. */ + private[client] def resolveWithSchema( + multiple: MultiSearch, + executeSubqueries: Boolean + ): ElasticResult[MultiSearch] = { // Story 22.6 — only `UNION ALL` is an Elasticsearch `_msearch`; UNION (de-duplicating), // UNION DISTINCT, INTERSECT [ALL] and EXCEPT [ALL] all need the relational engine. Refuse HERE, // before any branch is resolved or any request is built: this is the ONE seam every direct-API @@ -532,7 +632,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC val zero: ElasticResult[Seq[SingleSearch]] = ElasticResult.success(Seq.empty) multiple.requests.foldLeft(zero) { case (ElasticSuccess(acc), request) => - resolveWithSchema(request) match { + (if (executeSubqueries) resolveWithSchema(request) + else resolveWithSchema(request, executeSubqueries = false)) match { case ElasticSuccess(resolved) => ElasticResult.success(acc :+ resolved) case ElasticFailure(error) => ElasticResult.failure(error) } @@ -665,7 +766,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC multiple.requests.headOption.map(_.nestedHitsMappings).getOrElse(Map.empty), multiple.requests.map(rowInvariantsOf), unionAllLegProjections(multiple) - ) + ).map(_.copy(schema = unionAllSchema(multiple))) case _ => logger.error( @@ -806,7 +907,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC query, success.value, fieldAliases, - aggs + aggs, + SchemaColumns.schemaOf(context) ) ) case ElasticFailure(error) => @@ -1026,7 +1128,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC multiple.requests.headOption.map(_.nestedHitsMappings).getOrElse(Map.empty), multiple.requests.map(rowInvariantsOf), unionAllLegProjections(multiple) - ) + ).map(_.map(_.copy(schema = unionAllSchema(multiple)))) case _ => val query = statement.sql @@ -1099,7 +1201,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC query, success.value, fieldAliases, - aggs + aggs, + SchemaColumns.schemaOf(context) ) ) ) @@ -1951,7 +2054,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // Step 3: Enrich results enrichedResponse <- enrichResponseWithWindowValues(baseResponse, windowCache, request) - } yield enrichedResponse + } yield enrichedResponse.copy(schema = statementSchema(request)) } // ======================================================================== @@ -2447,7 +2550,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC extractOutputFieldNames(single), UnnestScope.rowExplosion(single), rowInvariantsOf(single) - )(TextColumns.of(single, context, setOperationVersion)).map(inGroupedRowOrder(single, _)) + )(SchemaColumns.of(single, context, setOperationVersion)) + .map(inGroupedRowOrder(single, _)) } } @@ -2488,7 +2592,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC extractOutputFieldNames(single), UnnestScope.rowExplosion(single), rowInvariantsOf(single) - )(ec, TextColumns.of(single, context, setOperationVersion)) + )(ec, SchemaColumns.of(single, context, setOperationVersion)) .map(_.map(inGroupedRowOrder(single, _))) } } @@ -2632,8 +2736,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // the plan that built THIS leg's request -- never one made from the merged map toClientAggregations(leg.sqlAggregations), convert, - // ...read as THIS leg reads its rows on its own: its VARCHAR columns as their text - Some(TextColumns.of(leg, context, setOperationVersion)) + // ...read as THIS leg reads its rows on its own: each value by its declared type + Some(SchemaColumns.of(leg, context, setOperationVersion)) ) } } @@ -2653,10 +2757,21 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC response.results.map(normalise) }, multiple.fieldAliases, - toClientAggregations(multiple.sqlAggregations) + toClientAggregations(multiple.sqlAggregations), + unionAllSchema(multiple) ) } + /** SCHEMA-2 (D12) -- a `UNION ALL`'s result schema, typed as its converters type its columns. */ + private[client] def unionAllSchema(multiple: MultiSearch): Option[ResultSchema] = + ResultSchemas.of(multiple, SetOperationValues.DateReading(setOperationVersion)) + + /** SCHEMA-2 (D13) -- the schema of `single` as written, read before any row: what a + * window-enriched statement's response carries (never its base query's). + */ + private[client] def statementSchema(single: SingleSearch): Option[ResultSchema] = + ResultSchemas.of(single, SetOperationValues.DateReading(setOperationVersion)) + private def requiresScrollPaging(limit: Option[Limit]): Boolean = limit match { case None => true @@ -2767,14 +2882,14 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // deterministic `_doc` order); the statement's LIMIT was stripped above, so ScrollApi's own // clamp cannot see it and the clamp must be applied here. The no-LIMIT branch inherits the // configured ceiling (`maxSlices = None`). - scrollApi - .scroll( - statement, - scrollApi.defaultScrollConfig.copy( - maxDocuments = maxDocuments, - maxSlices = if (single.limit.isDefined) Some(1) else None - ) + val (schema, source) = scrollApi.scrollWithSchema( + statement, + scrollApi.defaultScrollConfig.copy( + maxDocuments = maxDocuments, + maxSlices = if (single.limit.isDefined) Some(1) else None ) + ) + source .map(_._1) .drop(offset) .runWith(Sink.seq) @@ -2786,7 +2901,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC elasticQuery.query, rows, single.fieldAliases, - ListMap.empty + ListMap.empty, + schema ) ) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala b/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala index 6863efe09..28dcbd491 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala @@ -83,16 +83,43 @@ private[elastic] object SetOperationValues { private[client] def converters( multiple: MultiSearch, version: => Option[String] - ): Seq[Option[Array[Any => Any]]] = - MultiSearch.columnTypes(multiple.requests) match { + ): Seq[Option[Array[Any => Any]]] = { + val dates = DateReading(version) + // SCHEMA-2 (D12) -- the ONE typing the result schema reads too (`ResultSchemas.of`) + routedColumns(multiple.requests, multiple.resolvedOperators, dates) match { case Right(columns) if columns.exists(c => c.converts || readsMappedDates(c)) => val names = columnNames(multiple.requests, columns.size) - val dates = DateReading(version) multiple.requests.indices.map(leg => legConverters(multiple.requests(leg), leg, columns, names, dates) ) case _ => multiple.requests.map(_ => None) } + } + + /** SCHEMA-2 (D12) -- THE typing of a set operation's columns as core's `UNION ALL` reads them, + * for the result schema (`ResultSchemas.of`) and the converters ([[converters]]) alike (#292): + * `MultiSearch.columnTypes`, every TOP-LEVEL date position of a branch [[DateRoute.of]] answers + * as stored made `VARCHAR` -- and the column with it `VARCHAR` (DuckDB's text rule), its other + * branches converted to their text. A leg reads such a date as its text as stored (its + * `HitReading`), so the column holds one class. Every other column is `columnTypes`' own. + */ + private[client] def routedColumns( + requests: Seq[SingleSearch], + operators: Seq[SetOperator], + dates: DateReading + ): Either[String, Seq[SetOperationColumn]] = + MultiSearch + .columnTypes(requests, operators) + .map(_.map { column => + val branches = column.branches.map(_.map { branch => + DateRoute.of(branch.sqlType, branch.format, dates) match { + case DateRoute.Typed => branch + case DateRoute.AsStored(_) => SetOperationType(SQLTypes.Varchar) + } + }) + if (branches == column.branches) column + else column.copy(result = SetOperationType(SQLTypes.Varchar), branches = branches) + }) /** THE READER of each leg of a set operation, column by column -- in leg order, `None` for a leg * with nothing to read -- or, `Left`, the statement's refusal (`MultiSearch.branchTypes`' @@ -110,12 +137,16 @@ private[elastic] object SetOperationValues { * (`ElasticConversion.jsonNodeToAny`), AS ITS BRANCH DECLARES IT * (`SetOperationColumn.branches`), with its mapping format, as Elasticsearch indexed it -- * at every depth, where the branches agree too. One class per type: an `INT` is an - * `Integer`, a `BIGINT` a `Long`, a `SMALLINT` a `Short`, a `TINYINT` a `Byte`, a `DOUBLE` a - * `Double`, a `REAL` a `Float`, a `BOOLEAN` a `Boolean`, a `DATE` a `LocalDate`, a + * `Integer`, a `BIGINT` a `Long`, a `SMALLINT` and a `TINYINT` an `Integer` (D2), a `DOUBLE` + * a `Double`, a `REAL` a `Float`, a `BOOLEAN` a `Boolean`, a `DATE` a `LocalDate`, a * `TIMESTAMP` a `ZonedDateTime` in UTC, a `TIME` a `LocalTime`, a `VARCHAR` a `String`, a - * `VARBINARY` the UTF-8 bytes of its base64 text, a `STRUCT` a `ListMap` of its fields in - * the branch's order, a `GEO_POINT` a `ListMap` of `lat` and `lon` (`Double`s), an `ARRAY` a + * `VARBINARY` the bytes its base64 text encodes, a `STRUCT` a `ListMap` of its fields in the + * branch's order, a `GEO_POINT` a `ListMap` of `lat` and `lon` (`Double`s), an `ARRAY` a * `List` of its element's class. + * - A value core's hit reader already gave in that class's one form (SCHEMA-2, D12: a + * `LocalDate`, a `LocalTime` or a `ZonedDateTime` at UTC to the millisecond, at a date of a + * format core reads every value of for sure, `DateRoute.Typed`) is answered as it is, never + * re-read by the format. * - Elasticsearch's one-element array (`script_fields`) is ONE value, its empty array NULL. * - A value core cannot read fails the statement, in the words core's `UNION ALL` uses: a date * its format does not read, or does not read without guessing (a zone name), a point that is @@ -1196,21 +1227,51 @@ private[elastic] object SetOperationValues { value => failure(value, detail) case Some(dialect) => val format = MappedDateFormat(spec, dialect, dates.clock) - value => - format.readValue(value) match { - case MappedDateFormat.Read(instant) => result(instant) - case MappedDateFormat.Rejected => - failure( - value, - s"the date format '$spec' of field $field does not read it as ${dialect.label} does" - ) - case MappedDateFormat.Undecidable(reason) => - failure( - value, - s"core cannot read it with the date format '$spec' of field $field as " + - s"${dialect.label} does: $reason" - ) - } + // SCHEMA-2 (D12) -- a value already in the result class's ONE form (a canonical row, + // read by its column's `HitReading`) is passed through: re-reading a `LocalDate` as + // `epoch_second` would fail it. That form is exactly what the reader gives -- a `DATE` a + // `LocalDate`, a `TIME` a `LocalTime`, a `TIMESTAMP` a `ZonedDateTime` AT UTC, the instant + // to the millisecond Elasticsearch keeps -- never what the response parser makes of a text + // (`ElasticConversion.jsonNodeToAny`: an offset `2024-01-31T10:30:00+02:00`, a region + // `…[Europe/Paris]`, nanoseconds), which the format still reads as Elasticsearch indexed + // it, or fails (R-11 / R-15). And only under a format the hit reader reads by + // (`DateRoute.Typed`: core reads every value of it for sure): under any other it reads + // the text as stored, so a value in that form is the response parser's. One class test + // per value. + val canonical: Any => Any = + if (format.unread.isDefined) ((_: Any) => null) + else + source match { + case SQLTypes.Date => { case d: LocalDate => d; case _ => null } + case SQLTypes.Time => { + case t: LocalTime if t.getNano % 1000000 == 0 => t + case _ => null + } + case _ => { + case z: ZonedDateTime if z.getZone == ZoneOffset.UTC && z.getNano % 1000000 == 0 => + z + case _ => null + } + } + value => { + val passed = if (value == null) null else canonical(value) + if (passed != null) passed + else + format.readValue(value) match { + case MappedDateFormat.Read(instant) => result(instant) + case MappedDateFormat.Rejected => + failure( + value, + s"the date format '$spec' of field $field does not read it as ${dialect.label} does" + ) + case MappedDateFormat.Undecidable(reason) => + failure( + value, + s"core cannot read it with the date format '$spec' of field $field as " + + s"${dialect.label} does: $reason" + ) + } + } } } @@ -1229,7 +1290,7 @@ private[elastic] object SetOperationValues { case other => other } - private val double: Any => Any = { + private[client] val double: Any => Any = { case s: String => val t = s.trim if (t.isEmpty) null @@ -1237,7 +1298,7 @@ private[elastic] object SetOperationValues { case other => other } - private val real: Any => Any = { + private[client] val real: Any => Any = { case s: String => val t = s.trim if (t.isEmpty) null @@ -1245,19 +1306,19 @@ private[elastic] object SetOperationValues { case other => other } - private val boolean: Any => Any = { + private[client] val boolean: Any => Any = { case "true" => java.lang.Boolean.TRUE case "false" | "" => java.lang.Boolean.FALSE case other => other } - private val timestamp: Any => Any = { + private[client] val timestamp: Any => Any = { case s: String if EpochMillis.pattern.matcher(s).matches() => Instant.ofEpochMilli(s.toLong).atZone(ZoneOffset.UTC) case other => toTimestamp(other) } - private val date: Any => Any = { + private[client] val date: Any => Any = { case d: LocalDate => d case other => timestamp(other) match { @@ -1269,9 +1330,10 @@ private[elastic] object SetOperationValues { /** A value reaching a `VARBINARY` column (DuckDB's `BLOB`) as BYTES -- `Array[Byte]`, the class * of core's `VARBINARY` cell (`ValueCoercion.inferType`, `JavaValueConversion`). * - * - A `VARBINARY` value is the bytes of the text Elasticsearch holds for it, a `binary` - * field's base64, in UTF-8: the bytes the relational engine is handed for that field - * (arrow's `VarBinaryVector`) and the bytes the JDBC driver's `getBytes` answers. + * - A `VARBINARY` value is the bytes the text Elasticsearch holds for it encodes -- a `binary` + * field's base64, DECODED (SCHEMA-2, Q5): the bytes the document was indexed with, the bytes + * core's hit reader gives the column ([[base64]], one rule); a text that is not base64 is + * its UTF-8 bytes, as before. * - A text value is the bytes DuckDB's cast from `VARCHAR` to `BLOB` makes of it * ([[blobOfText]]). */ @@ -1280,7 +1342,7 @@ private[elastic] object SetOperationValues { case SQLTypes.VarBinary => { case null => null case b: Array[Byte] => b - case other => other.toString.getBytes(StandardCharsets.UTF_8) + case other => base64(other.toString) } case _ => { case null => null @@ -1289,6 +1351,14 @@ private[elastic] object SetOperationValues { } } + /** SCHEMA-2 (Q5) -- the bytes a `binary` field's base64 text encodes; a text that is not base64, + * its UTF-8 bytes. The ONE rule core's hit reader (`HitReading`) and a set operation's + * `VARBINARY` position read a `binary` value by. + */ + private[client] def base64(text: String): Array[Byte] = + try java.util.Base64.getDecoder.decode(text) + catch { case _: IllegalArgumentException => text.getBytes(StandardCharsets.UTF_8) } + /** DuckDB 1.5.5's cast of a `VARCHAR` to a `BLOB`, MEASURED value by value: every ASCII character * is its byte, `\xHH` (two hexadecimal digits) is the byte `HH`, and the cast FAILS -- the * statement with it, as DuckDB's does -- on a character outside ASCII, on a backslash followed @@ -1402,19 +1472,21 @@ private[elastic] object SetOperationValues { case other => other } + /** SCHEMA-2 (D2) -- a `SMALLINT` is an `Integer` (JDBC Table B-3), its value a `SHORT`'s. */ private val toShort: Any => Any = { case null => null - case s: java.lang.Short => s - case n: java.lang.Number => java.lang.Short.valueOf(n.shortValue()) - case b: java.lang.Boolean => java.lang.Short.valueOf(if (b) 1.toShort else 0.toShort) + case i: java.lang.Integer => java.lang.Integer.valueOf(i.shortValue().toInt) + case n: java.lang.Number => java.lang.Integer.valueOf(n.shortValue().toInt) + case b: java.lang.Boolean => java.lang.Integer.valueOf(if (b) 1 else 0) case other => other } + /** SCHEMA-2 (D2) -- a `TINYINT` is an `Integer` (JDBC Table B-3), its value a `BYTE`'s. */ private val toByte: Any => Any = { case null => null - case b: java.lang.Byte => b - case n: java.lang.Number => java.lang.Byte.valueOf(n.byteValue()) - case b: java.lang.Boolean => java.lang.Byte.valueOf(if (b) 1.toByte else 0.toByte) + case i: java.lang.Integer => java.lang.Integer.valueOf(i.byteValue().toInt) + case n: java.lang.Number => java.lang.Integer.valueOf(n.byteValue().toInt) + case b: java.lang.Boolean => java.lang.Integer.valueOf(if (b) 1 else 0) case other => other } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SubqueryResolver.scala b/core/src/main/scala/app/softnetwork/elastic/client/SubqueryResolver.scala index cbe517537..0f311013d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SubqueryResolver.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SubqueryResolver.scala @@ -82,13 +82,35 @@ object SubqueryResolver { * object's. */ def resolve(single: SingleSearch, api: SearchApi): ElasticResult[SingleSearch] = + resolve(single, api, executes = true) + + /** [[resolve]], or -- `executes = false` (SCHEMA-2, D14: `GatewayApi.describe`) -- its + * PRE-EXECUTION checks only, the same arms of [[resolveOne]] (a body that is no query, a + * correlated body, a bare-name correlation), with nothing executed: the statement comes back + * unchanged, or with the refusal `run` gives before executing anything. One walk, a parameter -- + * never a copy (#292). + */ + private[client] def resolve( + single: SingleSearch, + api: SearchApi, + executes: Boolean + ): ElasticResult[SingleSearch] = resolve( single, s => api.search(s)(EntityContext), // an absent key stays ABSENT — never null-filled api.scopeCorrelation(single, _), - api.innerColumnType + api.innerColumnType, + executes ) + private[client] def resolve( + single: SingleSearch, + execute: Execute, + correlation: SingleSearch => Option[String] = _ => None, + columnType: ColumnType = (_, _) => None + ): ElasticResult[SingleSearch] = + resolve(single, execute, correlation, columnType, executes = true) + /** Test seam: the inner executor, the bare-name correlation check and the inner column's mapped * type are injected, so the whole two-phase contract is provable Docker-free. `columnType(inner, * name)` answers `None` when no schema is loadable (the #306 skip list) — mode P stays the @@ -97,13 +119,14 @@ object SubqueryResolver { private[client] def resolve( single: SingleSearch, execute: Execute, - correlation: SingleSearch => Option[String] = _ => None, - columnType: ColumnType = (_, _) => None + correlation: SingleSearch => Option[String], + columnType: ColumnType, + executes: Boolean ): ElasticResult[SingleSearch] = single.where.flatMap(_.criteria) match { case None => ElasticResult.success(single) case Some(criteria) => - rewrite(criteria, execute, correlation, columnType) match { + rewrite(criteria, execute, correlation, columnType, executes) match { case Right(c) if c eq criteria => ElasticResult.success(single) case Right(c) => ElasticResult.success(single.copy(where = Some(Where(Some(c))))) case Left(error) => ElasticResult.failure(error) @@ -119,28 +142,29 @@ object SubqueryResolver { criteria: Criteria, execute: Execute, correlation: SingleSearch => Option[String], - columnType: ColumnType + columnType: ColumnType, + executes: Boolean ): Either[ElasticError, Criteria] = criteria match { case p: Predicate => for { - l <- rewrite(p.leftCriteria, execute, correlation, columnType) - r <- rewrite(p.rightCriteria, execute, correlation, columnType) + l <- rewrite(p.leftCriteria, execute, correlation, columnType, executes) + r <- rewrite(p.rightCriteria, execute, correlation, columnType, executes) } yield if ((l eq p.leftCriteria) && (r eq p.rightCriteria)) p else p.copy(leftCriteria = l, rightCriteria = r) case n: ElasticNested => - rewrite(n.criteria, execute, correlation, columnType).map(c => + rewrite(n.criteria, execute, correlation, columnType, executes).map(c => if (c eq n.criteria) n else n.copy(criteria = c) ) case n: ElasticChild => - rewrite(n.criteria, execute, correlation, columnType).map(c => + rewrite(n.criteria, execute, correlation, columnType, executes).map(c => if (c eq n.criteria) n else n.copy(criteria = c) ) case n: ElasticParent => - rewrite(n.criteria, execute, correlation, columnType).map(c => + rewrite(n.criteria, execute, correlation, columnType, executes).map(c => if (c eq n.criteria) n else n.copy(criteria = c) ) - case s: SubqueryCriteria => resolveOne(s, execute, correlation, columnType) + case s: SubqueryCriteria => resolveOne(s, execute, correlation, columnType, executes) case other => Right(other) } @@ -148,7 +172,8 @@ object SubqueryResolver { node: SubqueryCriteria, execute: Execute, correlation: SingleSearch => Option[String], - columnType: ColumnType + columnType: ColumnType, + executes: Boolean ): Either[ElasticError, Criteria] = node.inner match { // Unreachable after `validate()`; `GatewayApi.run(statement)` does not validate a @@ -166,6 +191,8 @@ object SubqueryResolver { case Some(inner) => correlation(inner) match { case Some(reason) => Left(bad(reason)) + // SCHEMA-2 (D14) -- `describe`: the checks above only, nothing executed + case None if !executes => Right(node) case None => node match { case in: InSubquery => resolveIn(in, inner, execute, columnType) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala index dc064b55f..4c107972f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala @@ -385,7 +385,9 @@ class CoreDqlExtension extends ExtensionSpi { totalRows = None, warning = warning ) - ) + ), + // SCHEMA-2 (D13) -- the schema every route of a `UNION ALL` carries + schema = seam.toOption.flatMap(client.unionAllSchema) ) ) } @@ -648,8 +650,11 @@ class CoreDqlExtension extends ExtensionSpi { implicit val context: ConversionContext = NativeContext // The client's configured defaults (page size, slice ceiling — #238) plus the quota cap; the // cap is `.take(max)` on the MERGED stream, so it binds on the total whatever the slicing. - val source = - client.scroll(single, client.defaultScrollConfig.copy(maxDocuments = Some(max.toLong))) + val (schema, source) = + client.scrollWithSchema( + single, + client.defaultScrollConfig.copy(maxDocuments = Some(max.toLong)) + ) val warning = s"Result capped to $max rows (license quota). " + s"Add an explicit LIMIT <= $max, or upgrade for more rows." @@ -659,6 +664,8 @@ class CoreDqlExtension extends ExtensionSpi { totalRows = None, // capped at source — pre-cap total intentionally not scanned (OQ-4) warning = warning ) - Future.successful(ElasticSuccess(QueryStream(source, truncation = Some(signal)))) + Future.successful( + ElasticSuccess(QueryStream(source, truncation = Some(signal), schema = schema)) + ) } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 3f43458ba..b05ae8e00 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -27,6 +27,7 @@ import app.softnetwork.elastic.client.{ ElasticQuery, ElasticResponse, LegProjection, + ResultSchema, SingleValueAggregateResult } import app.softnetwork.elastic.client.bulk._ @@ -1176,12 +1177,22 @@ class MetricsElasticClient( override def scroll(statement: SearchStatement, config: ScrollConfig)(implicit system: ActorSystem, context: ConversionContext - ): Source[(ListMap[String, Any], ScrollMetrics), NotUsed] = { + ): Source[(ListMap[String, Any], ScrollMetrics), NotUsed] = + scrollWithSchema(statement, config)._2 + + /** SCHEMA-2 (D13) -- recorded as `scroll`: [[scroll]] is its source, so no metric is lost or + * counted twice. + */ + override private[client] def scrollWithSchema(statement: SearchStatement, config: ScrollConfig)( + implicit + system: ActorSystem, + context: ConversionContext + ): (Option[ResultSchema], Source[(ListMap[String, Any], ScrollMetrics), NotUsed]) = { // Note: For streams, we measure at the beginning but not every element val startTime = System.currentTimeMillis() - val source = delegate.scroll(statement, config) + val (schema, source) = delegate.scrollWithSchema(statement, config) - source.watchTermination() { (_, done) => + schema -> source.watchTermination() { (_, done) => done.onComplete { result => val duration = System.currentTimeMillis() - startTime val success = result.isSuccess diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index 4f917a734..1b1b4ad5c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -19,7 +19,6 @@ package app.softnetwork.elastic import akka.actor.ActorSystem import app.softnetwork.elastic.sql.function.aggregate._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} -import app.softnetwork.elastic.sql.renderedTypeOf import com.typesafe.scalalogging.LazyLogging import org.slf4j.Logger @@ -50,13 +49,17 @@ package object client extends SerializationApi { * - the field aliases used * @param aggregations * - the aggregations expected + * @param schema + * - the statement's result schema, declared before any row (SCHEMA-2); `None` where none is + * known */ case class ElasticResponse( sql: Option[String] = None, query: JSONQuery, results: Seq[ListMap[String, Any]], fieldAliases: ListMap[String, String], - aggregations: ListMap[String, ClientAggregation] + aggregations: ListMap[String, ClientAggregation], + schema: Option[ResultSchema] = None ) sealed trait ElasticAuthMethod { @@ -320,8 +323,8 @@ package object client extends SerializationApi { * gives that column (`SetOperationValues`), `null` where it already has it -- `None` for a * branch with nothing to convert, which is every branch of a statement whose columns agree * @param context - * how the BRANCH's rows are read: the context its own search reads them with (`TextColumns`, - * the columns the branch declares VARCHAR) -- `None` for the caller's own + * how the BRANCH's rows are read: the context its own search reads them with (`SchemaColumns`, + * each value by its declared type) -- `None` for the caller's own */ case class LegProjection( fieldAliases: ListMap[String, String], @@ -591,7 +594,7 @@ package object client extends SerializationApi { /** The reader of each conversion of a bare UNNEST element column, keyed by its row column * (`ElementConversions`), applied where the row explosion places the element's value. Empty - * for every context but the one [[TextColumns.of]] builds for such a statement. + * for every context but the one [[SchemaColumns.of]] builds for such a statement. */ private[client] def elementConversions: Map[String, Any => Any] = Map.empty } @@ -608,44 +611,29 @@ package object client extends SerializationApi { */ private[client] case object DocumentContext extends ConversionContext - /** The rows of a statement read natively (as [[NativeContext]]), except that the columns it - * declares VARCHAR are never re-read as a temporal: their text is their value. - * - * 🔴 A text that looks like a date or a time was parsed into a `java.time` value whatever the - * column was declared, so `DATETIME_FORMAT(ts, '%H:%i:%s')`, whose script returns `"00:00:00"`, - * answered `LocalTime` `00:00`: another class than the VARCHAR it reports, and another text. - * Built once per statement by [[TextColumns.of]], only around a [[NativeContext]]. + /** SCHEMA-2 (D3) -- the rows of a statement read natively (as [[NativeContext]]), each value by + * its column's declared type where it is first read ([[HitReading]]), and the statement's + * declared [[ResultSchema]] carried to the response. The ONE per-statement native context, built + * by [[SchemaColumns.of]] around a [[NativeContext]] only. * * `doubleKeys` names the levels the statement sends as a numeric `terms` of doubles * (`SingleSearch.doubleKeyLevels`), whose non-finite keys a response writes as text * (`ElasticConversion.extractBucketKey`). */ - private[client] final case class TextColumns( - columns: Set[String], + private[client] final case class SchemaColumns( + reading: HitReading, doubleKeys: Set[String] = Set.empty )(val elements: Map[String, Any => Any] = Map.empty) extends ConversionContext { override private[client] def elementConversions: Map[String, Any => Any] = elements } - private[client] object TextColumns { + private[client] object SchemaColumns { - /** `context` for the rows of `single`: [[TextColumns]] when it is native and `single` declares - * a VARCHAR column or sends a level as a numeric `terms` of doubles, `context` itself - * otherwise. - * - * The columns are named from the types the statement declares, each under the name its row - * carries it by: - * - a column the SELECT reads bare (no function, no script), by its output name and by the - * source column its document holds; - * - a computed column (a function, a script field), by its output name ONLY: the column it - * reads keeps its own type. Naming that column too re-read `ts` of `SELECT ts, - * DATE_FORMAT(ts, '%Y') AS y` and `d` of `CAST(d AS VARCHAR) AS s` as texts; - * - `*`, the columns the resolved schema declares VARCHAR. - * - * It also carries the reader of each conversion of a bare UNNEST element column `single` - * projects (`ElementConversions`), whose dates are read as the Elasticsearch `version` reads - * them (asked only when a date is read). + /** `context` for the rows of `single`: [[SchemaColumns]] when it is native, `context` itself + * otherwise. Its dates -- and the conversion of each bare UNNEST element column `single` + * projects (`ElementConversions`) -- are read as the Elasticsearch `version` reads them, asked + * only when a date is read. */ def of( single: SingleSearch, @@ -654,21 +642,24 @@ package object client extends SerializationApi { ): ConversionContext = context match { case NativeContext => - val columns = single.select.fieldsWithComputedAliases.flatMap { field => - if (field.identifier.identifierName == "*") - single.schema.toSeq.flatMap(_.columns.collect { - case column if column.dataType.isText => column.name - }) - else if (!renderedTypeOf(field.identifier).isText) Nil - else if (field.identifier.functions.isEmpty && !field.isScriptField) - Seq(field.outputName, field.identifier.name).filter(_.nonEmpty) - else Seq(field.outputName).filter(_.nonEmpty) - }.toSet - val doubleKeys = single.doubleKeyLevels val elements = ElementConversions.of(single, version) - if (columns.isEmpty && doubleKeys.isEmpty && elements.isEmpty) context - else TextColumns(columns, doubleKeys)(elements) + SchemaColumns( + HitReading.of( + single, + SetOperationValues.DateReading(version), + HitReading.Default, + elements.keySet + ), + single.doubleKeyLevels + )(elements) case other => other } + + /** The schema a response read in `context` carries (D13). */ + def schemaOf(context: ConversionContext): Option[ResultSchema] = + context match { + case s: SchemaColumns => s.reading.schema + case _ => None + } } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/repl/Repl.scala b/core/src/main/scala/app/softnetwork/elastic/client/repl/Repl.scala index 5627c3f14..4443b1b73 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/repl/Repl.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/repl/Repl.scala @@ -571,6 +571,8 @@ class Repl( var displayedRows = 0 val allRows = scala.collection.mutable.ListBuffer[ListMap[String, Any]]() + // SCHEMA-2 (D13) -- the stream's schema, read before the consumption ends the stream + val schema = executor.activeStreamSchema val resultFuture = executor.consumeStream( batchSize = batchSize, @@ -596,7 +598,7 @@ class Repl( // Render accumulated rows if (allRows.nonEmpty) { val output = ResultRenderer.render( - QueryRows(allRows.toSeq), + QueryRows(allRows.toSeq, schema = schema), result.duration, config.format ) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/repl/StreamingReplExecutor.scala b/core/src/main/scala/app/softnetwork/elastic/client/repl/StreamingReplExecutor.scala index 2a9d822d8..456a51f9f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/repl/StreamingReplExecutor.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/repl/StreamingReplExecutor.scala @@ -21,7 +21,7 @@ package app.softnetwork.elastic.client.repl import akka.actor.ActorSystem import akka.stream.Materializer import akka.stream.scaladsl.{Sink, Source} -import app.softnetwork.elastic.client.ElasticClientApi +import app.softnetwork.elastic.client.{ElasticClientApi, ResultSchema} import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, @@ -48,7 +48,8 @@ class StreamingReplExecutor(gateway: ElasticClientApi)(implicit case class StreamContext( source: Source[ListMap[String, Any], _], name: String, - startTime: Long + startTime: Long, + schema: Option[ResultSchema] = None ) /** Execute SQL and handle streaming results @@ -62,9 +63,9 @@ class StreamingReplExecutor(gateway: ElasticClientApi)(implicit val executionTime = (System.nanoTime() - startTime).nanos elasticResult match { - case ElasticSuccess(QueryStream(source, _)) => - // Store stream context for later consumption - activeStream = Some(StreamContext(source.map(_._1), sql, startTime)) + case ElasticSuccess(QueryStream(source, _, schema)) => + // Store stream context for later consumption -- with its schema (SCHEMA-2, D13) + activeStream = Some(StreamContext(source.map(_._1), sql, startTime, schema)) ExecutionSuccess( StreamResult(estimatedSize = None, isActive = true), executionTime @@ -149,6 +150,9 @@ class StreamingReplExecutor(gateway: ElasticClientApi)(implicit */ def hasActiveStream: Boolean = activeStream.isDefined + /** The active stream's result schema (SCHEMA-2): the header its collected rows render under. */ + def activeStreamSchema: Option[ResultSchema] = activeStream.flatMap(_.schema) + def invalidateSchema(index: String): Unit = gateway.invalidateSchema(index) def invalidateAllSchemas(): Unit = gateway.invalidateAllSchemas() } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/CsvFormatter.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/CsvFormatter.scala index 280c2c761..863e3fd4c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/CsvFormatter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/CsvFormatter.scala @@ -23,8 +23,10 @@ object CsvFormatter { @tailrec def format(result: QueryResult): String = { result match { - case QueryRows(rows, _) if rows.nonEmpty => - val columns = rows.head.keys.toSeq + case QueryRows(rows, _, schema) if rows.nonEmpty => + // SCHEMA-2 (A-14) -- the header in schema order when the statement declares one: a row + // keeps its document's key order + val columns = schema.map(_.names).filter(_.nonEmpty).getOrElse(rows.head.keys.toSeq) val header = columns.mkString(",") val body = rows .map { row => @@ -34,7 +36,7 @@ object CsvFormatter { s"$header\n$body" case QueryStructured(response, _) if response.results.nonEmpty => - format(QueryRows(response.results)) + format(QueryRows(response.results, schema = response.schema)) case _ => "No data" @@ -43,8 +45,10 @@ object CsvFormatter { private def escapeCsv(value: Any): String = { val str = value match { - case null => "" - case other => other.toString + case null => "" + // SCHEMA-2 (D2) -- a VARBINARY's text is its base64, as in 0.24.0 + case b: Array[Byte] => java.util.Base64.getEncoder.encodeToString(b) + case other => other.toString } if (str.contains(",") || str.contains("\"") || str.contains("\n")) { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala index 772005936..d0096def7 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala @@ -27,7 +27,7 @@ object JsonFormatter { def format(result: QueryResult, executionTime: Duration): String = { result match { - case QueryRows(rows, _) => + case QueryRows(rows, _, _) => val json = JObject( "rows" -> JArray(rows.map(rowToJson).toList), "count" -> JInt(rows.size), @@ -76,12 +76,16 @@ object JsonFormatter { } private def valueToJson(value: Any): JValue = value match { - case null => JNull - case s: String => JString(s) - case i: Short => JInt(i) - case i: Int => JInt(i) - case l: Long => JLong(l) - case d: Double => JDouble(d) + case null => JNull + case s: String => JString(s) + case i: Short => JInt(i) + case i: Int => JInt(i) + case l: Long => JLong(l) + case d: Double => JDouble(d) + // SCHEMA-2 (D2) -- a REAL is a number, its bytes as JSON spells the float + case f: Float => JDouble(java.lang.Double.parseDouble(java.lang.Float.toString(f))) + // ...and a VARBINARY its base64 text, the text 0.24.0 answered + case b: Array[Byte] => JString(java.util.Base64.getEncoder.encodeToString(b)) case b: Boolean => JBool(b) case seq: Seq[_] => JArray(seq.map(valueToJson).toList) case map: Map[_, _] => JObject(map.map { case (k, v) => k.toString -> valueToJson(v) }.toList) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala index ea7d0e7ae..c282516c2 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala @@ -48,11 +48,11 @@ object ResultRenderer { case EmptyResult => renderEmpty() - case QueryRows(rows, _) => - renderTable(rows, executionTime) + case QueryRows(rows, _, schema) => + renderTable(rows, executionTime, schema) case QueryStructured(response, _) => - renderTable(response.results, executionTime) + renderTable(response.results, executionTime, response.schema) case DmlResult(inserted, updated, deleted, rejected) => renderDml(inserted, updated, deleted, rejected, executionTime) @@ -69,7 +69,7 @@ object ResultRenderer { case SQLResult(sql) => renderSql(sql) - case QueryStream(_, _) => + case QueryStream(_, _, _) => renderStreamInfo() case StreamResult(estimatedSize, _) => @@ -87,13 +87,21 @@ object ResultRenderer { // ==================== Table Rendering ==================== - private def renderTable(rows: Seq[Map[String, Any]], executionTime: Duration): String = { + private[result] def renderTable( + rows: Seq[Map[String, Any]], + executionTime: Duration, + schema: Option[app.softnetwork.elastic.client.ResultSchema] = None + ): String = { if (rows.isEmpty) { return s"${emoji("✅")} ${green("No rows returned")} ${gray(s"(${executionTime.toMillis}ms)")}" } - // Extract columns - val columnNames: Seq[String] = rows.headOption.map(_.keys.toSeq).getOrElse(Seq.empty) + // Extract columns -- SCHEMA-2 (A-14): in schema order when the statement declares one, a row + // keeping its document's key order + val columnNames: Seq[String] = schema + .map(_.names) + .filter(_.nonEmpty) + .getOrElse(rows.headOption.map(_.keys.toSeq).getOrElse(Seq.empty)) // Prepare headers with colors val headers = columnNames.map(col => bold(cyan(col))) @@ -164,6 +172,8 @@ object ResultRenderer { case b: Boolean => if (b) green("true") else red("false") case d: java.time.temporal.TemporalAccessor => magenta(d.toString) case seq: Seq[_] => s"[${seq.map(formatValue).mkString(", ")}]" + // SCHEMA-2 (D2) -- a VARBINARY's text is its base64, as in 0.24.0 + case b: Array[Byte] => java.util.Base64.getEncoder.encodeToString(b) case map: Map[_, _] => s"{${map.map { case (k, v) => s"$k: ${formatValue(v)}" }.mkString(", ")}}" case other => other.toString diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index d35b02d0d..4aa7fe547 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -428,25 +428,37 @@ package object result { * Story P0.5: the additive, default-`None` `truncation` field carries the single-index result * cap signal. Because the field has a default value, every `QueryRows(rows)` '''constructor''' * call across core / arrow / jdbc keeps compiling unchanged. Positional '''pattern''' matches - * (`case QueryRows(rows)`) must add a trailing `_` (`case QueryRows(rows, _)`) — the compiler's - * synthetic extractor now binds both fields. (A custom 1-arg companion `unapply` was rejected: - * Scala 2.12 still prefers the synthetic 2-arg extractor, so it is not cross-version safe.) + * (`case QueryRows(rows)`) must add a trailing `_` per field (`case QueryRows(rows, _, _)` since + * SCHEMA-2's `schema`) — the compiler's synthetic extractor binds every field. (A custom 1-arg + * companion `unapply` was rejected: Scala 2.12 still prefers the synthetic 2-arg extractor, so + * it is not cross-version safe.) */ case class QueryRows( rows: Seq[ListMap[String, Any]], - truncation: Option[ResultTruncation] = None + truncation: Option[ResultTruncation] = None, + schema: Option[ResultSchema] = None ) extends QueryResult + /** SCHEMA-2 (D13): `schema` -- the statement's result schema, declared before any row is read; + * `None` where none is known (no mapping, a DDL / DML / SHOW result). Positional patterns bind + * THREE fields (`case QueryRows(rows, _, _)`); the shape immune to the next arity change is + * `case q: QueryRows => q.rows`. + */ case class QueryStream( stream: Source[(ListMap[String, Any], ScrollMetrics), NotUsed], - truncation: Option[ResultTruncation] = None + truncation: Option[ResultTruncation] = None, + schema: Option[ResultSchema] = None ) extends QueryResult case class QueryStructured( response: ElasticResponse, truncation: Option[ResultTruncation] = None ) extends QueryResult { - def asQueryRows: QueryRows = QueryRows(response.results, truncation) + + /** The statement's result schema, carried by its response (A-18). */ + def schema: Option[ResultSchema] = response.schema + + def asQueryRows: QueryRows = QueryRows(response.results, truncation, response.schema) } case class StreamResult( diff --git a/core/src/test/scala/app/softnetwork/elastic/client/DateDialectVersionSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/DateDialectVersionSpec.scala index 90f86294a..7df091b5c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/DateDialectVersionSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/DateDialectVersionSpec.scala @@ -207,14 +207,15 @@ class DateDialectVersionSpec extends AnyFlatSpec with Matchers { ).foreach { case (sql, values) => withClue(sql)(rows(forbidden(), sql, values) shouldBe rows(answering("8.18.3"), sql, values)) } - // an agreeing date of a format of its own is read, a zone name kept as stored + // an agreeing date of a format of its own is read; a zone-name format makes its column + // VARCHAR (SCHEMA-2, D12: core reads no value of it for sure), every value its text as stored rows(forbidden(), union("c_c1", "c_c1"), Seq("\"2024/01/31\"")) shouldBe Seq(midnight) rows( forbidden(), union("tsz", "tsz"), Seq("\"2024-01-31 10:30 UTC\"", "\"2024-01-31 10:30 CET\"") ) shouldBe Seq( - shown(utc("2024-01-31T10:30:00Z")), + shown("2024-01-31 10:30 UTC"), shown("2024-01-31 10:30 CET") ) } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/DeclaredTextColumnSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/DeclaredTextColumnSpec.scala index 69868abf9..230e5f25f 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/DeclaredTextColumnSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/DeclaredTextColumnSpec.scala @@ -23,11 +23,11 @@ import app.softnetwork.elastic.sql.schema.Schema import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.time.{LocalDate, LocalTime, ZonedDateTime} +import java.time.{LocalTime, ZoneOffset, ZonedDateTime} import scala.collection.immutable.ListMap /** A column a statement declares VARCHAR is read as its text, never re-read as a temporal - * ([[TextColumns]]). The same statement is executed on every major by the shared testkit trait + * ([[SchemaColumns]]). The same statement is executed on every major by the shared testkit trait * `GroupedAggregateSemanticsSpec`. */ class DeclaredTextColumnSpec extends AnyFlatSpec with Matchers { @@ -47,7 +47,8 @@ class DeclaredTextColumnSpec extends AnyFlatSpec with Matchers { /** The value of `v` the hit `fields` entry `text` answers for `sql`. */ private def valueOf(sql: String, text: String): Any = { val single = statement(sql) - implicit val context: ConversionContext = TextColumns.of(single, NativeContext) + implicit val context: ConversionContext = + SchemaColumns.of(single, NativeContext, Some("8.18.3")) val response = s"""{"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [ | {"_id": "a3", "_source": {"id": "a3"}, "fields": {"v": ["$text"]}}]}}""".stripMargin @@ -77,7 +78,8 @@ class DeclaredTextColumnSpec extends AnyFlatSpec with Matchers { */ private def rowOf(sql: String, source: String, fields: String = "{}"): ListMap[String, Any] = { val single = statement(sql) - implicit val context: ConversionContext = TextColumns.of(single, NativeContext) + implicit val context: ConversionContext = + SchemaColumns.of(single, NativeContext, Some("8.18.3")) val response = s"""{"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [ | {"_id": "a1", "_source": $source, "fields": $fields}]}}""".stripMargin @@ -109,7 +111,8 @@ class DeclaredTextColumnSpec extends AnyFlatSpec with Matchers { """{"id": "a1", "d": "2024-01-03"}""", """{"s": ["2024-01-03"]}""" ) - cast("d") shouldBe LocalDate.of(2024, 1, 3) + // SCHEMA-2 (D2): `d`, a `date` field declared by no `_meta`, is a TIMESTAMP -- read at UTC + cast("d") shouldBe ZonedDateTime.of(2024, 1, 3, 0, 0, 0, 0, ZoneOffset.UTC) cast("s") shouldBe a[String] } @@ -119,13 +122,13 @@ class DeclaredTextColumnSpec extends AnyFlatSpec with Matchers { rowOf("SELECT id, kd AS k2 FROM t", source)("k2") shouldBe "2025-03-31T10:00:00" val star = rowOf("SELECT * FROM t", source) star("kd") shouldBe "2025-03-31T10:00:00" - // the control: a DATE column of the same row keeps its type - star("d") shouldBe LocalDate.of(2024, 1, 3) - // and without a schema nothing is declared VARCHAR + // the control: a date column of the same row keeps its type (SCHEMA-2, D2: a TIMESTAMP) + star("d") shouldBe ZonedDateTime.of(2024, 1, 3, 0, 0, 0, 0, ZoneOffset.UTC) + // and without a schema no schema is declared (SCHEMA-2, D10: rows as read) val context: ConversionContext = Parser("SELECT * FROM t") match { - case Right(s: SingleSearch) => TextColumns.of(s, NativeContext) + case Right(s: SingleSearch) => SchemaColumns.of(s, NativeContext) case other => fail(s"$other") } - context shouldBe NativeContext + SchemaColumns.schemaOf(context) shouldBe None } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala new file mode 100644 index 000000000..85799c64e --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/DescribeSpec.scala @@ -0,0 +1,277 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.query.Statement +import app.softnetwork.elastic.sql.schema.Schema +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.typesafe.config.ConfigFactory +import org.scalatest.BeforeAndAfterAll +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.util.concurrent.atomic.AtomicInteger +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future} + +/** SCHEMA-2 (D14, AC1) -- `describe(sql)` answers `run(sql)`'s schema, running nothing: no search + * (one-shot, `_msearch`, async twins, scroll), no subquery; and refuses iff `run`'s resolve + * refuses, with the same status and message. + */ +class DescribeSpec extends AnyFlatSpec with Matchers with ScalaFutures with BeforeAndAfterAll { + + implicit private val system: ActorSystem = ActorSystem("describe-spec") + override implicit val patienceConfig: PatienceConfig = + PatienceConfig(timeout = scaled(20.seconds)) + + override def afterAll(): Unit = { + system.terminate() + super.afterAll() + } + + private val mapper = new ObjectMapper() + + private def schemaOf(index: String, properties: String, meta: String = "{}"): Schema = + Index( + index, + s"""{"$index": {"aliases": {}, "mappings": {"_meta": {"columns": $meta}, + | "properties": $properties}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + ).schema + + private val s2: Schema = schemaOf( + "s2_t", + """{"id": {"type": "keyword"}, "k": {"type": "keyword"}, "ts": {"type": "date"}, + | "c_des": {"type": "date", "format": "epoch_second"}, + | "c_c1": {"type": "date", "format": "yyyy/MM/dd"}, + | "tsz": {"type": "date", "format": "yyyy-MM-dd HH:mm z"}, + | "n": {"type": "integer"}, "x": {"type": "double"}, "b": {"type": "boolean"}, + | "g": {"type": "geo_point"}, "bin": {"type": "binary"}, + | "obj": {"properties": {"a": {"type": "integer"}}}}""".stripMargin, + """{"c_des": {"data_type": "DATE"}}""" + ) + + private val other: Schema = + schemaOf("other_t", """{"oid": {"type": "keyword"}, "m": {"type": "integer"}}""") + + private val emptyHits: JsonNode = + mapper.readTree("""{"hits": {"total": {"value": 0, "relation": "eq"}, "hits": []}}""") + + /** Counts every search a statement could send; seeded schemas, a known version. */ + private class CountingClient extends NopeClientApi { + override protected def logger: Logger = LoggerFactory.getLogger(getClass) + + val searches = new AtomicInteger(0) + + override private[client] def executeVersion(): ElasticResult[String] = + ElasticResult.success("8.18.3") + + override private[client] def executeSingleSearch( + elasticQuery: ElasticQuery + ): ElasticResult[Option[JsonNode]] = { + searches.incrementAndGet() + ElasticResult.success(Some(emptyHits)) + } + + override private[client] def executeMultiSearch( + elasticQueries: ElasticQueries + ): ElasticResult[Option[JsonNode]] = { + searches.incrementAndGet() + ElasticResult.success(Some(responses(elasticQueries))) + } + + override private[client] def executeSingleSearchAsync(elasticQuery: ElasticQuery)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[JsonNode]]] = { + searches.incrementAndGet() + Future.successful(ElasticResult.success(Some(emptyHits))) + } + + override private[client] def executeMultiSearchAsync(elasticQueries: ElasticQueries)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[JsonNode]]] = { + searches.incrementAndGet() + Future.successful(ElasticResult.success(Some(responses(elasticQueries)))) + } + + /** A scroll is OPENED here: counted even when nobody materialises it. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: ActorSystem, + context: ConversionContext + ) = { + searches.incrementAndGet() + super.scrollWithSchema(statement, config) + } + + private def responses(elasticQueries: ElasticQueries): JsonNode = { + val root = mapper.createObjectNode() + val array = root.putArray("responses") + elasticQueries.queries.foreach(_ => array.add(emptyHits.deepCopy[JsonNode]())) + root + } + } + + private def client(): CountingClient = { + val c = new CountingClient + c.updateSchema("s2_t", s2) + c.updateSchema("other_t", other) + c + } + + private def runSchema(c: CountingClient, sql: String): ElasticResult[Option[ResultSchema]] = + c.run(sql).futureValue match { + case ElasticSuccess(q: QueryRows) => ElasticSuccess(q.schema) + case ElasticSuccess(q: QueryStream) => ElasticSuccess(q.schema) + case ElasticSuccess(q: QueryStructured) => ElasticSuccess(q.schema) + case ElasticSuccess(other: QueryResult) => fail(s"[$sql] answered $other") + case ElasticFailure(error) => ElasticFailure(error) + } + + /** P1, generated over the fixture's columns, not hand-picked. */ + private val p1: Seq[String] = { + val columns = Seq("id", "k", "ts", "c_des", "c_c1", "tsz", "n", "x", "b", "g", "bin", "obj") + columns.flatMap { c => + Seq( + s"SELECT $c FROM s2_t", + s"SELECT $c AS kk FROM s2_t LIMIT 10", + s"SELECT CAST($c AS VARCHAR) AS s FROM s2_t LIMIT 10" + ) + } ++ Seq( + "SELECT * FROM s2_t", + "SELECT * FROM s2_t LIMIT 10", + "SELECT k, COUNT(*) AS c, MIN(n) AS mi, MAX(n) AS ma FROM s2_t GROUP BY k", + "SELECT id, ROW_NUMBER() OVER (ORDER BY id) AS rn FROM s2_t LIMIT 10", + "SELECT id, n + 1 AS v FROM s2_t LIMIT 10", + "SELECT DATE_FORMAT(ts, '%Y') AS y FROM s2_t LIMIT 10", + "SELECT UPPER(k) AS u, n FROM s2_t", + "SELECT n AS v FROM s2_t LIMIT 5 UNION ALL SELECT n AS v FROM s2_t LIMIT 5", + "SELECT n AS v FROM s2_t LIMIT 5 UNION ALL SELECT x AS v FROM s2_t LIMIT 5", + "SELECT ts AS v FROM s2_t LIMIT 5 UNION ALL SELECT c_des AS v FROM s2_t LIMIT 5", + "SELECT tsz AS v FROM s2_t LIMIT 5 UNION ALL SELECT ts AS v FROM s2_t LIMIT 5", + "SELECT id FROM s2_t UNION ALL SELECT oid AS id FROM other_t" + ) + } + + "describe" should "answer run's schema over P1, running no search (AC1)" in { + p1.foreach { sql => + withClue(s"[$sql] ") { + val c = client() + val described = c.describe(sql) + c.searches.get() shouldBe 0 + val ran = runSchema(client(), sql) + (described, ran) match { + case (ElasticSuccess(d), ElasticSuccess(r)) => d shouldBe r + case (ElasticFailure(d), ElasticFailure(r)) => + d.statusCode shouldBe r.statusCode + d.message shouldBe r.message + case other => fail(s"describe and run disagree: $other") + } + } + } + // the cells answer a schema at all + client().describe("SELECT n, ts FROM s2_t") match { + case ElasticSuccess(Some(schema)) => + schema.columns.map(c => c.name -> c.sqlType) shouldBe Seq( + "n" -> SQLTypes.Int, + "ts" -> SQLTypes.Timestamp + ) + case other => fail(s"$other") + } + } + + it should "run no subquery, and refuse a bare-name correlation as run does" in { + val c = client() + val uncorrelated = "SELECT id FROM s2_t WHERE n IN (SELECT m FROM other_t) LIMIT 10" + c.describe(uncorrelated) match { + case ElasticSuccess(Some(schema)) => schema.names shouldBe Seq("id") + case other => fail(s"$other") + } + c.searches.get() shouldBe 0 + // a column of the outer index named bare in the inner statement: refused before execution + val correlated = "SELECT id FROM s2_t WHERE n IN (SELECT m FROM other_t WHERE m = x) LIMIT 10" + val described = client().describe(correlated) + val ran = runSchema(client(), correlated) + (described, ran) match { + case (ElasticFailure(d), ElasticFailure(r)) => + d.statusCode shouldBe r.statusCode + d.message shouldBe r.message + case (ElasticSuccess(d), ElasticSuccess(r)) => d shouldBe r + case other => fail(s"describe and run disagree: $other") + } + } + + it should "refuse a JOIN as run does, and answer None under another extension" in { + val join = "SELECT a.id FROM s2_t a JOIN other_t b ON a.id = b.oid" + (client().describe(join), runSchema(client(), join)) match { + case (ElasticFailure(d), ElasticFailure(r)) => + d.statusCode shouldBe r.statusCode + d.message shouldBe r.message + case other => fail(s"describe and run disagree on a JOIN: $other") + } + val foreign = new ExtensionSpi { + override def extensionId: String = "describe-foreign-test" + override def extensionName: String = "Foreign handler (test double)" + override def version: String = "test" + override def priority: Int = 1 + override def initialize( + config: com.typesafe.config.Config, + licenseRefreshStrategy: app.softnetwork.elastic.licensing.LicenseRefreshStrategy + ): Either[String, Unit] = Right(()) + override def canHandle(statement: Statement): Boolean = true + override def execute(statement: Statement, client: ElasticClientApi)(implicit + system: ActorSystem + ): Future[ElasticResult[QueryResult]] = Future.successful(ElasticSuccess(QueryRows(Nil))) + override def supportedSyntax: Seq[String] = Seq.empty + } + val c = new CountingClient { + override lazy val extensionRegistry: ExtensionRegistry = + new ExtensionRegistry(ConfigFactory.load(), licenseRefreshStrategy) { + override lazy val extensions: Seq[ExtensionSpi] = Seq(foreign) + } + } + c.describe(join) shouldBe ElasticSuccess(None) + c.searches.get() shouldBe 0 + } + + it should "answer a FROM-less statement, and refuse what is not a query (400)" in { + client().describe("SELECT 1 AS one") match { + case ElasticSuccess(Some(schema)) => + schema.columns.map(c => c.name -> c.sqlType) shouldBe Seq("one" -> SQLTypes.Int) + case other => fail(s"$other") + } + Seq("CREATE TABLE zz (a INT)", "SELECT 1; SELECT 2", "SELEC nothing").foreach { sql => + client().describe(sql) match { + case ElasticFailure(error) => error.statusCode shouldBe Some(400) + case other => fail(s"[$sql] answered $other") + } + } + } + + it should "answer None where run carries no schema (no mapping)" in { + val c = new CountingClient // no seed: every lookup misses + c.describe("SELECT id FROM nowhere_t") shouldBe ElasticSuccess(None) + c.searches.get() shouldBe 0 + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/FromlessSelectGatewaySpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/FromlessSelectGatewaySpec.scala index 86b504665..d5f3e7fb4 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/FromlessSelectGatewaySpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/FromlessSelectGatewaySpec.scala @@ -57,8 +57,8 @@ class FromlessSelectGatewaySpec private def rows(result: ElasticResult[QueryResult]): Seq[ListMap[String, Any]] = result match { - case ElasticSuccess(QueryRows(r, _)) => r - case other => fail(s"expected QueryRows, got $other") + case ElasticSuccess(QueryRows(r, _, _)) => r + case other => fail(s"expected QueryRows, got $other") } // ── Fixture B: a NopeClientApi whose search answers from a response QUEUE and which diff --git a/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala new file mode 100644 index 000000000..2f77f4488 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala @@ -0,0 +1,467 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.SetOperationValues.DateReading +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.schema.Schema +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.time.{LocalDate, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap + +/** SCHEMA-2 (D2, D4-D9) -- every value core reads from a hit is read by its column's declared type + * where it is first read: the class oracle is D2's table, never a value equality alone. + */ +class HitReadingSpec extends AnyFlatSpec with Matchers { + + private val mapping: String = + """{"s2_t": {"aliases": {}, "mappings": { + | "_meta": {"columns": {"c_des": {"data_type": "DATE"}}}, + | "properties": { + | "id": {"type": "keyword"}, + | "ts": {"type": "date"}, + | "c_des": {"type": "date", "format": "epoch_second"}, + | "c_c1": {"type": "date", "format": "yyyy/MM/dd"}, + | "tsz": {"type": "date", "format": "yyyy-MM-dd HH:mm z"}, + | "dm": {"type": "date", "ignore_malformed": true}, + | "n": {"type": "integer"}, "x": {"type": "double"}, "r": {"type": "float"}, + | "l": {"type": "long"}, "s": {"type": "short"}, "tb": {"type": "byte"}, + | "b": {"type": "boolean"}, "g": {"type": "geo_point"}, "bin": {"type": "binary"}, + | "k": {"type": "keyword"}, "kw": {"type": "keyword", "ignore_above": 5}, + | "nul": {"type": "keyword"}, + | "obj": {"properties": {"a": {"type": "integer"}, "d": {"type": "date"}}} + | }}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + + private val schema: Schema = Index("s2_t", mapping).schema + + private def statement(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(schema)) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + private def outputs(single: SingleSearch): Seq[String] = { + val fields = single.select.fieldsWithComputedAliases + if (fields.size == 1 && fields.head.identifier.identifierName == "*") Seq.empty + else fields.map(_.outputName) + } + + private def contextOf( + single: SingleSearch, + version: String = "8.18.3", + multiValued: MultiValued = HitReading.Default + ): ConversionContext = { + val elements = ElementConversions.of(single, Some(version)) + SchemaColumns( + HitReading.of(single, DateReading(Some(version)), multiValued, elements.keySet), + single.doubleKeyLevels + )(elements) + } + + /** The rows `sql` answers for the hits `hits` (a JSON array body). */ + private def rows( + sql: String, + hits: String, + version: String = "8.18.3", + multiValued: MultiValued = HitReading.Default + ): Seq[ListMap[String, Any]] = { + val single = statement(sql) + implicit val context: ConversionContext = contextOf(single, version, multiValued) + val response = s"""{"hits": {"total": {"value": 1, "relation": "eq"}, "hits": [$hits]}}""" + ElasticConversion + .parseResponse(response, single.fieldAliases, ListMap.empty, outputs(single)) + .get + } + + private def row(sql: String, source: String, extra: String = ""): ListMap[String, Any] = + rows(sql, s"""{"_id": "a1", "_source": $source$extra}""").head + + private def utc(y: Int, mo: Int, d: Int, h: Int = 0, mi: Int = 0): ZonedDateTime = + ZonedDateTime.of(y, mo, d, h, mi, 0, 0, ZoneOffset.UTC) + + "A hit read by its plan" should "answer each column in D2's class (types)" in { + val r = row( + "SELECT id, ts, c_des, c_c1, n, x, r, l, s, tb, b, k, bin FROM s2_t", + """{"id": "1", "ts": "2024-01-31T10:30:00", "c_des": 1706697000, "c_c1": "2024/01/31", + | "n": "5.5", "x": " 2.5 ", "r": 1.5, "l": 7, "s": 3, "tb": 2, "b": "", "k": 10, + | "bin": "aGVsbG8="}""".stripMargin + ) + r.keys.toSeq shouldBe Seq( + "id", + "ts", + "c_des", + "c_c1", + "n", + "x", + "r", + "l", + "s", + "tb", + "b", + "k", + "bin" + ) + r("id") shouldBe "1" + r("ts") shouldBe utc(2024, 1, 31, 10, 30) + r("ts").asInstanceOf[ZonedDateTime].getZone shouldBe ZoneOffset.UTC + r("c_des") shouldBe LocalDate.of(2024, 1, 31) + r("c_c1") shouldBe utc(2024, 1, 31) + r("n") shouldBe java.lang.Integer.valueOf(5) + r("x") shouldBe java.lang.Double.valueOf(2.5) + r("r") shouldBe java.lang.Float.valueOf(1.5f) + r("l") shouldBe java.lang.Long.valueOf(7L) + r("s") shouldBe a[java.lang.Integer] + r("tb") shouldBe a[java.lang.Integer] + r("b") shouldBe java.lang.Boolean.FALSE + r("k") shouldBe "10" + new String(r("bin").asInstanceOf[Array[Byte]], StandardCharsets.US_ASCII) shouldBe "hello" + } + + it should "answer a date its format cannot read for sure as its text as stored (AsStored)" in { + row("SELECT tsz FROM s2_t", """{"tsz": "2024-01-31 10:30 CET"}""")( + "tsz" + ) shouldBe "2024-01-31 10:30 CET" + } + + it should "read every spelling of a point as {lat, lon} (geo)" in { + val point = + ListMap("lat" -> java.lang.Double.valueOf(1.5), "lon" -> java.lang.Double.valueOf(2.5)) + Seq( + """{"lat": 1.5, "lon": 2.5}""", + "\"1.5,2.5\"", + "[2.5, 1.5]", + "\"POINT (2.5 1.5)\"" + ).foreach { g => + withClue(g)(row("SELECT g FROM s2_t", s"""{"g": $g}""")("g") shouldBe point) + } + // a geohash: its cell's south-west corner, the one rule (`SetOperationValues.geoPoint`) + row("SELECT g FROM s2_t", """{"g": "s0"}""")("g") shouldBe + SetOperationValues.geoPoint("s0", "g") + } + + it should "unwrap a one-element array and answer an empty one NULL (unwrap)" in { + row("SELECT n FROM s2_t", """{"n": [5]}""")("n") shouldBe java.lang.Integer.valueOf(5) + row("SELECT n FROM s2_t", """{"n": []}""")("n") shouldBe (null: Any) + // ONE value in arrays of one is that value, at the column and at a struct's field + row("SELECT n FROM s2_t", """{"n": [[5]]}""")("n") shouldBe java.lang.Integer.valueOf(5) + row("SELECT c_des FROM s2_t", """{"c_des": [[1706697000]]}""")("c_des") shouldBe + LocalDate.of(2024, 1, 31) + // several values in an array of one: as stored, its nesting kept (D6), never a failure + row("SELECT c_des FROM s2_t", """{"c_des": [[1706697000, 1706697001]]}""")("c_des") shouldBe + List(List(1706697000, 1706697001)) + row("SELECT obj FROM s2_t", """{"obj": {"a": [[1, 2]], "d": [["2024-01-31"]]}}""")( + "obj" + ) shouldBe ListMap("a" -> List(List(1, 2)), "d" -> utc(2024, 1, 31)) + // ...a statement failing several values fails it too (D7) + intercept[Exception]( + rows( + "SELECT n FROM s2_t", + """{"_id": "doc8", "_source": {"n": [[1, 2]]}}""", + multiValued = MultiValued.Fail + ) + ) + // a computed column: Elasticsearch's array is ONE value + val computed = + rows( + "SELECT id, n + 1 AS v FROM s2_t", + """{"_id": "a1", "_source": {"id": "1"}, "fields": {"v": [4]}}""" + ).head + computed("v") shouldBe 4 + val upper = rows( + "SELECT id, UPPER(k) AS u FROM s2_t", + """{"_id": "a1", "_source": {"id": "1"}, "fields": {"u": ["X"]}}""" + ).head + upper("u") shouldBe "X" + } + + it should "read several values per MultiValued (multi)" in { + row("SELECT n FROM s2_t", """{"n": [1, "2"]}""")("n") shouldBe List(1, 2) + val failure = intercept[Exception] { + rows( + "SELECT n FROM s2_t", + """{"_id": "doc7", "_source": {"n": [1, 2]}}""", + multiValued = MultiValued.Fail + ) + } + val message = Option(failure.getCause).fold(failure.getMessage)(_.getMessage) + message should include("Column n") + message should include("doc7") + message should include("ALTER TABLE s2_t ALTER COLUMN n TYPE ARRAY") + } + + it should "answer a date Elasticsearch ignored NULL, a keyword it ignored as stored (ignored)" in { + val r = row( + "SELECT dm, kw FROM s2_t", + """{"dm": "not-a-date", "kw": "abcdefgh"}""", + """, "_ignored": ["dm", "kw"]""" + ) + r("dm") shouldBe (null: Any) + r("kw") shouldBe "abcdefgh" + // without `_ignored`, an unreadable date is as stored (D6), never a failure + row("SELECT dm FROM s2_t", """{"dm": "not-a-date"}""")("dm") shouldBe "not-a-date" + // Elasticsearch's verdict is per VALUE: in an array, the values it indexed keep theirs and only + // the ones it refused are NULL + row("SELECT dm FROM s2_t", """{"dm": ["2024-01-01", "bad"]}""", """, "_ignored": ["dm"]""")( + "dm" + ) shouldBe List(utc(2024, 1, 1), null) + row("SELECT dm FROM s2_t", """{"dm": ["bad"]}""", """, "_ignored": ["dm"]""")("dm") shouldBe + (null: Any) + // ...and a date `_ignored` does not name is read as usual + row( + "SELECT dm, ts FROM s2_t", + """{"dm": "bad", "ts": "2024-01-31"}""", + """, "_ignored": ["dm"]""" + )( + "ts" + ) shouldBe utc(2024, 1, 31) + } + + it should "project strictly: no undeclared key, `_id` and the nested-hit keys kept (strict)" in { + row( + "SELECT id, n FROM s2_t", + """{"id": "1", "extra": 42, "n": 3, "k": "x"}""" + ).keys.toSeq shouldBe + Seq("id", "n") + // a computed-only projection returns only its columns (#418 part 2) + rows( + "SELECT n + 1 AS v FROM s2_t", + """{"_id": "a1", "_source": {"n": 3, "k": "x"}, "fields": {"v": [4]}}""" + ).head.keys.toSeq shouldBe Seq("v") + val reading = HitReading.of(statement("SELECT id FROM s2_t"), DateReading(Some("8.18.3"))) + reading.strict shouldBe true + reading.keep should contain(ElasticConversion.DocumentIdField) + // no strict projection for a statement whose rows core reorders or aggregates + HitReading + .of(statement("SELECT k, COUNT(*) FROM s2_t GROUP BY k"), DateReading(None)) + .strict shouldBe false + } + + it should "read a flat dotted key as its path, and fold it under * (flat)" in { + row("SELECT obj.a FROM s2_t", """{"obj.a": 5}""")("obj.a") shouldBe 5 + val star = row("SELECT * FROM s2_t", """{"id": "1", "obj.a": 5, "extra": 1}""") + star("obj") shouldBe ListMap("a" -> 5) + star.contains("extra") shouldBe false + star.contains("obj.a") shouldBe false + } + + it should "read a STRUCT's declared fields by name, each by its type (struct)" in { + val obj = + row("SELECT obj FROM s2_t", """{"obj": {"zz": 1, "a": "3", "d": "2024-01-31"}}""")("obj") + obj shouldBe ListMap("a" -> 3, "d" -> utc(2024, 1, 31)) + obj.asInstanceOf[ListMap[String, Any]]("a") shouldBe a[java.lang.Integer] + // an undeclared key spelling a selected path is kept, as read + val both = row("SELECT obj, obj.zz FROM s2_t", """{"obj": {"zz": 1, "a": 2}}""") + both("obj.zz") shouldBe 1 + } + + it should "answer a value its typed reader cannot read as stored, never a failure (unreadable)" in { + row("SELECT n FROM s2_t", """{"n": "abc"}""")("n") shouldBe "abc" + row("SELECT g FROM s2_t", """{"g": "not a point"}""")("g") shouldBe "not a point" + row("SELECT c_c1 FROM s2_t", """{"c_c1": "31-01-2024"}""")("c_c1") shouldBe "31-01-2024" + // several points, read one by one (D7's AsRead) + row("SELECT g FROM s2_t", """{"g": ["1.5,2.5", "3.5,4.5"]}""")("g") shouldBe List( + ListMap("lat" -> 1.5, "lon" -> 2.5), + ListMap("lat" -> 3.5, "lon" -> 4.5) + ) + // a number outside its type's range is no value of it -- Elasticsearch refused it + // (`ignore_malformed`) and holds it in `_source` only: as stored, never narrowed (wrapped) + row("SELECT n FROM s2_t", """{"n": 3000000000}""")("n") shouldBe java.lang.Long.valueOf( + 3000000000L + ) + row("SELECT n FROM s2_t", """{"n": "3000000000"}""")("n") shouldBe "3000000000" + row("SELECT n FROM s2_t", """{"n": 2147483647.5}""")("n") shouldBe java.lang.Double.valueOf( + 2147483647.5 + ) + row("SELECT l FROM s2_t", """{"l": 12345678901234567890}""")("l") shouldBe + new java.math.BigInteger("12345678901234567890") + row("SELECT l FROM s2_t", """{"l": "1e30"}""")("l") shouldBe "1e30" + row("SELECT s FROM s2_t", """{"s": 40000}""")("s") shouldBe java.lang.Integer.valueOf(40000) + row("SELECT tb FROM s2_t", """{"tb": "200"}""")("tb") shouldBe "200" + row("SELECT n FROM s2_t", """{"n": "1e1000000000"}""")("n") shouldBe "1e1000000000" + // ...the bounds themselves are values, checked as Elasticsearch checks them: `byte`, `short` + // and `integer` on the value, `long` on the truncated value + row("SELECT n FROM s2_t", """{"n": -2147483648}""")("n") shouldBe + java.lang.Integer.valueOf(Int.MinValue) + row("SELECT tb FROM s2_t", """{"tb": "-128"}""")("tb") shouldBe java.lang.Integer.valueOf(-128) + row("SELECT tb FROM s2_t", """{"tb": "127.9"}""")("tb") shouldBe "127.9" + row("SELECT l FROM s2_t", """{"l": "9223372036854775807.5"}""")("l") shouldBe + java.lang.Long.valueOf(Long.MaxValue) + row("SELECT l FROM s2_t", """{"l": "0.5e-1000000000"}""")("l") shouldBe java.lang.Long.valueOf( + 0L + ) + } + + it should "answer a page the same rows and classes on every parse route (pages)" in { + val fixture = SchemaPageFixture + val outputs = fixture.statement.select.fieldsWithComputedAliases.map(_.outputName) + // each client's page extractor reads its page through one of these two routes, in this context: + // the client modules' page specs assert their extractors against the same rows + Seq("6.8.23", "7.17.29", "8.18.3").foreach { version => + withClue(s"[$version] ") { + implicit val context: ConversionContext = fixture.context(version) + val oneShot = ElasticConversion + .parseResponse(fixture.page, fixture.fieldAliases, ListMap.empty, outputs) + .get + val tree = ElasticConversion + .parseSingleSearchResponse( + app.softnetwork.elastic.sql.serialization.JacksonConfig.objectMapper + .readTree(fixture.page), + fixture.fieldAliases, + ListMap.empty + ) + .get + Seq(oneShot, tree).foreach { parsed => + parsed shouldBe fixture.expected + fixture.classes(parsed) shouldBe fixture.classes(fixture.expected) + } + } + } + } + + it should "read a column qualified by its table alias by its type (qualified)" in { + val r = row("SELECT t.id, t.n FROM s2_t AS t", """{"id": "1", "n": "7", "k": "x"}""") + r.values.toSeq shouldBe Seq("1", 7) + r.values.toSeq(1) shouldBe a[java.lang.Integer] + } + + it should "leave a column the statement types as it comes as read (ANY)" in { + // `extra` is not mapped: ANY, as read today + row("SELECT id, extra FROM s2_t", """{"id": "1", "extra": [1, 2]}""")("extra") shouldBe List( + 1, + 2 + ) + } + + it should "read a NULL as NULL and a never-set keyword as an absent value" in { + val r = row("SELECT id, nul FROM s2_t", """{"id": "1", "ts": null}""") + r("nul") shouldBe (null: Any) + } + + "Elasticsearch's default date format" should "be read by hand exactly as MappedDateFormat reads it (default date)" in { + val corpus = Seq( + "2024-01-31", + "2024-01-31T10:30:00", + "2024-01-31T10:30:00Z", + "2024-01-31T10:30:00.5", + "2024-01-31T10:30:00.55Z", + "2024-01-31T10:30:00.555", + "2024-01-31T10:30:00.5555", + "2024-02-29T23:59:59.999Z", + "2023-02-29", + "2024-02-30T10:00:00", + "2024-13-01", + "2024-01-31T24:00:00", + "2024-01-31T10:30", + "2024-01-31T10:30:00+01:00", + "0001-01-01T00:00:00", + "9999-12-31T23:59:59.999Z", + "2024-1-31", + "1706697000000", + "2024-01-31 10:30:00", + "20240131" + ) + Seq("6.8.23", "7.17.29", "8.18.3").foreach { version => + val dialect = MappedDateFormat.Dialect.of(version).get + val format = MappedDateFormat( + app.softnetwork.elastic.sql.query.TemporalLiterals.DefaultDateFormat, + dialect, + java.time.Clock.systemUTC() + ) + corpus.foreach { text => + Option(HitReading.DefaultDates.read(text)).foreach { fast => + withClue(s"[$version] $text: ") { + format.read(text) shouldBe MappedDateFormat.Read(fast.toInstant) + } + } + } + } + // the shapes the hand reader takes + HitReading.DefaultDates.read("2024-01-31T10:30:00.555Z") shouldBe + ZonedDateTime.of(2024, 1, 31, 10, 30, 0, 555000000, ZoneOffset.UTC) + HitReading.DefaultDates.read("2024-01-31T10:30:00.5555") shouldBe null + HitReading.DefaultDates.read("2024-02-30T10:00:00") shouldBe null + } + + "The strict projection" should "hand an in-order row back as the same instance (identity)" in { + val single = statement("SELECT id, n FROM s2_t") + implicit val context: ConversionContext = contextOf(single) + val project = Projector.normalizer(Seq("id", "n")) + val inOrder = ListMap[String, Any]("id" -> "1", "n" -> 3) + (project(inOrder) eq inOrder) shouldBe true + // ...a trailing `_id` is kept, as the same instance + val withId = ListMap[String, Any]("id" -> "1", "n" -> 3, "_id" -> "a1") + (project(withId) eq withId) shouldBe true + // ...a trailing undeclared key is dropped + project(ListMap[String, Any]("id" -> "1", "n" -> 3, "k" -> "x")) shouldBe + ListMap[String, Any]("id" -> "1", "n" -> 3) + } + + "A set operation's date reader" should "pass a canonical value through (D12)" in { + val dates = DateReading(Some("8.18.3")) + def read(source: app.softnetwork.elastic.sql.`type`.SQLType, format: String) = + SetOperationValues.converter(source, source, "x", Some(format), "c", "c", dates) + read(SQLTypes.Date, "epoch_second")(LocalDate.of(2024, 1, 31)) shouldBe LocalDate.of( + 2024, + 1, + 31 + ) + read(SQLTypes.Timestamp, "yyyy/MM/dd")(utc(2024, 1, 31)) shouldBe utc(2024, 1, 31) + read(SQLTypes.Time, "HH:mm:ss.SSS")(java.time.LocalTime.of(10, 30, 0, 123000000)) shouldBe + java.time.LocalTime.of(10, 30, 0, 123000000) + // the raw forms still read by their format + read(SQLTypes.Date, "epoch_second")(1706697000) shouldBe LocalDate.of(2024, 1, 31) + // ...and so is a temporal the response parser made of a text, which is NOT that one form: an + // offset other than UTC (read by its format: Elasticsearch 8.18.3's week-based year here, + // MEASURED by `MappedDateFormatSpec`'s corpus), a region (never read without guessing: it + // fails, R-11 / R-15) + read(SQLTypes.Timestamp, "strict_date_optional_time||epoch_millis")( + ZonedDateTime.of(2024, 1, 31, 11, 30, 0, 0, java.time.ZoneOffset.ofHours(1)) + ) shouldBe utc(2024, 1, 31, 10, 30) + read(SQLTypes.Timestamp, "YYYY-MM-dd'T'HH:mm:ssXXX")( + ZonedDateTime.parse("2024-01-31T10:30:00+02:00") + ) shouldBe utc(2023, 12, 31, 8, 30) + an[IllegalArgumentException] should be thrownBy read( + SQLTypes.Timestamp, + "yyyy-MM-dd'T'HH:mm:ssXXX'['VV']'" + )(ZonedDateTime.parse("2024-07-31T10:30:00+02:00[Europe/Paris]")) + } + + it should "answer a SMALLINT and a TINYINT as Integers, a binary's base64 as its bytes (D2, Q5)" in { + SetOperationValues.converter(SQLTypes.Int, SQLTypes.SmallInt)(5) shouldBe + java.lang.Integer.valueOf(5) + SetOperationValues.converter(SQLTypes.Int, SQLTypes.TinyInt)(5) shouldBe + java.lang.Integer.valueOf(5) + new String( + SetOperationValues + .converter(SQLTypes.VarBinary, SQLTypes.VarBinary)("aGVsbG8=") + .asInstanceOf[Array[Byte]], + StandardCharsets.US_ASCII + ) shouldBe "hello" + } +} + +/** The protected projector, for the identity check. */ +private object Projector extends ElasticConversion { + def normalizer(fields: Seq[String])(implicit + context: ConversionContext + ): ListMap[String, Any] => ListMap[String, Any] = rowNormalizer(fields) +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/NonFiniteBucketKeySpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/NonFiniteBucketKeySpec.scala index 280e6cd34..1e95e4599 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/NonFiniteBucketKeySpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/NonFiniteBucketKeySpec.scala @@ -60,7 +60,7 @@ class NonFiniteBucketKeySpec extends AnyFlatSpec with Matchers { .map(_("kk")) } - private val ofStatement: SingleSearch => ConversionContext = TextColumns.of(_, NativeContext) + private val ofStatement: SingleSearch => ConversionContext = SchemaColumns.of(_, NativeContext) "a double key level" should "answer its non-finite keys as Doubles" in { statement("SELECT EXP(n) AS kk FROM t GROUP BY EXP(n)").doubleKeyLevels shouldBe Set("kk") diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala new file mode 100644 index 000000000..b760b4fd3 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/ResultSchemaSpec.scala @@ -0,0 +1,212 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.SetOperationValues.DateReading +import app.softnetwork.elastic.client.result.{ + CsvFormatter, + JsonFormatter, + QueryRows, + QueryStructured, + ResultRenderer +} +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes, ValueCoercion} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{FromlessSelect, MultiSearch, SingleSearch} +import app.softnetwork.elastic.sql.schema.Schema +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.collection.immutable.ListMap +import scala.concurrent.duration.Duration + +/** SCHEMA-2 (D1, D12, A-13, A-14, A-16) -- a statement's result schema, declared before any row + * from the types its items have as set-operation positions, and the presenters that read it. + */ +class ResultSchemaSpec extends AnyFlatSpec with Matchers { + + private val mapping: String = + """{"s2_t": {"aliases": {}, "mappings": { + | "_meta": {"columns": {"c_des": {"data_type": "DATE"}}}, + | "properties": { + | "id": {"type": "keyword"}, + | "ts": {"type": "date"}, + | "c_des": {"type": "date", "format": "epoch_second"}, + | "c_c1": {"type": "date", "format": "yyyy/MM/dd"}, + | "tsz": {"type": "date", "format": "yyyy-MM-dd HH:mm z"}, + | "n": {"type": "integer"}, "x": {"type": "double"}, "r": {"type": "float"}, + | "b": {"type": "boolean"}, "g": {"type": "geo_point"}, "bin": {"type": "binary"}, + | "nul": {"type": "keyword"}, "txt": {"type": "text"}, + | "obj": {"properties": {"z": {"type": "keyword"}, "a": {"type": "integer"}}}, + | "items": {"type": "nested", "properties": {"p": {"type": "integer"}}} + | }}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + + private val schema: Schema = Index("s2_t", mapping).schema + + private val es8 = DateReading(Some("8.18.3")) + + private def statement(sql: String): SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(schema)) + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + private def schemaOf(sql: String): ResultSchema = + ResultSchemas.of(statement(sql), es8).getOrElse(fail(s"[$sql] has no schema")) + + private def types(sql: String): Seq[(String, SQLType)] = + schemaOf(sql).columns.map(c => c.name -> c.sqlType) + + "A statement's result schema" should "type each column as its set-operation position (P1)" in { + types("SELECT id, ts, c_des, c_c1, n, x, r, b, bin, nul, txt FROM s2_t") shouldBe Seq( + "id" -> SQLTypes.Varchar, + "ts" -> SQLTypes.Timestamp, + "c_des" -> SQLTypes.Date, + "c_c1" -> SQLTypes.Timestamp, + "n" -> SQLTypes.Int, + "x" -> SQLTypes.Double, + "r" -> SQLTypes.Real, + "b" -> SQLTypes.Boolean, + "bin" -> SQLTypes.VarBinary, + "nul" -> SQLTypes.Varchar, + "txt" -> SQLTypes.Varchar + ) + // an alias names the column, the type stays the source's + types("SELECT n AS k, CAST(n AS VARCHAR) AS s FROM s2_t") shouldBe Seq( + "k" -> SQLTypes.Int, + "s" -> SQLTypes.Varchar + ) + // nullable is always true (A-16) + schemaOf("SELECT id, n FROM s2_t").columns.map(_.nullable).distinct shouldBe Seq(true) + } + + it should "answer a date its format cannot read for sure VARCHAR (AsStored)" in { + types("SELECT tsz FROM s2_t") shouldBe Seq("tsz" -> SQLTypes.Varchar) + // ...and every date VARCHAR when the Elasticsearch version is not known + ResultSchemas + .of(statement("SELECT ts FROM s2_t"), DateReading(None)) + .get + .columns + .map( + _.sqlType + ) shouldBe Seq(SQLTypes.Varchar) + } + + it should "give a STRUCT its fields by name, a point {lat, lon}, a list its element (A-13)" in { + val obj = schemaOf("SELECT obj, g, items FROM s2_t").columns + obj.head.sqlType shouldBe SQLTypes.Struct + obj.head.fields.map(f => f.name -> f.sqlType) shouldBe Seq( + "a" -> SQLTypes.Int, + "z" -> SQLTypes.Varchar + ) + obj(1).sqlType shouldBe SQLTypes.GeoPoint + obj(1).fields.map(f => f.name -> f.sqlType) shouldBe Seq( + "lat" -> SQLTypes.Double, + "lon" -> SQLTypes.Double + ) + obj(2).sqlType shouldBe SQLTypes.Array(SQLTypes.Struct) + obj(2).element.map(_.fields.map(_.name)) shouldBe Some(Seq("p")) + } + + it should "expand * to the mapping's columns in their table order (A-14)" in { + val star = schemaOf("SELECT * FROM s2_t") + star.names shouldBe schema.columns.map(_.name) + } + + it should "be None without a mapping, Some for a FROM-less statement" in { + val bare = Parser("SELECT id FROM s2_t") match { + case Right(s: SingleSearch) => s + case other => fail(s"$other") + } + ResultSchemas.of(bare, es8) shouldBe None + val fromless = Parser("SELECT 1 AS one, 'a' AS s") match { + case Right(f: FromlessSelect) => f + case other => fail(s"$other") + } + ResultSchemas.of(fromless).columns.map(c => c.name -> c.sqlType) shouldBe Seq( + "one" -> SQLTypes.Int, + "s" -> SQLTypes.Varchar + ) + } + + it should "type a computed column by its declared type (T0-5)" in { + // recorded, not asserted beyond being declared: NUMERIC / TEMPORAL cells are value-typed + val computed = types("SELECT n + 1 AS v, DATE_FORMAT(ts, '%Y') AS y FROM s2_t") + computed.map(_._1) shouldBe Seq("v", "y") + computed(1)._2 shouldBe SQLTypes.Varchar + computed.foreach(c => info(s"${c._1}: ${c._2.typeId}")) + } + + private def union(sql: String): MultiSearch = Parser(sql) match { + case Right(m: MultiSearch) => m.copy(requests = m.requests.map(_.update(Some(schema)))) + case other => fail(s"[$sql] expected a MultiSearch, got $other") + } + + "A UNION ALL's schema" should "be typed as its converters type it, named by its first branch (D12)" in { + ResultSchemas + .of(union("SELECT n AS k FROM s2_t UNION ALL SELECT x AS k FROM s2_t"), es8) + .map( + _.columns.map(c => c.name -> c.sqlType) + ) shouldBe Some(Seq("k" -> SQLTypes.Double)) + // a date one branch reads as stored: the column is VARCHAR, in the schema and the converters + val stored = union("SELECT tsz AS k FROM s2_t UNION ALL SELECT ts AS k FROM s2_t") + ResultSchemas.of(stored, es8).map(_.columns.map(_.sqlType)) shouldBe Some(Seq(SQLTypes.Varchar)) + val converters = SetOperationValues.converters(stored, Some("8.18.3")) + converters.forall(_.isDefined) shouldBe true + converters(1).get.apply(0)(java.time.ZonedDateTime.parse("2024-01-31T10:30:00Z")) shouldBe + "2024-01-31 10:30:00" + converters(0).get.apply(0)("2024-01-31 10:30 CET") shouldBe "2024-01-31 10:30 CET" + // an opaque first branch declares nothing + ResultSchemas.of(union("SELECT * FROM s2_t UNION ALL SELECT id FROM s2_t"), es8) shouldBe None + } + + "The presenters" should "take the header from the schema, the rows keeping their key order (A-14)" in { + val s = ResultSchema(Seq(ResultColumn("a", SQLTypes.Int), ResultColumn("b", SQLTypes.Varchar))) + val rows = Seq(ListMap[String, Any]("b" -> "x", "a" -> 1)) + CsvFormatter.format(QueryRows(rows, schema = Some(s))).split("\n").head shouldBe "a,b" + val table = ResultRenderer.renderAscii(QueryRows(rows, schema = Some(s)), Duration.Zero) + val header = table.split("\n")(1) + header.indexOf("a") should be < header.indexOf("b") + // no schema: the first row's keys, as before + CsvFormatter.format(QueryRows(rows)).split("\n").head shouldBe "b,a" + // a QueryStructured carries its response's schema (A-18) + QueryStructured( + ElasticResponse(None, "{}", rows, ListMap.empty, ListMap.empty, Some(s)) + ).schema shouldBe + Some(s) + } + + it should "print a VARBINARY as its base64 and a REAL as a number, as 0.24.0 did (D2)" in { + val bytes = "hello".getBytes("UTF-8") + ValueCoercion.coerceToString(bytes) shouldBe "aGVsbG8=" + CsvFormatter.format(QueryRows(Seq(ListMap[String, Any]("bin" -> bytes)))) should include( + "aGVsbG8=" + ) + ResultRenderer.renderAscii( + QueryRows(Seq(ListMap[String, Any]("bin" -> bytes))), + Duration.Zero + ) should include("aGVsbG8=") + val json = JsonFormatter + .format( + QueryRows(Seq(ListMap[String, Any]("r" -> java.lang.Float.valueOf(1.5f), "bin" -> bytes))), + Duration.Zero + ) + .replaceAll("\\s", "") + json should include("\"r\":1.5") + json should include("\"bin\":\"aGVsbG8=\"") + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SchemaMissInvalidationSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SchemaMissInvalidationSpec.scala index 7a8c3e256..1f09c2584 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SchemaMissInvalidationSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SchemaMissInvalidationSpec.scala @@ -323,4 +323,34 @@ class SchemaMissInvalidationSpec extends AnyFlatSpec with Matchers with BeforeAn } } } + + "The path reload memo (SCHEMA-2, D10)" should "reload a lacking path ONCE per TTL, and stay bounded" in { + val client = new RecordingClient + client.indices = Set("t") + def select(column: String): Unit = { + val _ = client.search(SelectStatement(s"SELECT id, $column FROM t")) + } + select("id") + val loaded = client.lookups.size + select("zz_0") + client.lookups.size shouldBe loaded + 1 // one reload for the path the mapping lacks... + select("zz_0") + client.lookups.size shouldBe loaded + 1 // ...and none within the TTL + // live entries are kept above the purge threshold + (1 to 300).foreach(i => select(s"zz_$i")) + client.schemaPathReloadCount shouldBe 301 + // an insert above the threshold drops the expired entries (each by its source's TTL) + client.cacheTtlMs = 1L + select("zz_a") // re-stamps the source's entry with the short TTL + Thread.sleep(5) + select("zz_b") + client.schemaPathReloadCount should be <= 2 + // thousands of distinct live names inside one TTL never grow it past four times the threshold + client.cacheTtlMs = 5 * 60 * 1000L + (1 to 1100).foreach(i => select(s"yy_$i")) + client.schemaPathReloadCount should be <= 1024 + val before = client.lookups.size + select("yy_1100") + client.lookups.size shouldBe before // the newest path is still remembered + } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SchemaPageFixture.scala b/core/src/test/scala/app/softnetwork/elastic/client/SchemaPageFixture.scala new file mode 100644 index 000000000..5e6ae76ff --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/SchemaPageFixture.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.SetOperationValues.DateReading +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.SingleSearch + +import java.time.{LocalDate, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap + +/** SCHEMA-2 (AC2, `HitReadingSpec` "pages") -- one canned page of hits, read in its statement's + * schema context: what every client's page extractor (scroll, `search_after`) answers, value AND + * class, exactly as core's one-shot parse does. Network-free; shared by the client modules' page + * specs (their tests depend on core's). + */ +private[client] object SchemaPageFixture { + + private val mapping: String = + """{"s2_p": {"aliases": {}, "mappings": { + | "_meta": {"columns": {"c_des": {"data_type": "DATE"}}}, + | "properties": { + | "id": {"type": "keyword"}, + | "ts": {"type": "date"}, + | "c_des": {"type": "date", "format": "epoch_second"}, + | "dm": {"type": "date", "ignore_malformed": true}, + | "n": {"type": "integer"}, "l": {"type": "long"}, "b": {"type": "boolean"}, + | "k": {"type": "keyword"} + | }}, "settings": {"index": {"number_of_shards": "1"}}}}""".stripMargin + + val sql: String = "SELECT id, ts, c_des, dm, n, l, b, k FROM s2_p" + + val statement: SingleSearch = Parser(sql) match { + case Right(s: SingleSearch) => s.update(Some(Index("s2_p", mapping).schema)) + case other => throw new IllegalStateException(s"[$sql] parsed as $other") + } + + def fieldAliases: ListMap[String, String] = statement.fieldAliases + + /** The statement's schema context, as `SearchApi` builds it for `version`. */ + def context(version: String): ConversionContext = { + val elements = ElementConversions.of(statement, Some(version)) + SchemaColumns( + HitReading.of(statement, DateReading(Some(version)), HitReading.Default, elements.keySet), + statement.doubleKeyLevels + )(elements) + } + + /** A scroll page in Elasticsearch's own shape: `_ignored` on the first hit, every value in a + * spelling a typed reader converts (text numbers, a one-element array, a keyword number, an INT + * outside its range kept as stored). + */ + val page: String = + """{"_scroll_id": "c1", "took": 1, "timed_out": false, + |"_shards": {"total": 1, "successful": 1, "skipped": 0, "failed": 0}, + |"hits": {"total": {"value": 2, "relation": "eq"}, "max_score": 1.0, "hits": [ + | {"_index": "s2_p", "_id": "1", "_score": 1.0, "_ignored": ["dm"], + | "_source": {"id": "1", "ts": "2024-01-31T10:30:00", "c_des": 1706697000, "dm": "bad", + | "n": "5", "l": 7, "b": "true", "k": 10}}, + | {"_index": "s2_p", "_id": "2", "_score": 1.0, + | "_source": {"id": "2", "ts": "2024-02-01", "c_des": [1706745600], "dm": "2024-02-01", + | "n": 3000000000, "l": "8", "b": false, "k": "x"}} + |]}}""".stripMargin + + private def utc(y: Int, mo: Int, d: Int, h: Int = 0, mi: Int = 0): ZonedDateTime = + ZonedDateTime.of(y, mo, d, h, mi, 0, 0, ZoneOffset.UTC) + + /** The rows the page answers, in D2's classes. */ + val expected: Seq[ListMap[String, Any]] = Seq( + ListMap[String, Any]( + "id" -> "1", + "ts" -> utc(2024, 1, 31, 10, 30), + "c_des" -> LocalDate.of(2024, 1, 31), + "dm" -> null, + "n" -> java.lang.Integer.valueOf(5), + "l" -> java.lang.Long.valueOf(7L), + "b" -> java.lang.Boolean.TRUE, + "k" -> "10" + ), + ListMap[String, Any]( + "id" -> "2", + "ts" -> utc(2024, 2, 1), + "c_des" -> LocalDate.of(2024, 2, 1), + "dm" -> utc(2024, 2, 1), + "n" -> java.lang.Long.valueOf(3000000000L), + "l" -> java.lang.Long.valueOf(8L), + "b" -> java.lang.Boolean.FALSE, + "k" -> "x" + ) + ) + + /** Each row's column -> class name: value equality alone lets an `Integer` pass for a `Long`. */ + def classes(rows: Seq[ListMap[String, Any]]): Seq[Map[String, String]] = + rows.map(_.map { case (k, v) => k -> (if (v == null) "null" else v.getClass.getName) }) +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ScrollSettingsSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ScrollSettingsSpec.scala index 5e414b333..8d79cfd0e 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/ScrollSettingsSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/ScrollSettingsSpec.scala @@ -134,6 +134,24 @@ class ScrollSettingsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAl val scrolledConfig = new AtomicReference[ScrollConfig]() val scrolledStatement = new AtomicReference[SearchStatement]() + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala index e6236558a..4a4ea65cf 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegReadersSpec.scala @@ -205,8 +205,9 @@ class SetOperationLegReadersSpec extends AnyFlatSpec with Matchers { val want: Seq[Any] = Seq( Integer.valueOf(5), java.lang.Long.valueOf(5L), - java.lang.Short.valueOf(5.toShort), - java.lang.Byte.valueOf(5.toByte), + // SCHEMA-2 (D2): a SMALLINT and a TINYINT are Integers (JDBC Table B-3) + Integer.valueOf(5), + Integer.valueOf(5), java.lang.Double.valueOf(5.0), java.lang.Float.valueOf(0.5f), java.lang.Boolean.TRUE, @@ -214,7 +215,8 @@ class SetOperationLegReadersSpec extends AnyFlatSpec with Matchers { utc("2024-01-31T10:30:00Z"), LocalTime.parse("10:30:00"), "a", - "YWI=".getBytes(java.nio.charset.StandardCharsets.UTF_8), + // SCHEMA-2 (Q5): a binary's base64 is the bytes it encodes + "ab".getBytes(java.nio.charset.StandardCharsets.UTF_8), // the branch's fields, in its order ListMap( "e" -> utc("2024-01-31T10:30:00Z"), diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala index fa601305f..c854bf2e1 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationLegSchemasSpec.scala @@ -279,24 +279,36 @@ class SetOperationLegSchemasSpec extends AnyFlatSpec with Matchers { agrees = Some(!column.converts) ) (DateRoute.of(SQLTypes.Timestamp, Some(spec), dates), legSlot) match { - case (_, None) => Seq(s"[$f] no slot") + case (_, None) => Seq(s"[$f] no slot") case (DateRoute.Typed, Some(s)) => + // SCHEMA-2 (D4, D12) -- a leg's row holds the value core's hit reader gives the column: + // the date its format reads (canonical), or, where it reads none, the value as stored + val canonical = new HitReading.Readers(ListMap.empty, dates, MultiValued.AsRead, "t") + .column("v", declared, Set.empty) + ._1 (if (s.sqlType != declared || s.asStored.nonEmpty) Seq(s"[$f] typed as $s") else Nil) ++ - formatRows.flatMap { case (_, json, millis, cause) => - val value = received(json) - (cause, Try(s.read(value)), Try(reference(value))) match { - case (None, Success(z: ZonedDateTime), Success(r)) - if shown(z) == shown(r) && z.toInstant.toEpochMilli == millis => - typedRead += 1 - None - case (Some(c), Failure(e: IllegalArgumentException), Failure(r)) - if valueCauses(c) && e.getMessage == r.getMessage && - e.getMessage.contains("field v") && e.getMessage.contains(s"'$spec'") => - typedFailed += 1 - None - case (c, got, want) => Some(s"[$f] $json ($c): read $got, legReaders $want") - } + formatRows.flatMap { case (_, json, millis, cause) => + val value = canonical(JacksonConfig.objectMapper.readTree(json)) + (cause, Try(s.read(value)), Try(reference(value))) match { + case (None, Success(z: ZonedDateTime), Success(r)) + if shown(z) == shown(r) && z.toInstant.toEpochMilli == millis => + typedRead += 1 + None + // SCHEMA-2 (D4): a value whose spelling or offset the response parser erased is + // read from its text now, before that parser: the instant Elasticsearch indexed + case (Some(c), Success(z: ZonedDateTime), Success(r)) + if c.startsWith("erased-") && shown(z) == shown(r) && + z.toInstant.toEpochMilli == millis => + typedRead += 1 + None + case (Some(c), Failure(e: IllegalArgumentException), Failure(r)) + if valueCauses(c) && e.getMessage == r.getMessage && + e.getMessage.contains("field v") && e.getMessage.contains(s"'$spec'") => + typedFailed += 1 + None + case (c, got, want) => Some(s"[$f] $json ($c): read $got, legReaders $want") } + } case (DateRoute.AsStored(reason), Some(s)) => (if (s.sqlType != varchar || s.asStored != Seq(LegSlot.AsStored("v", reason))) Seq(s"[$f] as stored, typed as $s") diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationMappedDatesSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationMappedDatesSpec.scala index 3e68ac391..5d73c929e 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationMappedDatesSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationMappedDatesSpec.scala @@ -207,26 +207,24 @@ class SetOperationMappedDatesSpec extends AnyFlatSpec with Matchers { plan(union("itemsd", "itemsd")) shouldBe Seq(None, None) } - it should "keep a date it cannot read as stored, where a converting column fails it" in { - // a region id: its offset comes from zone data -- the value as stored, never a failure + it should "answer a column of a format it cannot read for sure as VARCHAR, every value as stored" in { + // SCHEMA-2 (D12): a format core cannot read every value of for sure (a region id, whose + // offset comes from zone data) makes its column VARCHAR, as DuckDB types a text beside a + // date -- every value as stored, never a failure: its own text, the other branch's date in + // DuckDB's spelling answered(union("c_z", "ts"), 0, "\"2024-01-31 Europe/Paris\"") shouldBe "2024-01-31 Europe/Paris" answered(union("c_z", "ts"), 0, "[\"2024-01-31 Europe/Paris\"]") shouldBe - List("2024-01-31 Europe/Paris") - // the readable values of the column are still read - shown(answered(union("c_z", "ts"), 0, "\"2024-01-31 Z\"")) shouldBe - shown(utc("2024-01-31T00:00:00Z")) - shown(answered(union("c_z", "ts"), 1, "\"2024\"")) shouldBe - shown(utc("2024-01-01T00:00:00Z")) + "2024-01-31 Europe/Paris" + answered(union("c_z", "ts"), 0, "\"2024-01-31 Z\"") shouldBe "2024-01-31 Z" + answered(union("c_z", "ts"), 1, "\"2024\"") shouldBe "2024-01-01 00:00:00" // the Elasticsearch version not known: every date of a format as stored answered(union("c_es", "ts"), 0, "1706697000", version = None) shouldBe 1706697000 - // a column that converts (a TIMESTAMP beside a DATE) fails it, as before - intercept[IllegalArgumentException]( - answered(union("c_z", "d"), 0, "\"2024-01-31 Europe/Paris\"") - ).getMessage should include( - "Could not read '2024-01-31 Europe/Paris' as a TIMESTAMP when casting from source column k: " + - "core cannot read it with the date format 'yyyy-MM-dd VV' of field c_z as Elasticsearch 8 does" - ) + // ...beside a DATE too, which it failed before: the column is VARCHAR, so nothing converts + // to a TIMESTAMP + answered(union("c_z", "d"), 0, "\"2024-01-31 Europe/Paris\"") shouldBe + "2024-01-31 Europe/Paris" + answered(union("c_z", "d"), 1, "\"2024-01-31\"") shouldBe "2024-01-31" } "An agreeing date field of a mapping format of its own" should "be read at every depth, every other entry as stored" in { @@ -270,12 +268,20 @@ class SetOperationMappedDatesSpec extends AnyFlatSpec with Matchers { it should "ask the Elasticsearch version once, and only for a column it reads" in { var asked = 0 def version: Option[String] = { asked += 1; Some("8.18.3") } - SetOperationValues.converters(resolved(union("c_def", "ts")), version) + // a statement with no date column asks nothing + SetOperationValues.converters( + resolved("SELECT n, s FROM t UNION ALL SELECT n, s FROM t"), + version + ) asked shouldBe 0 + // SCHEMA-2 (D5, D12): a date column of a format -- the default one too -- is typed by it, as + // its legs' hit readers read it: the version once per statement + SetOperationValues.converters(resolved(union("c_def", "ts")), version) + asked shouldBe 1 SetOperationValues.converters( resolved("SELECT c_es, c_c1 FROM t UNION ALL SELECT ts, ts FROM t"), version ) - asked shouldBe 1 + asked shouldBe 2 } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationValuesSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationValuesSpec.scala index 614f7bbb9..209d19715 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationValuesSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationValuesSpec.scala @@ -342,7 +342,8 @@ class SetOperationValuesSpec extends AnyFlatSpec with Matchers { (SQLTypes.Timestamp, SQLTypes.Timestamp, Instant.parse("2024-01-31T10:30:00Z"), zoned), (SQLTypes.Timestamp, SQLTypes.Timestamp, java.lang.Long.valueOf(1706697000000L), zoned), (SQLTypes.Date, SQLTypes.Timestamp, LocalDate.parse("2024-01-31"), midnight), - (SQLTypes.VarBinary, SQLTypes.VarBinary, "YWI=", utf8("YWI=")), + // SCHEMA-2 (Q5): a `binary` field's base64 is the bytes it encodes + (SQLTypes.VarBinary, SQLTypes.VarBinary, "YWI=", utf8("ab")), (SQLTypes.VarBinary, SQLTypes.VarBinary, utf8("ab"), utf8("ab")), (SQLTypes.Varchar, SQLTypes.VarBinary, "x", utf8("x")), (SQLTypes.BigInt, SQLTypes.Varchar, Integer.valueOf(1), "1"), @@ -612,10 +613,12 @@ class SetOperationValuesSpec extends AnyFlatSpec with Matchers { private def bytesOf(v: Any): Seq[Int] = v.asInstanceOf[Array[Byte]].toSeq.map(_ & 0xff) - "A VARBINARY value reaching a VARBINARY column" should "be the bytes of the text Elasticsearch holds" in { + "A VARBINARY value reaching a VARBINARY column" should "be the bytes the text Elasticsearch holds encodes" in { val f = converter(SQLTypes.VarBinary, SQLTypes.VarBinary) - // a `binary` field's base64, read as its UTF-8 bytes -- what arrow and the JDBC driver read - bytesOf(f("YWI=")) shouldBe utf8("YWI=").toSeq.map(_ & 0xff) + // SCHEMA-2 (Q5): a `binary` field's base64, DECODED -- the bytes core's hit reader gives it + bytesOf(f("YWI=")) shouldBe utf8("ab").toSeq.map(_ & 0xff) + // a text that is not base64: its UTF-8 bytes, as before + bytesOf(f("not base64!")) shouldBe utf8("not base64!").toSeq.map(_ & 0xff) kind(f("YWI=")) shouldBe classOf[Array[Byte]] val raw = Array[Byte](1, 2, -1) f(raw) shouldBe raw @@ -830,7 +833,7 @@ class SetOperationValuesSpec extends AnyFlatSpec with Matchers { // a DOUBLE beside an INT: a Double in both fieldA(union("ox", "oi"), 0, Integer.valueOf(2)) shouldBe shown(java.lang.Double.valueOf(2.0)) // a VARBINARY beside a VARCHAR: bytes in both - fieldA(union("obin", "ov"), 0, "YWI=") shouldBe shown(utf8("YWI=")) + fieldA(union("obin", "ov"), 0, "YWI=") shouldBe shown(utf8("ab")) fieldA(union("obin", "ov"), 1, "x") shouldBe shown(utf8("x")) // a TIMESTAMP beside a DATE: a ZonedDateTime in both, whatever form the timestamp was read in val midnight = ZonedDateTime.parse("2024-01-31T00:00:00Z") diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ShowTablesTableTypeSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ShowTablesTableTypeSpec.scala index b8732fded..de61ca012 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/ShowTablesTableTypeSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/ShowTablesTableTypeSpec.scala @@ -67,8 +67,8 @@ class ShowTablesTableTypeSpec private def showTables(mappings: Map[String, String]): Seq[ListMap[String, Any]] = new StubMappings(mappings).run("SHOW TABLES").futureValue match { - case ElasticSuccess(QueryRows(rows, _)) => rows - case other => fail(s"expected QueryRows, got $other") + case ElasticSuccess(QueryRows(rows, _, _)) => rows + case other => fail(s"expected QueryRows, got $other") } /** Derived from the SAME sealed-hierarchy walk `TableTypeVocabularySpec` uses (reachable here diff --git a/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala index f42e7a2d1..f8d22b661 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala @@ -124,23 +124,27 @@ class TemporalLiteralSearchSpec extends AnyFlatSpec with Matchers with BeforeAnd @volatile var scrolledStatements: Seq[app.softnetwork.elastic.sql.query.SearchStatement] = Seq.empty - override def scroll( + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( statement: app.softnetwork.elastic.sql.query.SearchStatement, config: app.softnetwork.elastic.client.scroll.ScrollConfig )(implicit system: akka.actor.ActorSystem, - context: ConversionContext - ): akka.stream.scaladsl.Source[ - ( - scala.collection.immutable.ListMap[String, Any], - app.softnetwork.elastic.client.scroll.ScrollMetrics - ), - akka.NotUsed - ] = { + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = { scrolledStatements = scrolledStatements :+ statement // DELEGATE: the seam's own rejections (an unparseable literal throws before any stream // exists) are asserted by a neighbouring row, and swallowing them here would disarm it. - super.scroll(statement, config) + super.scrollWithSchema(statement, config) } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/UnionAllRoutingSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/UnionAllRoutingSpec.scala index 087d97409..e256b6faa 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/UnionAllRoutingSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/UnionAllRoutingSpec.scala @@ -163,6 +163,24 @@ class UnionAllRoutingSpec extends AnyFlatSpec with Matchers with BeforeAndAfterA legHitsResponse(elasticQueries) } + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll(statement: SearchStatement, config: ScrollConfig)(implicit system: ActorSystem, context: ConversionContext diff --git a/core/src/test/scala/app/softnetwork/elastic/client/UnnestElementConversionSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/UnnestElementConversionSpec.scala index 4bbdae2dd..c4da87fa7 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/UnnestElementConversionSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/UnnestElementConversionSpec.scala @@ -66,7 +66,7 @@ class UnnestElementConversionSpec extends AnyFlatSpec with Matchers { | {"_source": {"sku": null, "qty": null, "x": null, "at": null}}]}}}} |]}}""".stripMargin - /** The rows of `sql` read from [[response]] as core reads a statement's rows (`TextColumns.of`, + /** The rows of `sql` read from [[response]] as core reads a statement's rows (`SchemaColumns.of`, * the Elasticsearch 8 date dialect), or the failure. */ private def rows(sql: String): Try[Seq[ListMap[String, Any]]] = { @@ -75,7 +75,7 @@ class UnnestElementConversionSpec extends AnyFlatSpec with Matchers { case Left(reason) => Failure(new IllegalArgumentException(reason)) case Right(_) => implicit val context: ConversionContext = - TextColumns.of(single, NativeContext, Some("8.18.3")) + SchemaColumns.of(single, NativeContext, Some("8.18.3")) ElasticConversion.parseResponse( response, single.fieldAliases, diff --git a/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala index 3cf132448..717c02ea3 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtensionSpec.scala @@ -91,6 +91,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { val scrolledConfig = new AtomicReference[ScrollConfig]() val searchedStatement = new AtomicReference[SearchStatement]() + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() @@ -136,6 +154,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { @volatile var scrolls: Seq[(SearchStatement, ScrollConfig)] = Seq.empty @volatile var executed: Seq[SearchStatement] = Seq.empty + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() @@ -224,6 +260,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { rowsPerLeg.applyOrElse(i, (_: Int) => 0) } + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() @@ -267,6 +321,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { private class UnexpectedVariantClient extends NopeClientApi { override protected def logger: Logger = testLogger + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() @@ -323,6 +395,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { @volatile var scrolled: Seq[SearchStatement] = Seq.empty + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() @@ -357,6 +447,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { else super.resolveWithSchema(multiple) } + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() @@ -387,6 +495,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { @volatile private var legs = 0 private def nextLeg(): Int = synchronized { legs += 1; legs } + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() @@ -1603,6 +1729,24 @@ class CoreDqlExtensionSpec extends AnyFlatSpec with Matchers { case None => super.resolveWithSchema(multiple) } + /** SCHEMA-2 (D13) -- the production routes open a stream through `scrollWithSchema`. */ + override private[client] def scrollWithSchema( + statement: app.softnetwork.elastic.sql.query.SearchStatement, + config: app.softnetwork.elastic.client.scroll.ScrollConfig + )(implicit + system: akka.actor.ActorSystem, + context: app.softnetwork.elastic.client.ConversionContext + ): ( + Option[app.softnetwork.elastic.client.ResultSchema], + akka.stream.scaladsl.Source[ + ( + scala.collection.immutable.ListMap[String, Any], + app.softnetwork.elastic.client.scroll.ScrollMetrics + ), + akka.NotUsed + ] + ) = None -> scroll(statement, config) + override def scroll( statement: SearchStatement, config: ScrollConfig = ScrollConfig() diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md index 20d6bcdbd..ecefa20d0 100644 --- a/documentation/client/gateway.md +++ b/documentation/client/gateway.md @@ -174,6 +174,71 @@ Raw Elasticsearch response: QueryStructured(ElasticResponse) ``` +#### The result schema + +Every SELECT result carries its **declared schema**, known before any row is read: + +```scala +case class ResultColumn( + name: String, // the column's output name + sqlType: SQLType, // VARCHAR, INT, BIGINT, DOUBLE, REAL, BOOLEAN, DATE, TIMESTAMP, ... + nullable: Boolean = true, // always true in this release + fields: Seq[ResultColumn] = Nil, // a STRUCT's fields (by name), a GEO_POINT's lat / lon + element: Option[ResultColumn] = None // an ARRAY's element +) +case class ResultSchema(columns: Seq[ResultColumn]) + +QueryRows(rows, truncation, schema: Option[ResultSchema]) +QueryStream(stream, truncation, schema: Option[ResultSchema]) +QueryStructured(response, truncation).schema // = response.schema +``` + +- The columns are in SELECT order; `SELECT *` lists the mapping's columns in their table order. + A row keeps its document's key order: read a value **by name**, present it in schema order. +- `schema` is `None` when no mapping is known (an index that cannot be read, an alias on the ES 8/9 + clients) and for DDL / DML / SHOW results. +- A positional pattern binds every field: `case QueryRows(rows, _, _)`. The shape immune to a future + field is `case q: QueryRows => q.rows`. + +`describe(sql)` answers the schema `run(sql)` would carry **without running the query** (no search, +no subquery; a mapping may be read): + +```scala +gateway.describe("SELECT id, ts FROM events") // ElasticSuccess(Some(ResultSchema(...))) +``` + +It accepts one query (a SELECT, a set operation, a FROM-less SELECT) and refuses anything else with +a `400`; a statement `run` refuses before executing anything is refused with the same status and +message. A statement handled by another extension than core's (a cross-index JOIN under the arrow +extension) answers `None`. + +#### Value classes + +Each value is read by its column's declared type, where it is first read: + +| Column type | Value class | +| --- | --- | +| TINYINT, SMALLINT, INT | `java.lang.Integer` | +| BIGINT | `java.lang.Long` | +| DOUBLE | `java.lang.Double` | +| REAL | `java.lang.Float` | +| BOOLEAN | `java.lang.Boolean` | +| VARCHAR (keyword, text) | `String` -- a number stored in a keyword stays its text | +| DATE | `java.time.LocalDate` | +| TIMESTAMP | `java.time.ZonedDateTime` at UTC, read with the field's mapping format | +| TIME | `java.time.LocalTime` | +| VARBINARY | `Array[Byte]`, the decoded bytes (its text is still the base64) | +| STRUCT | `ListMap` of its declared fields | +| GEO_POINT | `ListMap("lat" -> Double, "lon" -> Double)`, whatever spelling was indexed | +| ARRAY | `List` of its elements | + +A one-element array is its element, an empty array NULL. A date field whose format core cannot read +for sure (a zone name, a region id) is a VARCHAR, its text as stored; a date value Elasticsearch +ignored (`ignore_malformed`) is NULL, and the other values of the same array keep theirs. A value a +typed column cannot read is answered as stored: a number outside its type's range, which +Elasticsearch did not index, is never narrowed. Aggregation values and window-enriched statements +keep their reading in this release. + --- ### DML Results diff --git a/documentation/client/search.md b/documentation/client/search.md index 2cd085254..f729d827a 100644 --- a/documentation/client/search.md +++ b/documentation/client/search.md @@ -129,10 +129,12 @@ case class ClientAggregation( } case class ElasticResponse( - query: String, // Original query - response: String, // Raw JSON response - fieldAliases: Map[String, String], // Field name mappings - aggregations: Map[String, ClientAggregation] // Aggregation definitions + sql: Option[String], // Original SQL, when any + query: String, // The Elasticsearch query + results: Seq[ListMap[String, Any]], // The rows + fieldAliases: ListMap[String, String], // Field name mappings + aggregations: ListMap[String, ClientAggregation], // Aggregation definitions + schema: Option[ResultSchema] = None // The declared result schema (see gateway.md) ) ``` diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSchemaReadingSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSchemaReadingSpec.scala new file mode 100644 index 000000000..2f8d3b59a --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSchemaReadingSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientSchemaReadingSpec extends SchemaReadingSpec diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestScrollPageFailureSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestScrollPageFailureSpec.scala index 02021662b..5e547b909 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestScrollPageFailureSpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestScrollPageFailureSpec.scala @@ -107,6 +107,28 @@ class JestScrollPageFailureSpec extends AnyWordSpec with Matchers { } } + "both extractors" should { + + "read a page in its statement's schema context: the rows and classes core's parse gives (SCHEMA-2, pages)" in { + val fixture = SchemaPageFixture + val context = fixture.context("6.8.23") + val scroll = client.extractAllResults( + fixture.page, + fixture.fieldAliases, + noAggregations, + retainDocumentId = false + )(context) + val searchAfter = + client.extractHitsOnly(fixture.page, fixture.fieldAliases, retainDocumentId = false)( + context + ) + Seq(scroll, searchAfter).foreach { rows => + rows shouldBe fixture.expected + fixture.classes(rows) shouldBe fixture.classes(fixture.expected) + } + } + } + "extractHitsOnly" should { "return one row per raw hit on a good page" in { diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala new file mode 100644 index 000000000..332457a4d --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientSchemaReadingSpec extends SchemaReadingSpec diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala index 6c00ceeed..5da6fc5ac 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala @@ -64,6 +64,27 @@ class RestHighLevelClientScrollPageFailureSpec extends AnyWordSpec with Matchers private val noAggregations = ListMap.empty[String, SQLAggregation] + "both extractors" should { + + "read a page in its statement's schema context: the rows and classes core's parse gives (SCHEMA-2, pages)" in { + val fixture = SchemaPageFixture + val context = fixture.context("6.8.23") + val page = tree(fixture.page) + val scroll = client.extractAllResults( + page, + fixture.fieldAliases, + noAggregations, + retainDocumentId = false + )(context) + val searchAfter = + client.extractHitsOnly(page, fixture.fieldAliases, retainDocumentId = false)(context) + Seq(scroll, searchAfter).foreach { rows => + rows shouldBe fixture.expected + fixture.classes(rows) shouldBe fixture.classes(fixture.expected) + } + } + } + "extractAllResults" should { "return one row per raw hit on a good page" in { diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala new file mode 100644 index 000000000..332457a4d --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSchemaReadingSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientSchemaReadingSpec extends SchemaReadingSpec diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala index d62f94f1c..ca83b6adf 100644 --- a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientScrollPageFailureSpec.scala @@ -66,6 +66,19 @@ class RestHighLevelClientScrollPageFailureSpec extends AnyWordSpec with Matchers "extractAllResults" should { + "read a page in its statement's schema context: the rows and classes core's parse gives (SCHEMA-2, pages)" in { + // `extractHitsOnly` is private here; it reads its page through the same core parse + val fixture = SchemaPageFixture + val rows = client.extractAllResults( + tree(fixture.page), + fixture.fieldAliases, + noAggregations, + retainDocumentId = false + )(fixture.context("7.17.29")) + rows shouldBe fixture.expected + fixture.classes(rows) shouldBe fixture.classes(fixture.expected) + } + "return one row per raw hit on a good page" in { client .extractAllResults(goodPage, ListMap.empty, noAggregations, retainDocumentId = false) diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 2def60925..8f3532f56 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -90,9 +90,9 @@ trait JavaClientConversion { _: JavaClientCompanion => * * This is the single-parse hits path (softclient4es-arrow#160): each `_source` node is * re-parented into the envelope untouched — no serialization, no re-parse. The row parser reads - * exactly `_id`, `_source`, `inner_hits` and `fields` per hit, so hits carrying `inner_hits` or - * `fields` (UNNEST legs, script fields) fall back to a whole-hit [[convertToTree]] to keep full - * shape fidelity — still token-level, never a string. + * exactly `_id`, `_source`, `_ignored`, `inner_hits` and `fields` per hit, so hits carrying + * `inner_hits` or `fields` (UNNEST legs, script fields) fall back to a whole-hit + * [[convertToTree]] to keep full shape fidelity — still token-level, never a string. */ protected def hitsToResponseNode(hits: _root_.java.util.List[Hit[ObjectNode]]): ObjectNode = { val root = JacksonConfig.objectMapper.createObjectNode() @@ -102,6 +102,16 @@ trait JavaClientConversion { _: JavaClientCompanion => val hitNode = hitsArray.addObject() Option(hit.id()).foreach(id => hitNode.put("_id", id)) Option(hit.source()).foreach(source => hitNode.set[JsonNode]("_source", source)) + // SCHEMA-2 (D8) -- the fields Elasticsearch ignored (`ignore_malformed`): core answers a + // date it ignored as NULL, as the clients reading the raw body do + val ignored = hit.ignored() + if (ignored != null && !ignored.isEmpty) { + val paths = hitNode.putArray("_ignored") + ignored.forEach { path => + paths.add(path) + () + } + } } else { hitsArray.add(convertToTree(hit)) } diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientDottedColumnSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientDottedColumnSpec.scala index b0f141873..08fe42c76 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientDottedColumnSpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientDottedColumnSpec.scala @@ -38,9 +38,9 @@ class JavaClientDottedColumnSpec extends DottedColumnSpec { /** Arrow's `ElasticSqlExecutor.extractRows`: every variant a gateway result takes. */ private def extractRows(result: QueryResult): Seq[ListMap[String, Any]] = result match { - case QueryRows(rows, _) => rows + case QueryRows(rows, _, _) => rows case QueryStructured(response, _) => response.results - case QueryStream(stream, _) => + case QueryStream(stream, _, _) => Await.result(stream.map(_._1).runWith(akka.stream.scaladsl.Sink.seq), 60.seconds) case _ => Seq.empty } diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala new file mode 100644 index 000000000..5d7bce9ae --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/arrow/JavaClientSchemaConsumerSpec.scala @@ -0,0 +1,205 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.arrow + +import akka.actor.ActorSystem +import akka.stream.scaladsl.Sink +import app.softnetwork.elastic.client.{ElasticClientApi, ResultSchema, SetOperationValues} +import app.softnetwork.elastic.client.result.{ + ElasticFailure, + ElasticSuccess, + QueryResult, + QueryRows, + QueryStream, + QueryStructured +} +import app.softnetwork.elastic.client.scroll.ScrollMetrics +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.sql.`type`.ValueCoercion +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.MultiSearch +import app.softnetwork.persistence.generateUUID +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.util.Try + +/** SCHEMA-2 (D15, AC7) -- the downstream consumers' forms, run on core's canonical rows, in a + * DOWNSTREAM package on purpose (SCHEMA-12 D11's precedent): arrow's `extractRows` (positional, 3 + * binders) and its leg readers (`DeclaredLegSchema.readers` = core's `LegSlot.read`, a no-op on a + * canonical row of every date format), the JDBC driver's `rows` match and `getString`, the + * extensions' `count(*)` fold, and the shape immune to the next arity change (`case q: + * QueryRows`). + */ +class JavaClientSchemaConsumerSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + private val table = "s2c_t" + + private val mapping: String = + """{"_meta": {"columns": {"c_des": {"data_type": "DATE"}}}, + | "properties": { + | "id": {"type": "keyword"}, "ts": {"type": "date"}, + | "c_des": {"type": "date", "format": "epoch_second"}, + | "c_c1": {"type": "date", "format": "yyyy/MM/dd"}, + | "n": {"type": "integer"}, "bin": {"type": "binary"}, "k": {"type": "keyword"} + |}}""".stripMargin + + private val docs: Seq[(String, String)] = Seq( + "c1" -> """{"id":"c1","ts":"2024-01-31T10:30:00","c_des":1706697000,"c_c1":"2024/01/31","n":1,"bin":"aGVsbG8=","k":"a"}""", + "c2" -> """{"id":"c2","ts":"2024-02-01T00:00:00Z","c_des":1706745600,"c_c1":"2024/02/01","n":2,"k":"b"}""" + ) + + override def beforeAll(): Unit = { + super.beforeAll() + Try(client.deleteIndex(table)) + client + .createIndex(table, settings = """{"number_of_shards": 1, "number_of_replicas": 0}""") + .get shouldBe true + client.setMapping(table, mapping).get shouldBe true + docs.foreach { case (id, source) => client.index(table, id, source) } + client.refresh(table) + () + } + + override def afterAll(): Unit = { + Try(client.deleteIndex(table)) + super.afterAll() + } + + private def run(sql: String): QueryResult = + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(result) => result + case ElasticFailure(error) => fail(s"[$sql] ${error.message}") + } + + /** Arrow's `ElasticSqlExecutor.extractRows`, positional patterns with three binders (D15). */ + private def extractRows(result: QueryResult): Seq[ListMap[String, Any]] = result match { + case QueryRows(rows, _, _) => rows + case QueryStructured(response, _) => response.results + case QueryStream(stream, _, _) => + Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) + case _ => Seq.empty + } + + /** The consumer shape immune to the next arity change, with the schema. */ + private def rowsAndSchema( + result: QueryResult + ): (Seq[ListMap[String, Any]], Option[ResultSchema]) = + result match { + case q: QueryRows => (q.rows, q.schema) + case q: QueryStructured => (q.response.results, q.schema) + case q: QueryStream => + (Await.result(q.stream.map(_._1).runWith(Sink.seq), 60.seconds), q.schema) + case other => fail(s"$other") + } + + "Arrow" should "read core's rows through both match shapes, by the schema's names" in { + Seq("", " LIMIT 10").foreach { limit => + val sql = s"SELECT id, ts, c_des, n FROM $table$limit" + val result = run(sql) + val (rows, schema) = rowsAndSchema(result) + extractRows(result).map(_.toMap) shouldBe rows.map(_.toMap) + val names = schema.getOrElse(fail("no schema")).names + names shouldBe Seq("id", "ts", "c_des", "n") + rows.foreach(row => names.foreach(n => row.contains(n) shouldBe true)) + } + } + + it should "read a canonical leg row through its leg readers unchanged, every date format (D12)" in { + val sqls = Seq( + s"SELECT ts AS d, n FROM $table UNION SELECT ts AS d, n FROM $table", + s"SELECT c_des AS d FROM $table UNION SELECT c_des AS d FROM $table", + s"SELECT c_c1 AS d FROM $table UNION SELECT c_c1 AS d FROM $table", + s"SELECT c_des AS d FROM $table UNION SELECT ts AS d FROM $table", + s"SELECT bin AS b, k FROM $table UNION SELECT bin AS b, k FROM $table" + ) + sqls.foreach { sql => + withClue(s"[$sql] ") { + val multiple = Parser(sql) match { + case Right(m: MultiSearch) => + m.copy(requests = m.requests.map(r => r.update(Some(client.loadSchema(table).get)))) + case other => fail(s"$other") + } + val legs = SetOperationValues + .legSchemas(multiple.requests, multiple.resolvedOperators, Some("8.18.3")) + .fold(refusal => fail(refusal), identity) + multiple.requests.zip(legs).foreach { case (leg, schema) => + // each leg as core reads it on its own: its canonical rows + val rows = extractRows(run(leg.sql)) + val names = leg.select.fieldsWithComputedAliases.map(_.outputName) + schema.foreach { s => + rows.foreach { row => + names.zip(s.slots).foreach { + case (name, Some(slot)) => + val cell = row.getOrElse(name, null) + val read = slot.read(cell) + (read, cell) match { + case (a: Array[Byte], b: Array[Byte]) => a.toSeq shouldBe b.toSeq + case _ => read shouldBe cell + } + case _ => + } + } + } + } + } + } + } + + "The JDBC driver" should "read the rows by its match and spell a binary as its base64" in { + Seq("", " LIMIT 10").foreach { limit => + val rows = run(s"SELECT id, bin FROM $table WHERE id = 'c1'$limit") match { + case QueryRows(rows, _, _) => rows + case QueryStructured(response, _) => response.results + case QueryStream(stream, _, _) => + Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) + case other => fail(s"$other") + } + rows.size shouldBe 1 + ValueCoercion.coerceToString(rows.head("bin")) shouldBe "aGVsbG8=" + } + } + + "The extensions" should "fold a count(*) as they do" in { + val count = run(s"SELECT count(*) as nb FROM $table") match { + case QueryRows(rows, _, _) => rows.headOption.map(_("nb")) + case QueryStream(stream, _, _) => + Await.result( + stream.runWith( + Sink.fold[Option[Any], (ListMap[String, Any], ScrollMetrics)](None) { + case (_, (row, _)) => row.get("nb") + } + ), + 60.seconds + ) + case QueryStructured(response, _) => response.results.headOption.map(_("nb")) + case other => fail(s"$other") + } + count.map(_.asInstanceOf[Number].longValue()) shouldBe Some(docs.size.toLong) + } +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala new file mode 100644 index 000000000..016e3c3e6 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientSchemaReadingSpec extends SchemaReadingSpec diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala index 5ed77c63b..90d6148ed 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala @@ -110,6 +110,33 @@ class JavaClientConversionSpec extends AnyWordSpec with Matchers { hitNode.has("_source") shouldBe false } + "carry the fields Elasticsearch ignored on the fast branch (SCHEMA-2, D8)" in { + val src = sourceNode() + val hit = Hit.of[ObjectNode] { builder => + builder.index("idx") + builder.id("5") + builder.source(src) + builder.ignored("dm") + builder + } + val hitNode = Companion + .envelope(Collections.singletonList(hit)) + .path("hits") + .path("hits") + .get(0) + hitNode.get("_source") should be theSameInstanceAs src + hitNode.path("_ignored").isArray shouldBe true + hitNode.path("_ignored").size() shouldBe 1 + hitNode.path("_ignored").get(0).asText() shouldBe "dm" + // a hit Elasticsearch ignored nothing of carries no `_ignored` + Companion + .envelope(Collections.singletonList(hitOf(Some("6"), Some(sourceNode())))) + .path("hits") + .path("hits") + .get(0) + .has("_ignored") shouldBe false + } + "produce an empty hits array for no hits" in { val hits = Companion .envelope(Collections.emptyList[Hit[ObjectNode]]()) diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 2def60925..8f3532f56 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -90,9 +90,9 @@ trait JavaClientConversion { _: JavaClientCompanion => * * This is the single-parse hits path (softclient4es-arrow#160): each `_source` node is * re-parented into the envelope untouched — no serialization, no re-parse. The row parser reads - * exactly `_id`, `_source`, `inner_hits` and `fields` per hit, so hits carrying `inner_hits` or - * `fields` (UNNEST legs, script fields) fall back to a whole-hit [[convertToTree]] to keep full - * shape fidelity — still token-level, never a string. + * exactly `_id`, `_source`, `_ignored`, `inner_hits` and `fields` per hit, so hits carrying + * `inner_hits` or `fields` (UNNEST legs, script fields) fall back to a whole-hit + * [[convertToTree]] to keep full shape fidelity — still token-level, never a string. */ protected def hitsToResponseNode(hits: _root_.java.util.List[Hit[ObjectNode]]): ObjectNode = { val root = JacksonConfig.objectMapper.createObjectNode() @@ -102,6 +102,16 @@ trait JavaClientConversion { _: JavaClientCompanion => val hitNode = hitsArray.addObject() Option(hit.id()).foreach(id => hitNode.put("_id", id)) Option(hit.source()).foreach(source => hitNode.set[JsonNode]("_source", source)) + // SCHEMA-2 (D8) -- the fields Elasticsearch ignored (`ignore_malformed`): core answers a + // date it ignored as NULL, as the clients reading the raw body do + val ignored = hit.ignored() + if (ignored != null && !ignored.isEmpty) { + val paths = hitNode.putArray("_ignored") + ignored.forEach { path => + paths.add(path) + () + } + } } else { hitsArray.add(convertToTree(hit)) } diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala new file mode 100644 index 000000000..016e3c3e6 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSchemaReadingSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientSchemaReadingSpec extends SchemaReadingSpec diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala index 5ed77c63b..90d6148ed 100644 --- a/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/java/JavaClientConversionSpec.scala @@ -110,6 +110,33 @@ class JavaClientConversionSpec extends AnyWordSpec with Matchers { hitNode.has("_source") shouldBe false } + "carry the fields Elasticsearch ignored on the fast branch (SCHEMA-2, D8)" in { + val src = sourceNode() + val hit = Hit.of[ObjectNode] { builder => + builder.index("idx") + builder.id("5") + builder.source(src) + builder.ignored("dm") + builder + } + val hitNode = Companion + .envelope(Collections.singletonList(hit)) + .path("hits") + .path("hits") + .get(0) + hitNode.get("_source") should be theSameInstanceAs src + hitNode.path("_ignored").isArray shouldBe true + hitNode.path("_ignored").size() shouldBe 1 + hitNode.path("_ignored").get(0).asText() shouldBe "dm" + // a hit Elasticsearch ignored nothing of carries no `_ignored` + Companion + .envelope(Collections.singletonList(hitOf(Some("6"), Some(sourceNode())))) + .path("hits") + .path("hits") + .get(0) + .has("_ignored") shouldBe false + } + "produce an empty hits array for no hits" in { val hits = Companion .envelope(Collections.emptyList[Hit[ObjectNode]]()) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 9df7883e7..b37c53178 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -4150,6 +4150,24 @@ package object query { Operand(branch, field, declaredType, positionType(declaredType, column)) } + /** SCHEMA-2 (D1) -- the type of ONE projection item, as a set operation types its position + * ([[positionType]]): the result schema of a statement and the typing of its set-operation + * positions are this one rule (#292), never two. + */ + private[elastic] def fieldType(field: Field): SetOperationType = + operandOf(0, field).valueType + + /** SCHEMA-2 (D1) -- each item a statement DECLARES with its type ([[fieldType]]); `None` for a + * statement that projects a bare `*`, whose width is its mapping's. + */ + private[elastic] def projectionTypes(s: SingleSearch): Option[Seq[(Field, SetOperationType)]] = + declared(s).map(_ => s.select.fieldsWithComputedAliases.map(f => f -> fieldType(f))) + + /** SCHEMA-2 (D1) -- a mapped column's type, as [[positionType]] types it: `SELECT *`'s columns. + */ + private[elastic] def columnType(c: Column): SetOperationType = + positionType(c.dataType, Some(c)) + /** A declared type as a set operation types it, built once at every depth: a scalar folded * ([[unionOperandType]]); a list with its element's type; a `STRUCT` with its fields -- its * column's, read from the schema, BY NAME: the schema read from a mapping holds an object's diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/ValueCoercion.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/ValueCoercion.scala index 2e058f9bd..696a3da74 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/ValueCoercion.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/ValueCoercion.scala @@ -61,8 +61,11 @@ object ValueCoercion { // ─── Coercions ─────────────────────────────────────────────────────────────── def coerceToString(value: Any): String = value match { - case null => null - case s: String => s + case null => null + case s: String => s + // SCHEMA-2 (D2) -- core reads a VARBINARY as its decoded bytes; its TEXT stays the base64 + // Elasticsearch holds, as in 0.24.0 (JDBC `getString`, arrow's VARCHAR writer) + case b: Array[Byte] => java.util.Base64.getEncoder.encodeToString(b) case seq: Seq[_] => seq.mkString("[", ", ", "]") case map: Map[_, _] => map.map { case (k, v) => s"$k: $v" }.mkString("{", ", ", "}") case other => other.toString diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/AggregateWithoutGroupBySpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/AggregateWithoutGroupBySpec.scala index b17bfe469..f7c2bc8ce 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/AggregateWithoutGroupBySpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/AggregateWithoutGroupBySpec.scala @@ -260,9 +260,9 @@ trait AggregateWithoutGroupBySpec extends AnyFlatSpecLike with ElasticDockerTest private def rows(sql: String): Either[String, Seq[ListMap[String, Any]]] = Try { Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryRows(rs, _, _)) => Right(rs) case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => Left(s"unexpected result $other") case ElasticFailure(error) => Left(error.message) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ClientErrorStatusSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ClientErrorStatusSpec.scala index 94d4a1566..d86bc7ebd 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ClientErrorStatusSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ClientErrorStatusSpec.scala @@ -235,9 +235,9 @@ trait ClientErrorStatusSpec extends AnyFlatSpecLike with ElasticDockerTestKit wi shardFixture val result = Await.result(client.run(rowSql), timeout) def rows: Seq[Any] = result match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(response, _)) => response.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Await.result(stream.map(_._1).runWith(Sink.seq), timeout) case ElasticSuccess(other) => Seq(other) case ElasticFailure(_) => Seq.empty @@ -463,9 +463,9 @@ trait ClientErrorStatusSpec extends AnyFlatSpecLike with ElasticDockerTestKit wi */ private def answeredRows(sql: String): Seq[ListMap[String, Any]] = Await.result(client.run(sql), timeout) match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(r, _)) => r.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Try(Await.result(stream.map(_._1).runWith(Sink.seq), timeout)).fold( e => fail(s"$sql -> the stream failed: ${e.getMessage}"), rows => rows diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala index 36d32d8c8..ed1bec4e3 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -358,9 +358,9 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with private def gatewayRows(sql: String): Either[String, Seq[ListMap[String, Any]]] = Try { Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryRows(rs, _, _)) => Right(rs) case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => Left(s"unexpected result $other") case ElasticFailure(error) => Left(error.message) @@ -391,18 +391,21 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with /** A number answered in exactly this class, on every row -- what tells a `Double` from an * `Integer`, which `Num` and `Numeric` accept alike: a `Double` for a fractional type; for a - * whole one the class Elasticsearch's JSON hands it back in, an `Integer` when it fits one and a - * `Long` past it (a BIGINT 3 is read as an `Integer` 3, as `CAST(n AS BIGINT)` itself is). + * whole one its declared type's class (SCHEMA-2, D2), an `INT` an `Integer` and a `BIGINT` a + * `Long`, whatever class Elasticsearch's JSON hands it back in (0.24.0 read a BIGINT 3 as an + * `Integer` 3). */ private final case class Classed(value: Double, cls: Class[_]) extends Expected private def doubleOf(value: Double): Expected = Classed(value, classOf[java.lang.Double]) - private def wholeOf(value: Long): Expected = - Classed( - value.toDouble, - if (value.isValidInt) classOf[java.lang.Integer] else classOf[java.lang.Long] - ) + /** A whole number of a `BIGINT` -- the widest whole type of its arguments: an INT beside a + * BIGINT, an integer literal included. + */ + private def wholeOf(value: Long): Expected = Classed(value.toDouble, classOf[java.lang.Long]) + + /** A whole number of an `INT`. */ + private def intOf(value: Int): Expected = Classed(value.toDouble, classOf[java.lang.Integer]) /** An instant read from the clock while the statement ran: between `from` and `to`. */ private final case class Within(from: Instant, to: Instant) extends Expected @@ -2866,7 +2869,7 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with // a cast over a KEYWORD is the type it casts to, never its column's: whole numbers cv( "GREATEST(CAST(s AS INT), n)", - answers(wholeOf(3), wholeOf(5), wholeOf(12), Null, wholeOf(10)) + answers(intOf(3), intOf(5), intOf(12), Null, intOf(10)) ), cv("GREATEST(n, 3000000000)", bigint), cv( diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateFunctionExecutionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateFunctionExecutionSpec.scala index 94b6fcd23..9d0202fcb 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DateFunctionExecutionSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateFunctionExecutionSpec.scala @@ -150,9 +150,9 @@ trait DateFunctionExecutionSpec extends AnyFlatSpecLike with ElasticDockerTestKi /** id -> the single projected column, through `GatewayApi.run` (JDBC / REPL / sidecar venue). */ private def gatewayById(sql: String, column: String): Map[String, String] = { val rows = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(response, _)) => response.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case ElasticSuccess(other) => fail(s"Unexpected result: $other\n$sql") case ElasticFailure(error) => fail(s"Query failed: ${error.message}\n$sql") @@ -576,9 +576,9 @@ trait DateFunctionExecutionSpec extends AnyFlatSpecLike with ElasticDockerTestKi private def diffRows(sql: String): Either[String, Seq[ListMap[String, Any]]] = Try { Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => Right(rows) + case ElasticSuccess(QueryRows(rows, _, _)) => Right(rows) case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => Left(s"unexpected result $other") case ElasticFailure(error) => Left(error.message) @@ -1714,9 +1714,9 @@ trait DateFunctionExecutionSpec extends AnyFlatSpecLike with ElasticDockerTestKi case ElasticFailure(error) => Left("run" -> error) case ElasticSuccess(result) => Try(result match { - case QueryRows(rows, _) => rows + case QueryRows(rows, _, _) => rows case QueryStructured(response, _) => response.results - case QueryStream(stream, _) => + case QueryStream(stream, _, _) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case other => throw new IllegalStateException(s"Unexpected result: $other") }).toEither.left.map("consumed" -> _) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala index 5324a7229..9b79a8c47 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DottedColumnSpec.scala @@ -179,9 +179,9 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma /** Every variant a gateway result can take, as a consumer reads it (arrow's `extractRows`). */ protected def rowsOfResult(result: QueryResult): Seq[ListMap[String, Any]] = result match { - case QueryRows(rows, _) => rows + case QueryRows(rows, _, _) => rows case QueryStructured(response, _) => response.results - case QueryStream(stream, _) => + case QueryStream(stream, _, _) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case other => fail(s"unexpected result $other") } @@ -528,22 +528,31 @@ trait DottedColumnSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Ma Seq("dn", "dz", "dn4").foreach(id => withClue(s"$id: ")(s2(id).keySet shouldBe Set("id", "obj.a")) ) - // a non-null flat spelling won, an array: the parent stays, as main returns it - s2("db")("obj") shouldBe Map("a" -> 30) - s2("dl")("obj") shouldBe List(Map("a" -> 5), Map("a" -> 50)) + // SCHEMA-2 (D9): where a non-null flat spelling won, or the parent is an array, it is + // dropped too -- SCHEMA-12 A-1's recorded leak ends: the projection's columns only + Seq("db", "dl").foreach(id => withClue(s"$id: ")(s2(id).keySet shouldBe Set("id", "obj.a"))) val s3 = byId(rowsOk(s"SELECT id, obj.x.y FROM $table", route)) - s3("dx").keySet shouldBe Set("id", "obj.x.y") - s3("dmx")("obj") shouldBe Map("x.y" -> 10) - // a requested parent, an aliased source under it, an alias collision, a star (A-5, D5) + Seq("dx", "dmx").foreach(id => + withClue(s"$id: ")(s3(id).keySet shouldBe Set("id", "obj.x.y")) + ) + // a requested parent and a star keep it: it is one of their columns (A-5, D5) Seq( s"SELECT id, obj, obj.a FROM $table", - s"SELECT id, obj.a, obj.b AS k FROM $table", - s"SELECT id, obj.a, obj.a AS k FROM $table", - s"SELECT id, a AS z, obj.a FROM $table", s"SELECT *, obj.a FROM $table" ).foreach { sql => withClue(s"$sql: ")(byId(rowsOk(sql, route))("dn") should contain key "obj") } + // an aliased source under it, an alias collision, a column beside it: SCHEMA-2 (D9) drops + // it -- the projection's columns only --, each column keeping its value + val aliased = byId(rowsOk(s"SELECT id, obj.a, obj.b AS k FROM $table", route)) + aliased("dn").keySet shouldBe Set("id", "obj.a", "k") + String.valueOf(aliased("ds")("k")) shouldBe "6" + val collision = byId(rowsOk(s"SELECT id, obj.a, obj.a AS k FROM $table", route)) + collision("dn").keySet shouldBe Set("id", "obj.a", "k") + String.valueOf(collision("dn")("k")) shouldBe "1" + val beside = byId(rowsOk(s"SELECT id, a AS z, obj.a FROM $table", route)) + beside("dn").keySet shouldBe Set("id", "z", "obj.a") + String.valueOf(beside("dn")("z")) shouldBe "7" } } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index c27680b6b..c79896244 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -2526,6 +2526,8 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { |FROM dql_users |ORDER BY id ASC;""".stripMargin + // SCHEMA-2 (D9, #418 part 2): the projection's columns only -- the parent `profile` its dotted + // columns read is no longer leaked into the row val res = client.run(sql).futureValue assertSelectResult( System.nanoTime(), @@ -2535,29 +2537,25 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { "id" -> 1, "full_name" -> "Alice", "city" -> "Paris", - "followers" -> 100, - "profile" -> Map("city" -> "Paris", "followers" -> 100) + "followers" -> 100 ), Map( "id" -> 2, "full_name" -> "Bob", "city" -> "Lyon", - "followers" -> 50, - "profile" -> Map("city" -> "Lyon", "followers" -> 50) + "followers" -> 50 ), Map( "id" -> 3, "full_name" -> "Chloe", "city" -> "Paris", - "followers" -> 200, - "profile" -> Map("city" -> "Paris", "followers" -> 200) + "followers" -> 200 ), Map( "id" -> 4, "full_name" -> "David", "city" -> "Marseille", - "followers" -> 10, - "profile" -> Map("city" -> "Marseille", "followers" -> 10) + "followers" -> 10 ) ) ) @@ -3444,6 +3442,7 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { |FROM dql_users |ORDER BY id ASC;""".stripMargin + // SCHEMA-2 (D5, #418 part 1): a computed column's one-element array is ONE value val res = client.run(sql).futureValue assertSelectResult( System.nanoTime(), @@ -3451,55 +3450,55 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { Seq( Map( "id" -> 1, - "name_concat" -> Seq("Alice_suffix"), - "name_sub" -> Seq("Al"), - "name_lower" -> Seq("alice"), - "name_upper" -> Seq("ALICE"), - "name_trim" -> Seq("Alice"), - "name_len" -> Seq(5), - "name_repl" -> Seq("Xlice"), - "name_left" -> Seq("A"), - "name_right" -> Seq("e"), - "name_rev" -> Seq("ecilA") + "name_concat" -> "Alice_suffix", + "name_sub" -> "Al", + "name_lower" -> "alice", + "name_upper" -> "ALICE", + "name_trim" -> "Alice", + "name_len" -> 5, + "name_repl" -> "Xlice", + "name_left" -> "A", + "name_right" -> "e", + "name_rev" -> "ecilA" ), Map( "id" -> 2, - "name_concat" -> Seq("Bob_suffix"), - "name_sub" -> Seq("Bo"), - "name_lower" -> Seq("bob"), - "name_upper" -> Seq("BOB"), - "name_trim" -> Seq("Bob"), - "name_len" -> Seq(3), - "name_repl" -> Seq("Bob"), - "name_left" -> Seq("B"), - "name_right" -> Seq("b"), - "name_rev" -> Seq("boB") + "name_concat" -> "Bob_suffix", + "name_sub" -> "Bo", + "name_lower" -> "bob", + "name_upper" -> "BOB", + "name_trim" -> "Bob", + "name_len" -> 3, + "name_repl" -> "Bob", + "name_left" -> "B", + "name_right" -> "b", + "name_rev" -> "boB" ), Map( "id" -> 3, - "name_concat" -> Seq("Chloe_suffix"), - "name_sub" -> Seq("Ch"), - "name_lower" -> Seq("chloe"), - "name_upper" -> Seq("CHLOE"), - "name_trim" -> Seq("Chloe"), - "name_len" -> Seq(5), - "name_repl" -> Seq("Chloe"), - "name_left" -> Seq("C"), - "name_right" -> Seq("e"), - "name_rev" -> Seq("eolhC") + "name_concat" -> "Chloe_suffix", + "name_sub" -> "Ch", + "name_lower" -> "chloe", + "name_upper" -> "CHLOE", + "name_trim" -> "Chloe", + "name_len" -> 5, + "name_repl" -> "Chloe", + "name_left" -> "C", + "name_right" -> "e", + "name_rev" -> "eolhC" ), Map( "id" -> 4, - "name_concat" -> Seq("David_suffix"), - "name_sub" -> Seq("Da"), - "name_lower" -> Seq("david"), - "name_upper" -> Seq("DAVID"), - "name_trim" -> Seq("David"), - "name_len" -> Seq(5), - "name_repl" -> Seq("David"), - "name_left" -> Seq("D"), - "name_right" -> Seq("d"), - "name_rev" -> Seq("divaD") + "name_concat" -> "David_suffix", + "name_sub" -> "Da", + "name_lower" -> "david", + "name_upper" -> "DAVID", + "name_trim" -> "David", + "name_len" -> 5, + "name_repl" -> "David", + "name_left" -> "D", + "name_right" -> "d", + "name_rev" -> "divaD" ) ) ) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayIntegrationTestKit.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayIntegrationTestKit.scala index ee54eafbd..bebb8d5c9 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayIntegrationTestKit.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayIntegrationTestKit.scala @@ -128,7 +128,7 @@ trait GatewayIntegrationTestKit extends AnyFlatSpecLike with Matchers with Scala } res.isSuccess shouldBe true res.toOption.get match { - case QueryStream(stream, _) => + case QueryStream(stream, _, _) => val sink = Sink.fold[Seq[ListMap[String, Any]], (ListMap[String, Any], ScrollMetrics)](Seq.empty) { case (acc, (row, _)) => @@ -224,7 +224,7 @@ trait GatewayIntegrationTestKit extends AnyFlatSpecLike with Matchers with Scala renderResults(startTime, res) res.isSuccess shouldBe true res.toOption.get match { - case QueryStream(stream, _) => + case QueryStream(stream, _, _) => val sink = Sink.fold[Seq[ListMap[String, Any]], (ListMap[String, Any], ScrollMetrics)](Seq.empty) { case (acc, (row, _)) => acc :+ normalizeRow(row) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GroupByCompletenessSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GroupByCompletenessSpec.scala index 3ab724347..6cee6077e 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GroupByCompletenessSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GroupByCompletenessSpec.scala @@ -344,25 +344,19 @@ trait GroupByCompletenessSpec extends AnyFlatSpecLike with ElasticDockerTestKit } } - // 🔴 PINS A KNOWN DEFECT, not a contract -- delete this test when the defect is fixed. - // - // The same constant reaches the caller DIFFERENTLY on the two paths, MEASURED on real ES 8.18: - // aggregation path (GROUP BY): `flag -> 2` -- a clean scalar, projected from the AST - // row path (no GROUP BY): `flag -> List(2)` -- array-wrapped - // - // The wrap is the pre-existing `script_fields` defect (`jsonNodeToAny` keeps an ES per-field - // array as a List and nothing unwraps it), recorded in - // docs/issues/local-21.3-script-fields-values-are-array-wrapped.md. The aggregation side is the - // CORRECT one and is deliberately NOT wrapped to match: matching would spread a defect to a path - // that does not have it. Pinned so the divergence is visible rather than discovered. - it should "expose the known row-path array wrap, which the aggregation path does not share" in { - implicit val wrapCtx: ConversionContext = NativeContext + // The same constant reaches the caller the SAME way on the two paths: the aggregation path + // (GROUP BY) projects `flag -> 2` from the AST; the row path (no GROUP BY) reads Elasticsearch's + // `script_fields` array, which held it as `List(2)` until SCHEMA-2 (D5, #418 part 1) read a + // computed column's one-element array as ONE value. + it should "answer a constant as one value on the row path, as the aggregation path does" in { + implicit val rowCtx: ConversionContext = NativeContext client.search( SelectStatement("SELECT id, 2 AS flag FROM group_by_completeness LIMIT 2") ) match { case ElasticSuccess(response) => + response.results should have size 2 response.results.foreach { row => - withClue(s"row path, row=$row: ") { row("flag") shouldBe List(2) } + withClue(s"row path, row=$row: ") { row("flag") shouldBe 2 } } case ElasticFailure(error) => fail(s"Query failed: ${error.message}") } @@ -1522,9 +1516,9 @@ trait GroupByCompletenessSpec extends AnyFlatSpecLike with ElasticDockerTestKit /** The rows of a statement, through `GatewayApi.run` (JDBC / REPL / Flight SQL). */ private def gatewayRows(sql: String): Either[String, Seq[ListMap[String, Any]]] = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => Right(rows) + case ElasticSuccess(QueryRows(rows, _, _)) => Right(rows) case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => Left(s"unexpected result $other") case ElasticFailure(error) => Left(error.message) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GroupedAggregateSemanticsSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GroupedAggregateSemanticsSpec.scala index 0e92194a1..f5494eb2c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GroupedAggregateSemanticsSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GroupedAggregateSemanticsSpec.scala @@ -126,9 +126,9 @@ trait GroupedAggregateSemanticsSpec /** The rows of a statement, through `GatewayApi.run` (JDBC / REPL / Flight SQL). */ private def rowsOf(sql: String): Seq[ListMap[String, Any]] = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(response, _)) => response.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case ElasticSuccess(other) => fail(s"[$sql] unexpected result $other") case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") @@ -567,9 +567,10 @@ trait GroupedAggregateSemanticsSpec instant(cell(formatted, "ts")) shouldBe at("2025-03-31T10:00:00Z") cell(formatted, "ts") shouldBe a[ZonedDateTime] cell(formatted, "y") shouldBe "2025" - // DuckDB: d DATE 2024-01-03, s VARCHAR + // DuckDB: d DATE 2024-01-03, s VARCHAR -- `d` is a `date` field no type declares: a + // TIMESTAMP (SCHEMA-2, D2), its midnight at UTC val cast = rowsOf(s"SELECT id, d, CAST(d AS VARCHAR) AS s FROM $index WHERE id = 'a1'").head - cell(cast, "d") shouldBe LocalDate.of(2024, 1, 3) + cell(cast, "d") shouldBe ZonedDateTime.of(2024, 1, 3, 0, 0, 0, 0, ZoneOffset.UTC) cell(cast, "s") shouldBe a[String] } @@ -579,8 +580,8 @@ trait GroupedAggregateSemanticsSpec "2025-03-31T10:00:00" val star = rowsOf(s"SELECT * FROM $index WHERE id = 'a1'").head cell(star, "kd") shouldBe "2025-03-31T10:00:00" - // the control: the DATE and TIMESTAMP columns of the same row keep their types - cell(star, "d") shouldBe LocalDate.of(2024, 1, 3) + // the control: the date columns of the same row keep their type, a TIMESTAMP (SCHEMA-2, D2) + cell(star, "d") shouldBe ZonedDateTime.of(2024, 1, 3, 0, 0, 0, 0, ZoneOffset.UTC) cell(star, "ts") shouldBe a[ZonedDateTime] } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/NonDocMappingTypeSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/NonDocMappingTypeSpec.scala index 49817433e..9e6bec5ec 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/NonDocMappingTypeSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/NonDocMappingTypeSpec.scala @@ -103,9 +103,9 @@ trait NonDocMappingTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wi private def ids(sql: String): Either[String, Set[String]] = Try { Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryRows(rs, _, _)) => Right(rs) case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => Left(s"unexpected result $other") case ElasticFailure(error) => Left(error.message) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/PredicateFunctionResultSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/PredicateFunctionResultSpec.scala index 4e6c4c4a9..8f53cb863 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/PredicateFunctionResultSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/PredicateFunctionResultSpec.scala @@ -178,9 +178,9 @@ trait PredicateFunctionResultSpec extends AnyFlatSpecLike with ElasticDockerTest private def gatewaySelected(where: String): Set[String] = { val sql = s"SELECT id FROM $index WHERE $where" val rows = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(response, _)) => response.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case ElasticSuccess(other) => fail(s"Unexpected result: $other\n$sql") case ElasticFailure(error) => fail(s"Query failed: ${error.message}\n$sql") @@ -506,9 +506,9 @@ trait PredicateFunctionResultSpec extends AnyFlatSpecLike with ElasticDockerTest private def gatewayWhereIds(where: String): Set[String] = { val sql = s"SELECT id FROM $precedenceIndex WHERE $where" val rows = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(response, _)) => response.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case ElasticSuccess(other) => fail(s"Unexpected result: $other\n$sql") case ElasticFailure(error) => fail(s"Query failed: ${error.message}\n$sql") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaMissAfterCreateTableSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaMissAfterCreateTableSpec.scala index e2f809a9d..357bea571 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaMissAfterCreateTableSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaMissAfterCreateTableSpec.scala @@ -102,9 +102,9 @@ trait SchemaMissAfterCreateTableSpec private def answer(sql: String): Either[String, Seq[ListMap[String, Any]]] = Try { Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryRows(rs, _, _)) => Right(rs) case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => Left(s"unexpected result $other") case ElasticFailure(error) => Left(error.message) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala new file mode 100644 index 000000000..32c5078cf --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SchemaReadingSpec.scala @@ -0,0 +1,388 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import akka.stream.scaladsl.Sink +import app.softnetwork.elastic.client.result.{ + ElasticFailure, + ElasticResult, + ElasticSuccess, + QueryResult, + QueryRows, + QueryStream, + QueryStructured +} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.query.SelectStatement +import app.softnetwork.persistence.generateUUID +import org.apache.http.entity.ContentType +import org.apache.http.nio.entity.NStringEntity +import org.elasticsearch.client.Request +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.time.{LocalDate, LocalTime, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.util.Try + +/** SCHEMA-2 -- the declared [[ResultSchema]] and the typed hit rows, on a real cluster. + * + * - "analysis rows": table T of the story (AC3) on fixture `s2_t`; + * - "classes": every non-null value of a column whose type is not value-typed has D2's class, on + * every hit route (AC2): one-shot search, `searchAsync`, scroll / PIT, search_after, the + * `_msearch` legs and the per-leg `UNION ALL` route, UNNEST inner hits; + * - "q4": `SELECT *` = the cached mapping, an unmapped path read as ANY, an explicit path's one + * reload (AC6, Elasticsearch 7+); + * - "describe": `describe(sql)` = `run(sql)`'s schema (AC1). + * + * PUBLIC API ONLY but `describe` and the schema: the same fixture runs as the T0 baseline probe on + * a main clone. + */ +trait SchemaReadingSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + protected val table = "s2_t" + protected val q4 = "q4_t" + protected val q4d = "q4d_t" + + protected lazy val major: Int = Try(elasticVersion.takeWhile(_ != '.').toInt).getOrElse(8) + + private val mapping: String = + """{"_meta": {"columns": {"c_des": {"data_type": "DATE"}}}, + | "properties": { + | "id": {"type": "keyword"}, + | "ts": {"type": "date"}, + | "c_des": {"type": "date", "format": "epoch_second"}, + | "c_c1": {"type": "date", "format": "yyyy/MM/dd"}, + | "dm": {"type": "date", "ignore_malformed": true}, + | "n": {"type": "integer"}, "x": {"type": "double"}, + | "b": {"type": "boolean"}, "g": {"type": "geo_point"}, "bin": {"type": "binary"}, + | "k": {"type": "keyword"}, "kw": {"type": "keyword", "ignore_above": 5}, + | "nul": {"type": "keyword"}, + | "obj": {"properties": {"a": {"type": "integer"}}}, + | "items": {"type": "nested", "properties": {"p": {"type": "integer"}}} + |}}""".stripMargin + + /** One document per value of table T (`_id` = `id`); a point as WKT (`g4`) where the major + * indexes it -- Elasticsearch 7+: 6.8.23 reads a text point as `lat,lon` or a geohash only. + */ + protected def docs: Seq[(String, String)] = Seq( + "t1" -> + """{"id":"t1","ts":"2024-01-31T10:30:00","c_des":1706697000,"c_c1":"2024/01/31","n":"5.5","x":" 2.5 ","b":"","bin":"aGVsbG8=","k":"10:30","dm":"not-a-date","kw":"abcdefgh","g":{"lat":1.5,"lon":2.5},"obj":{"a":1},"items":[{"p":1},{"p":2}]}""", + "t3" -> """{"id":"t3","n":3,"g":"1.5,2.5"}""", + "g3" -> """{"id":"g3","g":[2.5,1.5]}""" + ) ++ (if (major >= 7) Seq("g4" -> """{"id":"g4","g":"POINT (2.5 1.5)"}""") else Nil) ++ Seq( + "g5" -> """{"id":"g5","g":"s0"}""" + ) + + private def drop(index: String): Unit = { + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $index"), 60.seconds)) + Try(client.deleteIndex(index)) + } + + private def load(index: String, mappingJson: String, documents: Seq[(String, String)]): Unit = { + client + .createIndex(index, settings = """{"number_of_shards": 2, "number_of_replicas": 0}""") + .get shouldBe true + client.setMapping(index, mappingJson).get shouldBe true + documents.foreach { case (id, source) => + client.index(index, id, source) match { + case ElasticSuccess(true) => + case other => fail(s"indexing $id into $index: $other") + } + } + client.refresh(index) + } + + override def beforeAll(): Unit = { + super.beforeAll() + Seq(table, q4, q4d).foreach(drop) + load(table, mapping, docs) + () + } + + override def afterAll(): Unit = { + Seq(table, q4, q4d).foreach(drop) + super.afterAll() + } + + // ------------------------------------------------------------------------------------------- + // Results, routes, the class oracle + // ------------------------------------------------------------------------------------------- + + protected def runOk(sql: String): QueryResult = + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(result) => result + case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") + } + + /** The rows and the schema of a result, as a consumer reads them (`case q: QueryRows`). */ + protected def answer(result: QueryResult): (Seq[ListMap[String, Any]], Option[ResultSchema]) = + result match { + case q: QueryRows => (q.rows, q.schema) + case q: QueryStructured => (q.response.results, q.schema) + case q: QueryStream => + (Await.result(q.stream.map(_._1).runWith(Sink.seq), 60.seconds), q.schema) + case other => fail(s"unexpected result $other") + } + + protected def rowsOf(sql: String): Seq[ListMap[String, Any]] = answer(runOk(sql))._1 + + /** The routes a hit row is read on (AC2). */ + protected def onRoute( + route: String, + sql: String + ): (Seq[ListMap[String, Any]], Option[ResultSchema]) = + route match { + case "search" => + client.search(SelectStatement(s"$sql LIMIT 100"))(NativeContext) match { + case ElasticSuccess(r) => (r.results, r.schema) + case ElasticFailure(error) => fail(s"[$route] $sql: ${error.message}") + } + case "searchAsync" => answer(runOk(s"$sql LIMIT 100")) + case "scroll" => answer(runOk(sql)) + case "search_after" => answer(runOk(s"$sql ORDER BY id")) + case "msearch" => answer(runOk(s"$sql LIMIT 100 UNION ALL $sql LIMIT 100")) + case "per-leg" => answer(runOk(s"$sql UNION ALL $sql")) + } + + protected val routes: Seq[String] = + Seq("search", "searchAsync", "scroll", "search_after", "msearch", "per-leg") + + /** D2's class for a value of `column`, at every depth; `true` for a value-typed column. */ + protected def classOk(column: ResultColumn, value: Any): Boolean = + value == null || (column.sqlType match { + case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int => value.isInstanceOf[Integer] + case SQLTypes.BigInt => value.isInstanceOf[java.lang.Long] + case SQLTypes.Double => value.isInstanceOf[java.lang.Double] + case SQLTypes.Real => value.isInstanceOf[java.lang.Float] + case SQLTypes.Boolean => value.isInstanceOf[java.lang.Boolean] + case SQLTypes.Varchar => value.isInstanceOf[String] + case SQLTypes.Date => value.isInstanceOf[LocalDate] + case SQLTypes.Time => value.isInstanceOf[LocalTime] + case SQLTypes.Timestamp => + value.isInstanceOf[ZonedDateTime] && + value.asInstanceOf[ZonedDateTime].getZone == ZoneOffset.UTC + case SQLTypes.VarBinary => value.isInstanceOf[Array[Byte]] + case SQLTypes.GeoPoint | SQLTypes.Struct => + value match { + case m: scala.collection.Map[_, _] => + val map = m.asInstanceOf[scala.collection.Map[String, Any]] + column.fields.forall(f => classOk(f, map.getOrElse(f.name, null))) + case _ => false + } + case SQLTypes.Array(_) => + value match { + case s: Seq[_] => column.element.forall(e => s.forall(classOk(e, _))) + case _ => false + } + case _ => true // NUMERIC, TEMPORAL, ANY: value-typed (R-13) + }) + + protected def classMisses( + rows: Seq[ListMap[String, Any]], + schema: ResultSchema + ): Seq[String] = + for { + row <- rows + column <- schema.columns + value = row.getOrElse(column.name, null) + if !classOk(column, value) + } yield s"${row.getOrElse("id", "?")}.${column.name}: $value (${value.getClass.getName}) is not ${column.sqlType.typeId}" + + // ------------------------------------------------------------------------------------------- + // Analysis rows (table T) + // ------------------------------------------------------------------------------------------- + + private def utc(y: Int, mo: Int, d: Int, h: Int = 0, mi: Int = 0): ZonedDateTime = + ZonedDateTime.of(y, mo, d, h, mi, 0, 0, ZoneOffset.UTC) + + private def one(sql: String): ListMap[String, Any] = + rowsOf(sql) match { + case Seq(row) => row + case other => fail(s"[$sql] answered ${other.size} rows: $other") + } + + "Table T" should "give the analysis rows' answers (AC3)" in { + val t1 = one(s"SELECT id, ts, c_des, c_c1, n, x, b, bin, nul FROM $table WHERE id = 't1'") + // 1: a date of the default format, undeclared: TIMESTAMP at UTC + t1("ts") shouldBe utc(2024, 1, 31, 10, 30) + // 2: epoch_second declared DATE + t1("c_des") shouldBe LocalDate.of(2024, 1, 31) + // 3: a custom format + t1("c_c1") shouldBe utc(2024, 1, 31) + // 5: numbers Elasticsearch accepted as text + t1("n") shouldBe Integer.valueOf(5) + t1("x") shouldBe java.lang.Double.valueOf(2.5) + // 6: an empty text in a boolean field + t1("b") shouldBe java.lang.Boolean.FALSE + // 8: binary, decoded + t1("bin").asInstanceOf[Array[Byte]].toSeq shouldBe "hello".getBytes("US-ASCII").toSeq + // 12: a keyword never set + t1("nul") shouldBe (null: Any) + // 7: every spelling of a point + val points = rowsOf(s"SELECT id, g FROM $table WHERE id IN ('t1', 't3', 'g3', 'g4')") + points.map(_("g")).distinct shouldBe Seq(ListMap("lat" -> 1.5, "lon" -> 2.5)) + one(s"SELECT g FROM $table WHERE id = 'g5'")("g") match { + case m: scala.collection.Map[_, _] => m.keySet shouldBe Set("lat", "lon") + case other => fail(s"geohash answered $other") + } + // 10: a computed column, unwrapped + val v = one(s"SELECT id, n + 1 AS v FROM $table WHERE id = 't3'")("v") + v.asInstanceOf[Number].longValue() shouldBe 4L + v should not be a[Seq[_]] + // 13: a DATE beside a TIMESTAMP is a TIMESTAMP in both legs; a keyword is its text + val union = rowsOf( + s"SELECT c_des AS d, k FROM $table WHERE id = 't1' UNION ALL " + + s"SELECT ts AS d, k FROM $table WHERE id = 't1'" + ) + union.map(_("d")).toSet shouldBe Set(utc(2024, 1, 31), utc(2024, 1, 31, 10, 30)) + union.map(_("k")).distinct shouldBe Seq("10:30") + // 17: a date Elasticsearch ignored is NULL; a keyword over ignore_above keeps its text + val t17 = one(s"SELECT dm, kw FROM $table WHERE id = 't1'") + t17("dm") shouldBe (null: Any) + t17("kw") shouldBe "abcdefgh" + } + + // ------------------------------------------------------------------------------------------- + // Classes (AC2) + // ------------------------------------------------------------------------------------------- + + "Every hit route" should "answer every value of a typed column in D2's class (AC2)" in { + val sql = s"SELECT id, ts, c_des, c_c1, n, x, b, bin, k, nul, obj FROM $table" + routes.foreach { route => + val (rows, schema) = onRoute(route, sql) + withClue(s"[$route] ") { + rows should not be empty + val s = schema.getOrElse(fail("no schema")) + s.names shouldBe Seq("id", "ts", "c_des", "c_c1", "n", "x", "b", "bin", "k", "nul", "obj") + classMisses(rows, s) shouldBe empty + // strict projection: no undeclared key + rows.foreach(row => + (row.keySet -- s.names - ElasticConversion.DocumentIdField) shouldBe empty + ) + } + } + // a point column, its documents' every spelling (several points excluded, D7) + routes.foreach { route => + val (rows, schema) = onRoute(route, s"SELECT id, g FROM $table") + withClue(s"[$route] g ")( + classMisses(rows, schema.getOrElse(fail("no schema"))) shouldBe empty + ) + } + } + + it should "read an UNNEST element by its column's type (inner hits)" in { + val (rows, schema) = + answer(runOk(s"SELECT t.id, i.p FROM $table AS t JOIN UNNEST(t.items) AS i LIMIT 100")) + rows.map(_("i.p")).toSet shouldBe Set(1, 2) + schema.foreach(s => classMisses(rows, s) shouldBe empty) + } + + // ------------------------------------------------------------------------------------------- + // Q4 (AC6) + // ------------------------------------------------------------------------------------------- + + private def rawPut(path: String, body: String): Unit = { + val request = new Request("PUT", path) + request.setEntity(new NStringEntity(body, ContentType.APPLICATION_JSON)) + restClient.performRequest(request) + () + } + + private def typeOf(result: QueryResult, column: String): Option[String] = + answer(result)._2.flatMap(_.columns.find(_.name == column)).map(_.sqlType.typeId) + + "SELECT * and an unmapped path" should "follow the cached mapping (AC6, q4)" in { + assume(major >= 7, "Q4 runs on Elasticsearch 7+ (typeless mapping PUT)") + load( + q4, + """{"dynamic": false, "properties": {"id": {"type": "keyword"}, + | "obj": {"properties": {"a": {"type": "integer"}}}}}""".stripMargin, + Seq("1" -> """{"id":"1","extra":42,"extra2":7,"obj":{"a":1,"zz":1}}""") + ) + val star = runOk(s"SELECT * FROM $q4") + val (starRows, starSchema) = answer(star) + starSchema.map(_.names) shouldBe Some(Seq("id", "obj")) + starRows.head.contains("extra") shouldBe false + starRows.head("obj") shouldBe ListMap("a" -> 1) + // an unmapped column: ANY, as read + val extra = runOk(s"SELECT extra FROM $q4") + answer(extra)._1.head("extra") shouldBe 42 + typeOf(extra, "extra") shouldBe Some("ANY") + // an unmapped path under a mapped parent + answer(runOk(s"SELECT obj.zz FROM $q4"))._1.head("obj.zz") shouldBe 1 + // a field mapped since (raw PUT, not through core): the memoized `extra` reads no mapping... + rawPut( + s"/$q4/_mapping", + """{"properties": {"extra": {"type": "long"}, "extra2": {"type": "long"}}}""" + ) + typeOf(runOk(s"SELECT extra FROM $q4"), "extra") shouldBe Some("ANY") + // ...a path never selected reloads it once: BIGINT, describe too + val extra2 = runOk(s"SELECT extra2 FROM $q4") + typeOf(extra2, "extra2") shouldBe Some("BIGINT") + answer(extra2)._1.head("extra2") shouldBe java.lang.Long.valueOf(7L) + client.describe(s"SELECT extra2 FROM $q4") match { + case ElasticSuccess(Some(s)) => s.columns.map(_.sqlType.typeId) shouldBe Seq("BIGINT") + case other => fail(s"$other") + } + } + + it should "show a field mapped since only after an explicit selection reloads it (A-17)" in { + assume(major >= 7, "Q4 runs on Elasticsearch 7+") + load(q4d, """{"properties": {"id": {"type": "keyword"}}}""", Seq("1" -> """{"id":"1"}""")) + answer(runOk(s"SELECT * FROM $q4d"))._2.map(_.names) shouldBe Some(Seq("id")) + client.index(q4d, "2", """{"id":"2","nf":5}""") + client.refresh(q4d) + answer(runOk(s"SELECT * FROM $q4d"))._2.map(_.names) shouldBe Some(Seq("id")) + val nf = runOk(s"SELECT nf FROM $q4d") + typeOf(nf, "nf") shouldBe Some("BIGINT") + answer(runOk(s"SELECT * FROM $q4d"))._2.map(_.names.toSet) shouldBe Some(Set("id", "nf")) + } + + // ------------------------------------------------------------------------------------------- + // describe (AC1) + // ------------------------------------------------------------------------------------------- + + "describe" should "answer run's schema (AC1)" in { + Seq( + s"SELECT id, ts, c_des, n FROM $table", + s"SELECT * FROM $table", + s"SELECT n AS k, CAST(n AS VARCHAR) AS s FROM $table LIMIT 5", + s"SELECT k, COUNT(*) AS c FROM $table GROUP BY k", + s"SELECT id, n + 1 AS v FROM $table LIMIT 5", + s"SELECT c_des AS d FROM $table LIMIT 5 UNION ALL SELECT ts AS d FROM $table LIMIT 5", + s"SELECT id FROM $table WHERE n IN (SELECT n FROM $table WHERE id = 't3') LIMIT 5" + ).foreach { sql => + withClue(s"[$sql] ") { + val described: ElasticResult[Option[ResultSchema]] = client.describe(sql) + described shouldBe ElasticSuccess(answer(runOk(sql))._2) + } + } + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala index bf6589c8b..8f7bb5a65 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala @@ -973,9 +973,9 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit } case _ => Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryRows(rs, _, _)) => Right(rs) case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => Left(s"unexpected result $other") case ElasticFailure(error) => Left(error.message) @@ -1081,6 +1081,9 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit private def utf8(s: String): Array[Byte] = s.getBytes(java.nio.charset.StandardCharsets.UTF_8) + /** A `binary` field's value as core reads it: the bytes its base64 encodes (SCHEMA-2, Q5). */ + private def decoded(base64: String): Array[Byte] = java.util.Base64.getDecoder.decode(base64) + private val routes = Seq(("run", false), ("search", false), ("search", true)) private def textUnion(items: Seq[String], limited: Boolean): String = @@ -1089,12 +1092,12 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit .mkString(" UNION ALL ") /** A `VARBINARY` beside text: DuckDB's `BLOB` (the lead's ruling of 2026-10-07). Every row is - * BYTES (`Array[Byte]`, core's `VARBINARY` cell): a `binary` field's base64 read as its UTF-8 - * bytes, as the relational engine and the JDBC driver read it, and a text value cast to bytes as - * DuckDB 1.5.5 casts a `VARCHAR` to a `BLOB` (MEASURED: an ASCII character is its byte). + * BYTES (`Array[Byte]`, core's `VARBINARY` cell): a `binary` field's base64 DECODED -- the bytes + * it was indexed with (SCHEMA-2, Q5) --, and a text value cast to bytes as DuckDB 1.5.5 casts a + * `VARCHAR` to a `BLOB` (MEASURED: an ASCII character is its byte). */ it should "answer a VARBINARY beside text as DuckDB's BLOB, on every route" in { - val bin: TextDoc => Any = d => binaryOf.get(d.id).map(utf8).orNull + val bin: TextDoc => Any = d => binaryOf.get(d.id).map(decoded).orNull val s: TextDoc => Any = d => d.s.map(utf8).orNull val x: TextDoc => Any = _ => utf8("x") val cases: Seq[Seq[(String, TextDoc => Any)]] = Seq( @@ -1181,7 +1184,7 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit val kw: TextDoc => Any = d => temporalKeywordOf.get(d.id).orNull val textOfN: Doc => Any = r => opt(r.n.map(_.toString)) val bytesOfKw: TextDoc => Any = d => temporalKeywordOf.get(d.id).map(utf8).orNull - val bin: TextDoc => Any = d => binaryOf.get(d.id).map(utf8).orNull + val bin: TextDoc => Any = d => binaryOf.get(d.id).map(decoded).orNull val wrong = routes.flatMap { case (route, limited) => val limit = if (limited) " LIMIT 100" else "" val asText = @@ -1496,8 +1499,8 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit ("TIMESTAMP", "t3") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), ("TIMESTAMP", "t4") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), ("TIMESTAMP", "t5") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), - ("VARBINARY", "t1") -> Map("VARBINARY" -> utf8("YWI="), "VARCHAR" -> "YWI="), - ("VARBINARY", "t2") -> Map("VARBINARY" -> utf8("eA=="), "VARCHAR" -> "eA=="), + ("VARBINARY", "t1") -> Map("VARBINARY" -> decoded("YWI="), "VARCHAR" -> "YWI="), + ("VARBINARY", "t2") -> Map("VARBINARY" -> decoded("eA=="), "VARCHAR" -> "eA=="), ("VARCHAR", "t1") -> Map("VARBINARY" -> utf8("x"), "VARCHAR" -> "x"), ("VARCHAR", "t2") -> Map("VARBINARY" -> utf8("y"), "VARCHAR" -> "y") ) @@ -1674,8 +1677,11 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit m.asInstanceOf[scala.collection.Map[String, Any]].getOrElse("a", null) case (_, other) => ("unexpected", other) } + // SCHEMA-2 (D5): a date of the default format too is read by it, as Elasticsearch indexed it + // -- the year 2024 written as a text and as a number is 2024-01-01 --, where 0.24.0 kept it as + // read (`"2024"`, `2024`) val wrong = for { - f <- formatted.filter(_.format.isDefined) + f <- formatted (depth, item, partner) <- Seq( ("column", s"c_${f.key}", if (f.declared == "DATE") "pd" else "pts"), ("field", s"f_${f.key}", if (f.declared == "DATE") "fd" else "fts") @@ -1714,39 +1720,23 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit ) } } yield error - // the control: dates of the default format keep their values as read -- the year 2024 written - // as a text and as a number -- at the column and at an object's field - val control = for { - (depth, item, partner) <- Seq(("column", "c_def", "pts"), ("field", "f_def", "fts")) - (route, limited) <- routes - sql = Seq(item, partner) - .map(i => s"SELECT $i AS k FROM $formatTable" + (if (limited) " LIMIT 100" else "")) - .mkString(" UNION ALL ") - error <- rowsOf(sql, route) match { - case Left(e) => Some(s"[$route] [$sql] failed: $e") - case Right(rows) => - val got = rows.map(r => shown(at(depth, r.getOrElse("k", null)))) - if (got.contains(("2024", "String")) && got.contains((2024, "Integer"))) None - else Some(s"[$route] [$sql] read ${got.mkString(", ")}") - } - } yield error - (wrong ++ control) shouldBe empty + wrong shouldBe empty } /** A date core cannot read without guessing -- a region id, whose offset comes from zone data -- - * and a value of several values -- an array of arrays -- where its branches AGREE are answered - * as stored, at the column and at an object's field, on every route, as they always were: no - * statement core answered fails. Where its branches disagree the date converts, and fails the - * statement, naming the value, the field and the format. + * and a value of several values -- an array of arrays -- are answered as stored, at the column + * and at an object's field, on every route: no statement core answered fails. SCHEMA-2 (D12): at + * the column, such a format makes the column VARCHAR, as DuckDB types a text beside a date + * -- the other branch's date in DuckDB's spelling --, beside a DATE too, which failed before. */ - it should "keep an agreeing date it cannot read as stored, and fail it where it converts" in { + it should "keep a date it cannot read as stored, its column VARCHAR at the top" in { def sql(left: String, right: String, limited: Boolean): String = Seq(left, right) .map(i => s"SELECT $i AS k FROM $zoneTable" + (if (limited) " LIMIT 100" else "")) .mkString(" UNION ALL ") val kept = for { (left, right, stored, partner) <- Seq( - ("z", "ts", zoned: Any, ZonedDateTime.parse("2024-02-01T00:00:00Z"): Any), + ("z", "ts", zoned: Any, "2024-02-01 00:00:00": Any), ( "fz", "fts", @@ -1783,18 +1773,18 @@ trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit wit if (got == want) None else Some(s"[$route] [$statement] answered $got instead of $want") } } yield error - val failed = for { + val besideDate = for { (route, limited) <- routes statement = sql("z", "d", limited) error <- rowsOf(statement, route) match { - case Left(msg) - if msg.contains(s"Could not read '$zoned' as a TIMESTAMP") && - msg.contains(s"the date format '$zoneFormat' of field z") => - None - case other => Some(s"[$route] [$statement] answered $other instead of failing") + case Right(rows) => + val got = rows.map(_.getOrElse("k", null)).sortBy(String.valueOf) + val want = Seq(zoned, null, null, "2024-02-01").sortBy(String.valueOf) + if (got == want) None else Some(s"[$route] [$statement] answered $got instead of $want") + case Left(e) => Some(s"[$route] [$statement] failed: $e") } } yield error - (kept ++ failed) shouldBe empty + (kept ++ besideDate) shouldBe empty } it should "refuse by name only a date beside a number, before anything runs" in { diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/TemporalLiteralSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/TemporalLiteralSpec.scala index a3c60c4b1..74cd23323 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/TemporalLiteralSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/TemporalLiteralSpec.scala @@ -173,9 +173,9 @@ trait TemporalLiteralSpec extends AnyFlatSpecLike with ElasticDockerTestKit with /** Gateway venue: `GatewayApi.run` -- what the JDBC driver, the REPL and the sidecar call. */ private def gatewayIds(sql: String): Set[String] = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => ids(rows) + case ElasticSuccess(QueryRows(rows, _, _)) => ids(rows) case ElasticSuccess(QueryStructured(response, _)) => ids(response.results) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => ids(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) case ElasticSuccess(other) => fail(s"Unexpected result: $other") case ElasticFailure(error) => fail(s"Query failed: ${error.message}\n$sql") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/TypesPredicatesFunctionsSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/TypesPredicatesFunctionsSpec.scala index 3e08e0154..2167a7125 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/TypesPredicatesFunctionsSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/TypesPredicatesFunctionsSpec.scala @@ -182,9 +182,9 @@ trait TypesPredicatesFunctionsSpec extends AnyFlatSpecLike with ElasticDockerTes /** The rows of `sql` through `GatewayApi.run`. */ private def gatewayRowsOf(sql: String): Seq[ListMap[String, Any]] = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(response, _)) => response.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case ElasticSuccess(other) => fail(s"Unexpected result: $other\n$sql") case ElasticFailure(error) => fail(s"Query failed: ${error.message}\n$sql") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala index adb0dcecd..a8327d347 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestFunctionRefusalExecutionSpec.scala @@ -169,9 +169,9 @@ trait UnnestFunctionRefusalExecutionSpec private def viaGateway(sql: String): Either[ElasticError, Seq[ListMap[String, Any]]] = { val answered: Try[Either[ElasticError, Seq[ListMap[String, Any]]]] = Try(Await.result(client.run(sql), 60.seconds)).flatMap { - case ElasticSuccess(QueryRows(rows, _)) => Success(Right(rows)) + case ElasticSuccess(QueryRows(rows, _, _)) => Success(Right(rows)) case ElasticSuccess(QueryStructured(response, _)) => Success(Right(response.results)) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Try(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)).map(Right(_)) case ElasticSuccess(other) => Success(Left(ElasticError(message = s"unexpected result $other"))) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestInnerHitsSizeExecutionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestInnerHitsSizeExecutionSpec.scala index 1bcac325d..bf8deba73 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestInnerHitsSizeExecutionSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/UnnestInnerHitsSizeExecutionSpec.scala @@ -139,9 +139,9 @@ trait UnnestInnerHitsSizeExecutionSpec private def viaGateway(sql: String): Either[ElasticError, Seq[ListMap[String, Any]]] = { val answered: Try[Either[ElasticError, Seq[ListMap[String, Any]]]] = Try(Await.result(client.run(sql), 60.seconds)).flatMap { - case ElasticSuccess(QueryRows(rows, _)) => Success(Right(rows)) + case ElasticSuccess(QueryRows(rows, _, _)) => Success(Right(rows)) case ElasticSuccess(QueryStructured(response, _)) => Success(Right(response.results)) - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Try(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)).map(Right(_)) case ElasticSuccess(other) => Success(Left(ElasticError(message = s"unexpected result $other"))) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala index 5228f0d6a..0bd9a256f 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala @@ -2143,9 +2143,9 @@ trait WindowFunctionSpec private def percentileGatewayRows(sql: String): Seq[ListMap[String, Any]] = Await.result(client.run(sql), 60.seconds) match { - case ElasticSuccess(QueryRows(rows, _)) => rows + case ElasticSuccess(QueryRows(rows, _, _)) => rows case ElasticSuccess(QueryStructured(response, _)) => response.results - case ElasticSuccess(QueryStream(stream, _)) => + case ElasticSuccess(QueryStream(stream, _, _)) => Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds) case ElasticSuccess(other) => fail(s"Unexpected result: $other\n$sql") case ElasticFailure(error) => fail(s"Query failed: ${error.message}\n$sql") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala index ae0ad69b2..618fbf862 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplGatewayIntegrationSpec.scala @@ -1672,7 +1672,7 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { val watcherStatus = executeSync("SHOW WATCHER STATUS my_watcher_interval") watcherStatus match { - case ExecutionSuccess(QueryRows(rows, _), _) => + case ExecutionSuccess(QueryRows(rows, _, _), _) => rows.size shouldBe 1 val row = rows.head log.info(s"Watcher Status: $row") @@ -1685,7 +1685,7 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { if (supportsQueryWatchers) { val watchers = executeSync("SHOW WATCHERS") watchers match { - case ExecutionSuccess(QueryRows(rows, _), _) => + case ExecutionSuccess(QueryRows(rows, _, _), _) => rows.find(row => row.get("id").contains("my_watcher_interval")) match { case Some(row) => row.get("is_healthy") shouldBe Some(true) @@ -1778,7 +1778,7 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { val executePolicy = "EXECUTE ENRICH POLICY my_policy" val result = executeSync(executePolicy) result match { - case ExecutionSuccess(QueryRows(rows, _), _) => + case ExecutionSuccess(QueryRows(rows, _, _), _) => rows.size shouldBe 1 val row = rows.head log.info(s"Policy Execution Result: $row") @@ -1903,11 +1903,11 @@ trait ReplGatewayIntegrationSpec extends ReplIntegrationTestKit { private def conversionRows(res: ExecutionResult): Seq[ListMap[String, Any]] = { res shouldBe a[ExecutionSuccess] res.asInstanceOf[ExecutionSuccess].result match { - case q: QueryRows => q.rows - case q: QueryStructured => q.response.results - case QueryStream(stream, _) => stream.map(_._1).runWith(Sink.seq).futureValue - case StreamResult(_, _) => testRepl.consumeStreamSync() - case other => fail(s"Unexpected result variant for a SELECT: $other") + case q: QueryRows => q.rows + case q: QueryStructured => q.response.results + case QueryStream(stream, _, _) => stream.map(_._1).runWith(Sink.seq).futureValue + case StreamResult(_, _) => testRepl.consumeStreamSync() + case other => fail(s"Unexpected result variant for a SELECT: $other") } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplIntegrationTestKit.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplIntegrationTestKit.scala index 966b0eb3a..82fd5c14c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplIntegrationTestKit.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/repl/ReplIntegrationTestKit.scala @@ -158,7 +158,7 @@ trait ReplIntegrationTestKit val success = res.asInstanceOf[ExecutionSuccess] success.result match { - case QueryStream(stream, _) => + case QueryStream(stream, _, _) => val sink = Sink.fold[Seq[Map[String, Any]], (Map[String, Any], ScrollMetrics)](Seq.empty) { case (acc, (row, _)) => acc :+ normalizeRow(row) } From bbb5eca643c680c87e0cfadc853c27f5a07a8ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 30 Oct 2026 17:16:25 +0100 Subject: [PATCH 2/3] fix(core): date formats built once per statement; integer reads follow Elasticsearch's range rules (SCHEMA-2) Story SCHEMA-2. Reviewed: r0:must-fix, r1:must-fix, r2:must-fix. Co-Authored-By: Claude Opus 5.5 --- .../elastic/client/DateRoute.scala | 36 +-- .../elastic/client/ElementConversions.scala | 13 +- .../elastic/client/HitReading.scala | 235 +++++++++++++----- .../elastic/client/ResultSchema.scala | 18 +- .../elastic/client/SetOperationValues.scala | 47 +++- .../softnetwork/elastic/client/package.scala | 6 +- .../elastic/client/HitReadingSpec.scala | 113 ++++++++- documentation/client/gateway.md | 9 +- 8 files changed, 384 insertions(+), 93 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/DateRoute.scala b/core/src/main/scala/app/softnetwork/elastic/client/DateRoute.scala index 1a401d338..f60c9f3c7 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/DateRoute.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/DateRoute.scala @@ -52,21 +52,27 @@ private[client] object DateRoute { dates: SetOperationValues.DateReading ): DateRoute = (sqlType, format) match { - case (SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time, Some(spec)) => - dates.dialect match { - case None => - AsStored( - s"the date format '$spec' is read as the Elasticsearch version does, " + - dates.elasticVersion.fold("which is not known")(v => s"and core reads dates of no $v") - ) - case Some(dialect) => - MappedDateFormat(spec, dialect, dates.clock).unread.fold[DateRoute](Typed) { why => - AsStored( - s"core cannot read every value of the date format '$spec' as ${dialect.label} " + - s"does: $why" - ) - } + case (SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time, Some(spec)) => dates.route(spec) + case _ => Typed + } + + /** The route of a date field of mapping format `spec`, from the format alone: decided ONCE per + * statement and format ([[SetOperationValues.DateReading.route]]), over the statement's one + * built format ([[SetOperationValues.DateReading.format]]). + */ + private[client] def ofFormat(spec: String, dates: SetOperationValues.DateReading): DateRoute = + dates.format(spec) match { + case None => + AsStored( + s"the date format '$spec' is read as the Elasticsearch version does, " + + dates.elasticVersion.fold("which is not known")(v => s"and core reads dates of no $v") + ) + case Some(mapped) => + mapped.unread.fold[DateRoute](Typed) { why => + AsStored( + s"core cannot read every value of the date format '$spec' as " + + s"${mapped.dialect.label} does: $why" + ) } - case _ => Typed } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElementConversions.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElementConversions.scala index b3e571d82..2a4e0840c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElementConversions.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElementConversions.scala @@ -71,13 +71,16 @@ private[client] object ElementConversions { * -- for a statement without one. `version` is the Elasticsearch version its dates are read as, * asked only when a date is read. */ - def of(single: SingleSearch, version: => Option[String]): Map[String, Any => Any] = { + def of(single: SingleSearch, version: => Option[String]): Map[String, Any => Any] = + of(single, DateReading(version)) + + /** The same, reading its dates by the statement's `dates` -- the one each of its date formats is + * built by, shared with its hit readers (SCHEMA-2). + */ + private[client] def of(single: SingleSearch, dates: DateReading): Map[String, Any => Any] = { val conversions = UnnestScope.elementConversions(single) if (conversions.isEmpty) Map.empty - else { - val dates = DateReading(version) - conversions.map(c => c.output -> converter(single, c, dates)).toMap - } + else conversions.map(c => c.output -> converter(single, c, dates)).toMap } private def converter( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala index 99d3ced51..adc91c404 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala @@ -25,6 +25,7 @@ import app.softnetwork.elastic.sql.query.{ UnnestScope } import app.softnetwork.elastic.sql.query.MultiSearch.SetOperationType +import app.softnetwork.elastic.sql.schema.Column import com.fasterxml.jackson.databind.JsonNode import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneOffset, ZonedDateTime} @@ -119,7 +120,29 @@ private[client] object HitReading { multiValued: MultiValued = Default, elements: Set[String] = Set.empty ): HitReading = { - val schema = ResultSchemas.of(single, dates) + // each item's and each mapped column's type, computed ONCE per statement: the schema and the + // readers read the same value; so do the date formats and routes, built once by `dates` + val itemTypes = new java.util.IdentityHashMap[Field, SetOperationType]() + val typeOf: Field => SetOperationType = f => { + val known = itemTypes.get(f) + if (known ne null) known + else { + val t = MultiSearch.fieldType(f) + itemTypes.put(f, t) + t + } + } + val columnTypes = new java.util.IdentityHashMap[Column, SetOperationType]() + val columnTypeOf: Column => SetOperationType = c => { + val known = columnTypes.get(c) + if (known ne null) known + else { + val t = MultiSearch.columnType(c) + columnTypes.put(c, t) + t + } + } + val schema = ResultSchemas.of(single, dates, typeOf, columnTypeOf) val table = single.sources.headOption.getOrElse("") val aliases = single.fieldAliases val readers = new Readers(aliases, dates, multiValued, table) @@ -153,7 +176,7 @@ private[client] object HitReading { if (star) table.columns.foreach { c => if (!source.containsKey(c.name)) - put(c.name, MultiSearch.columnType(c), keepUnder(c.name)) + put(c.name, columnTypeOf(c), keepUnder(c.name)) } } // explicit columns: a bare column (or a dotted column's parent and its flat spelling) by its @@ -164,7 +187,7 @@ private[client] object HitReading { val id = f.identifier if (id.functions.nonEmpty || f.isScriptField) { val name = f.outputName - val t = MultiSearch.fieldType(f) + val t = typeOf(f) if (name.nonEmpty && !scripted.containsKey(name)) { if (single.schema.isDefined) scripted.put(name, readers.scripted(name, t)) // no schema (D10): rows as today -- a column declared VARCHAR is its text, never re-read @@ -173,7 +196,7 @@ private[client] object HitReading { } } else if (!id.nested && id.name.nonEmpty && single.schema.isDefined) { val name = id.name - val t = MultiSearch.fieldType(f) + val t = typeOf(f) val dot = name.indexOf('.') if (dot < 0) { if (!source.containsKey(name)) put(name, t, keepUnder(name)) @@ -203,7 +226,7 @@ private[client] object HitReading { mappings.foreach { case (subField, output) => if (subField.indexOf('.') < 0 && !elements.contains(output)) fields.find(_.outputName == output).foreach { f => - val t = MultiSearch.fieldType(f) + val t = typeOf(f) if (readers.typed(t)) byField.put(subField, readers.column(output, t, Set.empty)._1) } @@ -490,12 +513,13 @@ private[client] object HitReading { ): (JsonNode => Any, Option[JsonNode => Any]) = sqlType match { case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int => - val (min, max) = sqlType match { - case SQLTypes.TinyInt => (Byte.MinValue.toLong, Byte.MaxValue.toLong) - case SQLTypes.SmallInt => (Short.MinValue.toLong, Short.MaxValue.toLong) - case _ => (Int.MinValue.toLong, Int.MaxValue.toLong) + val range = sqlType match { + case SQLTypes.TinyInt => WholeRange.ForTinyInt + case SQLTypes.SmallInt => WholeRange.ForSmallInt + case _ => WholeRange.ForInt } - val range = new WholeRange(min, max, truncatedFirst = false) + val min = range.min + val max = range.max ( node => if (node.isInt) { @@ -503,14 +527,14 @@ private[client] object HitReading { if (v >= min && v <= max) Integer.valueOf(v) else asRead(node) } else range.read(node) match { - case l: java.lang.Long => Integer.valueOf(l.intValue()) - case null => null - case _ => asRead(node) + case i: java.lang.Integer => i + case null => null + case _ => asRead(node) }, None ) case SQLTypes.BigInt => - val range = new WholeRange(Long.MinValue, Long.MaxValue, truncatedFirst = true) + val range = WholeRange.ForBigInt ( node => if (node.isInt || node.isLong) java.lang.Long.valueOf(node.longValue()) @@ -603,7 +627,9 @@ private[client] object HitReading { kind: app.softnetwork.elastic.sql.`type`.SQLType, spec: String ): JsonNode => Any = { - val format = MappedDateFormat(spec, dates.dialect.get, dates.clock) + // the statement's one format of `spec`, the one its route was decided on (`Typed`: the major + // is known) + val format = dates.format(spec).get val fast = spec == app.softnetwork.elastic.sql.query.TemporalLiterals.DefaultDateFormat val result: Instant => Any = kind match { case SQLTypes.Date => _.atZone(ZoneOffset.UTC).toLocalDate @@ -665,51 +691,144 @@ private[client] object HitReading { /** What [[WholeRange.read]] answers for a value that is no value of its type. */ private object OutOfType - /** SCHEMA-2 (D6) -- an integral type's values, `[min, max]`, read as Elasticsearch reads a value - * into that type: a whole number as it is, a decimal or a numeric text truncated toward zero - * (`coerce`, T row 5), a blank text NULL. A value outside the range is no value of the type -- - * Elasticsearch refused it (`ignore_malformed`) and holds it in `_source` only -- so `read` - * answers [[OutOfType]] and the column keeps it as stored (`Unreadable.KeepAsStored`): narrowing - * it would wrap it. The range is checked as Elasticsearch checks it: `long` on the truncated - * value (`truncatedFirst`, `Numbers.toLong`), `integer` / `short` / `byte` on the value itself. + /** SCHEMA-2 (D6) -- an integral type's values read as Elasticsearch reads a value into that type, + * `coerce` on (T row 5), the same on 6.8.23, 7.17.29 and 8.18.3 (`NumberFieldMapper.value` / + * `parse(XContentParser, coerce)`, `AbstractXContentParser`, Jackson's `getIntValue` / + * `getShortValue` / `getLongValue`): + * + * - an empty text is NULL (`textLength() == 0`); + * - `integer`, `short` and `byte` read a JSON number through `getIntValue` -- a whole number + * within `int`; a fractional one, as a `double`, within `int`, truncated toward zero --, + * then `short` and `byte` check their own range on that `int`; a text through + * `Double.parseDouble`, which trims and reads what it reads (`5f`, `NaN` as 0), `short` + * checking its range on that `double`, `integer` and `byte` the `int` range on it, then + * `byte` its own range on the truncated `int`; + * - `long` reads a JSON number through `getLongValue` -- a fractional one, as a `double`, + * within `long`, truncated --; a text through `Long.parseLong`, else `new BigDecimal` + * (neither trims), truncated toward zero and checked on the truncated value. + * + * A value Elasticsearch refuses is no value of the type -- it holds it in `_source` only + * (`ignore_malformed`) -- so `read` answers [[OutOfType]] and the column keeps it as stored + * (`Unreadable.KeepAsStored`): narrowing it would wrap it. So is a `long` text with more than 19 + * digits of scale either way where the majors disagree (7.17 and 8.18 refuse a scale below -19 + * and read 0 above 19; 6.8 reads the value). No digit count or scale is computed in `Int` + * arithmetic, and no value is materialised beyond the digits its text holds. */ - private[client] final class WholeRange(min: Long, max: Long, truncatedFirst: Boolean) { - - private[this] val low = java.math.BigDecimal.valueOf(min) - private[this] val high = java.math.BigDecimal.valueOf(max) - - /** A `java.lang.Long` within the range, `null` for a blank text, else [[OutOfType]]. */ - def read(node: JsonNode): Any = - if (node.isIntegralNumber) { - if (node.canConvertToLong) whole(node.longValue()) else OutOfType - } else if (node.isBigDecimal) decimal(node.decimalValue()) - else if (node.isNumber) { - val d = node.doubleValue() - if (java.lang.Double.isNaN(d) || java.lang.Double.isInfinite(d)) OutOfType - else decimal(java.math.BigDecimal.valueOf(d)) - } else if (node.isTextual) { - val text = node.asText().trim - if (text.isEmpty) null - else - try decimal(new java.math.BigDecimal(text)) - catch { case _: NumberFormatException => OutOfType } - } else OutOfType - - private def whole(value: Long): Any = - if (value >= min && value <= max) java.lang.Long.valueOf(value) else OutOfType - - private def decimal(d: java.math.BigDecimal): Any = { - // the digits before the point, read off the scale: `1e1000000000` is never materialised - val integerDigits = d.precision() - d.scale() - if (d.signum() == 0 || integerDigits <= 0) java.lang.Long.valueOf(0L) - else if (integerDigits > 19) OutOfType - else { - val truncated = d.setScale(0, java.math.RoundingMode.DOWN) - val ranged = if (truncatedFirst) truncated else d - if (ranged.compareTo(low) >= 0 && ranged.compareTo(high) <= 0) - java.lang.Long.valueOf(truncated.longValue()) + private[client] sealed abstract class WholeRange { + + /** A `java.lang.Integer` (`TINYINT`, `SMALLINT`, `INT`) or a `java.lang.Long` (`BIGINT`), + * `null` for an empty text, else [[OutOfType]]. + */ + def read(node: JsonNode): Any + } + + private[client] object WholeRange { + + /** `byte`: `parser.intValue(coerce)`, then the byte range on that `int`. */ + val ForTinyInt: Ints = + new Ints( + Byte.MinValue.toInt, + Byte.MaxValue.toInt, + Int.MinValue.toDouble, + Int.MaxValue.toDouble + ) + + /** `short`: a number through `getShortValue` (`getIntValue`, then the short range on that + * `int`); a text checked on its `double`. + */ + val ForSmallInt: Ints = new Ints( + Short.MinValue.toInt, + Short.MaxValue.toInt, + Short.MinValue.toDouble, + Short.MaxValue.toDouble + ) + + /** `integer`: the `int` range, on the `double` of a fractional number or a text. */ + val ForInt: Ints = + new Ints(Int.MinValue, Int.MaxValue, Int.MinValue.toDouble, Int.MaxValue.toDouble) + + /** The `int`-family reader: values `[min, max]`; a text's `double` checked within `[textLow, + * textHigh]` before it is truncated. + */ + private[client] final class Ints(val min: Int, val max: Int, textLow: Double, textHigh: Double) + extends WholeRange { + + def read(node: JsonNode): Any = + if (node.isIntegralNumber) { + if (node.canConvertToLong) { + val v = node.longValue() + if (v >= min && v <= max) java.lang.Integer.valueOf(v.toInt) else OutOfType + } else OutOfType + } else if (node.isNumber) truncated(node.doubleValue(), IntLow, IntHigh) + else if (node.isTextual) { + val text = node.asText() + if (text.isEmpty) null + else { + val d = + try java.lang.Double.valueOf(text) + catch { case _: NumberFormatException => null } + if (d eq null) OutOfType else truncated(d.doubleValue(), textLow, textHigh) + } + } else OutOfType + + /** `d` within `[low, high]`, truncated toward zero (`(int) d`: `NaN` is 0), then within the + * type's range. + */ + private def truncated(d: Double, low: Double, high: Double): Any = + if (d < low || d > high) OutOfType + else { + val i = d.toInt + if (i >= min && i <= max) java.lang.Integer.valueOf(i) else OutOfType + } + } + + // Jackson's MIN_INT_D / MAX_INT_D / MIN_LONG_D / MAX_LONG_D + private val IntLow: Double = Int.MinValue.toDouble + private val IntHigh: Double = Int.MaxValue.toDouble + private val LongLow: Double = Long.MinValue.toDouble + private val LongHigh: Double = Long.MaxValue.toDouble + private val LongMin = java.math.BigDecimal.valueOf(Long.MinValue) + private val LongMax = java.math.BigDecimal.valueOf(Long.MaxValue) + + /** `long`. */ + object ForBigInt extends WholeRange { + + def read(node: JsonNode): Any = + if (node.isIntegralNumber) { + if (node.canConvertToLong) java.lang.Long.valueOf(node.longValue()) else OutOfType + } else if (node.isNumber) { + val d = node.doubleValue() + if (d < LongLow || d > LongHigh) OutOfType else java.lang.Long.valueOf(d.toLong) + } else if (node.isTextual) text(node.asText()) else OutOfType - } + + private def text(text: String): Any = + if (text.isEmpty) null + else { + val whole = + try java.lang.Long.valueOf(text) + catch { case _: NumberFormatException => null } + if (whole ne null) whole + else { + val d = + try new java.math.BigDecimal(text) + catch { case _: NumberFormatException => null } + if (d eq null) OutOfType + else { + // the digits before the point, in Long arithmetic, read off the scale + val integerDigits = d.precision().toLong - d.scale() + if (d.scale() < -19) OutOfType + else if (d.signum() == 0 || integerDigits <= 0) java.lang.Long.valueOf(0L) + else if (d.scale() > 19 || integerDigits > 19) OutOfType + else { + val truncated = d.setScale(0, java.math.RoundingMode.DOWN) + if (truncated.compareTo(LongMin) >= 0 && truncated.compareTo(LongMax) <= 0) + java.lang.Long.valueOf(truncated.longValue()) + else OutOfType + } + } + } + } } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala b/core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala index 4857e68ed..0929a3bc2 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ResultSchema.scala @@ -17,8 +17,9 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} -import app.softnetwork.elastic.sql.query.{FromlessSelect, MultiSearch, SingleSearch} +import app.softnetwork.elastic.sql.query.{Field, FromlessSelect, MultiSearch, SingleSearch} import app.softnetwork.elastic.sql.query.MultiSearch.{SetOperationColumn, SetOperationType} +import app.softnetwork.elastic.sql.schema.Column /** ONE column of a statement's result, declared BEFORE any row is read. * @@ -67,11 +68,22 @@ private[client] object ResultSchemas { * `IndicesApi`, an es8/es9 alias, a mapping that could not be read). */ def of(single: SingleSearch, dates: SetOperationValues.DateReading): Option[ResultSchema] = + of(single, dates, MultiSearch.fieldType _, MultiSearch.columnType _) + + /** The same, over the types its caller computed once per statement for its own use (`typeOf` + * = `MultiSearch.fieldType`, `columnTypeOf` = `MultiSearch.columnType`): [[HitReading.of]]. + */ + private[client] def of( + single: SingleSearch, + dates: SetOperationValues.DateReading, + typeOf: Field => SetOperationType, + columnTypeOf: Column => SetOperationType + ): Option[ResultSchema] = single.schema.map { table => ResultSchema(single.select.fieldsWithComputedAliases.flatMap { field => if (HitReading.isStar(field)) - table.columns.map(c => columnOf(c.name, MultiSearch.columnType(c), dates)) - else Seq(columnOf(field.outputName, MultiSearch.fieldType(field), dates)) + table.columns.map(c => columnOf(c.name, columnTypeOf(c), dates)) + else Seq(columnOf(field.outputName, typeOf(field), dates)) }) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala b/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala index 28dcbd491..50dc9c0d3 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala @@ -430,6 +430,47 @@ private[elastic] object SetOperationValues { lazy val elasticVersion: Option[String] = version lazy val dialect: Option[MappedDateFormat.Dialect] = elasticVersion.flatMap(MappedDateFormat.Dialect.of) + + // SCHEMA-2 (AC5) -- each mapping format the statement reads, and its route, built ONCE per + // statement (its major and clock are this reading's): the result schema, the hit readers, the + // UNNEST element conversions and the set-operation converters read the same value (#292) + private[this] var formats: java.util.HashMap[String, MappedDateFormat] = _ + private[this] var routes: java.util.HashMap[String, DateRoute] = _ + + /** The mapping format `spec` as this statement's major reads it, built once per statement; + * `None` where the major is not known. + */ + def format(spec: String): Option[MappedDateFormat] = + dialect.map { major => + synchronized { + if (formats eq null) formats = new java.util.HashMap[String, MappedDateFormat]() + val known = formats.get(spec) + if (known ne null) known + else { + val built = MappedDateFormat(spec, major, clock) + formats.put(spec, built) + built + } + } + } + + /** How a date field of mapping format `spec` is read ([[DateRoute.ofFormat]]), decided once per + * statement. + */ + private[client] def route(spec: String): DateRoute = + synchronized { + if (routes eq null) routes = new java.util.HashMap[String, DateRoute]() + val known = routes.get(spec) + if (known ne null) known + else { + val decided = DateRoute.ofFormat(spec, this) + routes.put(spec, decided) + decided + } + } + + /** How many mapping formats this statement has built (one per format it reads). */ + private[client] def formatCount: Int = synchronized(if (formats eq null) 0 else formats.size) } private[client] object DateReading { @@ -1219,14 +1260,14 @@ private[elastic] object SetOperationValues { s"Conversion Error: Could not read '$value' as a $typeName when casting from source " + s"column $column: $detail" ) - dates.dialect match { + dates.format(spec) match { case None => val detail = s"the date format '$spec' of field $field reads it as the Elasticsearch version does, " + dates.elasticVersion.fold("which is not known")(v => s"and core reads dates of no $v") value => failure(value, detail) - case Some(dialect) => - val format = MappedDateFormat(spec, dialect, dates.clock) + case Some(format) => + val dialect = format.dialect // SCHEMA-2 (D12) -- a value already in the result class's ONE form (a canonical row, // read by its column's `HitReading`) is passed through: re-reading a `LocalDate` as // `epoch_second` would fail it. That form is exactly what the reader gives -- a `DATE` a diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index 1b1b4ad5c..1a24aaa1e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -642,11 +642,13 @@ package object client extends SerializationApi { ): ConversionContext = context match { case NativeContext => - val elements = ElementConversions.of(single, version) + // ONE date reading per statement: each date format built once, for every reader + val dates = SetOperationValues.DateReading(version) + val elements = ElementConversions.of(single, dates) SchemaColumns( HitReading.of( single, - SetOperationValues.DateReading(version), + dates, HitReading.Default, elements.keySet ), diff --git a/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala index 2f77f4488..f9cea1c71 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala @@ -297,12 +297,12 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { row("SELECT s FROM s2_t", """{"s": 40000}""")("s") shouldBe java.lang.Integer.valueOf(40000) row("SELECT tb FROM s2_t", """{"tb": "200"}""")("tb") shouldBe "200" row("SELECT n FROM s2_t", """{"n": "1e1000000000"}""")("n") shouldBe "1e1000000000" - // ...the bounds themselves are values, checked as Elasticsearch checks them: `byte`, `short` - // and `integer` on the value, `long` on the truncated value + // ...the bounds themselves are values, checked as Elasticsearch checks them (integral coercion + // below): `byte` on the int-truncated value, `long` on the truncated value row("SELECT n FROM s2_t", """{"n": -2147483648}""")("n") shouldBe java.lang.Integer.valueOf(Int.MinValue) row("SELECT tb FROM s2_t", """{"tb": "-128"}""")("tb") shouldBe java.lang.Integer.valueOf(-128) - row("SELECT tb FROM s2_t", """{"tb": "127.9"}""")("tb") shouldBe "127.9" + row("SELECT tb FROM s2_t", """{"tb": "127.9"}""")("tb") shouldBe java.lang.Integer.valueOf(127) row("SELECT l FROM s2_t", """{"l": "9223372036854775807.5"}""")("l") shouldBe java.lang.Long.valueOf(Long.MaxValue) row("SELECT l FROM s2_t", """{"l": "0.5e-1000000000"}""")("l") shouldBe java.lang.Long.valueOf( @@ -310,6 +310,86 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { ) } + // integral coercion: NumberFieldMapper / AbstractXContentParser 6.8.23, 7.17.29 and 8.18.3, + // Jackson getIntValue / getShortValue / getLongValue; a value Elasticsearch refuses is kept as + // stored (D6) + private def read(column: String, json: String): Any = + row(s"SELECT $column FROM s2_t", s"""{"$column": $json}""")(column) + + private def int(i: Int): java.lang.Integer = java.lang.Integer.valueOf(i) + + private val integrals = Seq("tb", "s", "n", "l") + + it should "keep an exponent near 2^31 as stored in every integral type (integral exponent)" in { + // `integer` / `short` / `byte` read Infinity (Double.parseDouble), `long` refuses a scale below + // -19: no value of the type, never 0 + Seq("1e2147483647", "-1e2147483647", "123e2147483646").foreach { text => + integrals.foreach { c => + withClue(s"$c '$text': ")(read(c, s""""$text"""") shouldBe text) + } + } + } + + it should "check TINYINT and SMALLINT ranges as Elasticsearch does (integral range)" in { + // byte: `parser.intValue(coerce)` -- a number truncated through Jackson's getIntValue, a text + // through Double.parseDouble -- then the byte range on that int + read("tb", "127.9") shouldBe int(127) + read("tb", "-128.5") shouldBe int(-128) + read("tb", "\"127.9\"") shouldBe int(127) + read("tb", "\"-128.5\"") shouldBe int(-128) + read("tb", "128.5") shouldBe java.lang.Double.valueOf(128.5) + read("tb", "-129.5") shouldBe java.lang.Double.valueOf(-129.5) + read("tb", "\"128.5\"") shouldBe "128.5" + read("tb", "\"-129.5\"") shouldBe "-129.5" + // short: a JSON number through getShortValue (getIntValue, then the short range on that int); + // a text checked on its double (`shortValue`'s string branch) + read("s", "32767.5") shouldBe int(32767) + read("s", "-32768.5") shouldBe int(-32768) + read("s", "32768.5") shouldBe java.lang.Double.valueOf(32768.5) + read("s", "-32769.5") shouldBe java.lang.Double.valueOf(-32769.5) + read("s", "\"32767\"") shouldBe int(32767) + read("s", "\"32767.5\"") shouldBe "32767.5" + read("s", "\"-32768.5\"") shouldBe "-32768.5" + // integer: the int range on the double, either node kind + read("n", "2147483646.5") shouldBe int(2147483646) + read("n", "\"2147483646.5\"") shouldBe int(2147483646) + read("n", "\"2147483647.5\"") shouldBe "2147483647.5" + } + + it should "trim a text only where Elasticsearch does (integral whitespace)" in { + // Double.parseDouble trims, so `integer`, `short` and `byte` read ' 5 '; `long`'s + // Long.parseLong and BigDecimal do not, so Elasticsearch refuses it there; a blank text is + // refused by every integral type; an empty text is NULL (`textLength() == 0`) + Seq("tb", "s", "n").foreach(c => withClue(c)(read(c, "\" 5 \"") shouldBe int(5))) + read("l", "\" 5 \"") shouldBe " 5 " + integrals.foreach { c => + withClue(c) { + read(c, "\" \"") shouldBe " " + read(c, "\"\"") shouldBe (null: Any) + } + } + } + + it should "read a text and a fractional number by Elasticsearch's own parse (integral parse)" in { + // Double.parseDouble reads what it reads: a float suffix, and NaN as 0 (`(int) NaN`) + Seq("tb", "s", "n").foreach { c => + withClue(c) { + read(c, "\"5f\"") shouldBe int(5) + read(c, "\"NaN\"") shouldBe int(0) + read(c, "\"Infinity\"") shouldBe "Infinity" + } + } + read("l", "\"NaN\"") shouldBe "NaN" + // long: a JSON fractional number through getLongValue -- the double, truncated; a text with + // more than 19 digits of scale either way, where 6.8 and 7.17 / 8.18 disagree, as stored + read("l", "1.2345678901234567E18") shouldBe + java.lang.Long.valueOf(1.2345678901234567e18.toLong) + read("l", "-7.9") shouldBe java.lang.Long.valueOf(-7L) + read("l", "\"0e20\"") shouldBe "0e20" + read("l", "\"5.000000000000000000001\"") shouldBe "5.000000000000000000001" + read("l", "\"0.000000000000000000001\"") shouldBe java.lang.Long.valueOf(0L) + } + it should "answer a page the same rows and classes on every parse route (pages)" in { val fixture = SchemaPageFixture val outputs = fixture.statement.select.fieldsWithComputedAliases.map(_.outputName) @@ -401,6 +481,33 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { HitReading.DefaultDates.read("2024-02-30T10:00:00") shouldBe null } + "A statement's date fields" should "share ONE built format and route per mapping format (AC5)" in { + val dates = DateReading(Some("8.18.3")) + // ts, dm and obj.d: Elasticsearch's default format; c_c1 `yyyy/MM/dd`; c_des `epoch_second` + val reading = + HitReading.of(statement("SELECT ts, dm, obj, c_c1, c_des, n FROM s2_t"), dates) + reading.schema.map(_.names) shouldBe Some(Seq("ts", "dm", "obj", "c_c1", "c_des", "n")) + dates.formatCount shouldBe 3 + val default = app.softnetwork.elastic.sql.query.TemporalLiterals.DefaultDateFormat + (dates.format(default).get eq dates.format(default).get) shouldBe true + (dates.route("yyyy/MM/dd") eq dates.route("yyyy/MM/dd")) shouldBe true + dates.formatCount shouldBe 3 + // ...and the values read through it are the ones the date cells above pin + val r = row( + "SELECT ts, c_c1, c_des FROM s2_t", + """{"ts": "2024-01-31T10:30:00", "c_c1": "2024/01/31", "c_des": 1706697000}""" + ) + r.values.toSeq shouldBe Seq( + utc(2024, 1, 31, 10, 30), + utc(2024, 1, 31), + LocalDate.of(2024, 1, 31) + ) + // an unknown major builds none, and routes every date field as stored + val unknown = DateReading(None) + DateRoute.of(SQLTypes.Timestamp, Some(default), unknown) shouldBe a[DateRoute.AsStored] + unknown.formatCount shouldBe 0 + } + "The strict projection" should "hand an in-order row back as the same instance (identity)" in { val single = statement("SELECT id, n FROM s2_t") implicit val context: ConversionContext = contextOf(single) diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md index ecefa20d0..f0ca70c48 100644 --- a/documentation/client/gateway.md +++ b/documentation/client/gateway.md @@ -234,10 +234,11 @@ Each value is read by its column's declared type, where it is first read: A one-element array is its element, an empty array NULL. A date field whose format core cannot read for sure (a zone name, a region id) is a VARCHAR, its text as stored; a date value Elasticsearch -ignored (`ignore_malformed`) is NULL, and the other values of the same array keep theirs. A value a -typed column cannot read is answered as stored: a number outside its type's range, which -Elasticsearch did not index, is never narrowed. Aggregation values and window-enriched statements -keep their reading in this release. +ignored (`ignore_malformed`) is NULL, and the other values of the same array keep theirs. An +integral column reads a fractional number or a numeric text as Elasticsearch indexed it, truncated +toward zero. A value a typed column cannot read is answered as stored: a value Elasticsearch did not +index (a number outside its type's range, a `long` text with surrounding spaces) is never narrowed. +Aggregation values and window-enriched statements keep their reading in this release. --- From 1000e1489dce6bc97f30c8548f4275fe17d0ba2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 30 Oct 2026 17:39:49 +0100 Subject: [PATCH 3/3] fix(core): a long text Elasticsearch 6.8 refuses stays as stored (SCHEMA-2) Story SCHEMA-2. Reviewed: r0:must-fix, r1:must-fix, r2:must-fix, r3:clean. Co-Authored-By: Claude Opus 5.5 --- .../elastic/client/HitReading.scala | 22 +++++++++++++---- .../elastic/client/HitReadingSpec.scala | 24 ++++++++++++++++--- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala index adc91c404..0bcd8b8ef 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/HitReading.scala @@ -709,10 +709,14 @@ private[client] object HitReading { * * A value Elasticsearch refuses is no value of the type -- it holds it in `_source` only * (`ignore_malformed`) -- so `read` answers [[OutOfType]] and the column keeps it as stored - * (`Unreadable.KeepAsStored`): narrowing it would wrap it. So is a `long` text with more than 19 - * digits of scale either way where the majors disagree (7.17 and 8.18 refuse a scale below -19 - * and read 0 above 19; 6.8 reads the value). No digit count or scale is computed in `Int` - * arithmetic, and no value is materialised beyond the digits its text holds. + * (`Unreadable.KeepAsStored`): narrowing it would wrap it. So is a `long` text where the majors + * disagree: 7.17 and 8.18 refuse a scale below -19 and read 0 above 19, where 6.8 reads the + * truncated value -- except a non-zero value with a scale of 2^29 + 8 (536,870,920) or more, + * which 6.8 refuses: its `toBigInteger` divides by `BigInteger.TEN.pow(scale)`, which reports an + * overflow from that scale on (`bitLength(10) * scale / 32 > 2^26` on its JDK 15.0.1), and the + * `ArithmeticException` is a refusal. A value below 1 with a smaller scale, and a zero at any + * scale, are 0 on every major. No digit count or scale is computed in `Int` arithmetic, and no + * value is materialised beyond the digits its text holds. */ private[client] sealed abstract class WholeRange { @@ -790,6 +794,11 @@ private[client] object HitReading { private val LongMin = java.math.BigDecimal.valueOf(Long.MinValue) private val LongMax = java.math.BigDecimal.valueOf(Long.MaxValue) + /** The smallest scale 6.8.23 refuses in a non-zero `long` text: `BigInteger.TEN.pow(scale)` + * overflows once `4L * scale / 32 > (1 << 26)` (JDK 15.0.1's `pow`). + */ + private val Es6RefusedScale: Int = (1 << 29) + 8 + /** `long`. */ object ForBigInt extends WholeRange { @@ -818,7 +827,10 @@ private[client] object HitReading { // the digits before the point, in Long arithmetic, read off the scale val integerDigits = d.precision().toLong - d.scale() if (d.scale() < -19) OutOfType - else if (d.signum() == 0 || integerDigits <= 0) java.lang.Long.valueOf(0L) + else if (d.signum() == 0) java.lang.Long.valueOf(0L) + // 6.8 refuses it, 7.17 / 8.18 read 0: the majors disagree + else if (d.scale() >= Es6RefusedScale) OutOfType + else if (integerDigits <= 0) java.lang.Long.valueOf(0L) else if (d.scale() > 19 || integerDigits > 19) OutOfType else { val truncated = d.setScale(0, java.math.RoundingMode.DOWN) diff --git a/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala index f9cea1c71..7fb2bf0c9 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/HitReadingSpec.scala @@ -305,9 +305,8 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { row("SELECT tb FROM s2_t", """{"tb": "127.9"}""")("tb") shouldBe java.lang.Integer.valueOf(127) row("SELECT l FROM s2_t", """{"l": "9223372036854775807.5"}""")("l") shouldBe java.lang.Long.valueOf(Long.MaxValue) - row("SELECT l FROM s2_t", """{"l": "0.5e-1000000000"}""")("l") shouldBe java.lang.Long.valueOf( - 0L - ) + // ...and a `long` text below 1 whose scale 6.8 refuses (integral scale below) stays as stored + row("SELECT l FROM s2_t", """{"l": "0.5e-1000000000"}""")("l") shouldBe "0.5e-1000000000" } // integral coercion: NumberFieldMapper / AbstractXContentParser 6.8.23, 7.17.29 and 8.18.3, @@ -330,6 +329,25 @@ class HitReadingSpec extends AnyFlatSpec with Matchers { } } + it should "keep a long text below 1 as stored from the scale 6.8 refuses (integral scale)" in { + // 6.8.23's `toBigInteger` divides by BigInteger.TEN.pow(scale), which overflows from a scale of + // 2^29 + 8 on (refused, `_ignored`); 7.17.29 / 8.18.3 read 0 above a scale of 19: the majors + // disagree, so the value stays as stored on both + val refused = + Seq("1e-700000000", "-1e-700000000", "1e-2147483647", "0.5e-536870919", "5e-536870920") + refused.map(t => t -> read("l", s""""$t"""")) shouldBe refused.map(t => t -> t) + // below that scale, and a zero at any scale, every major reads 0 + Seq("1e-536870919", "1e-100000", "5e-20", "0e-700000000", "0.0e-2147483646").foreach { t => + withClue(t)(read("l", s""""$t"""") shouldBe java.lang.Long.valueOf(0L)) + } + // `integer` / `short` / `byte` read the text through Double.parseDouble (0) on every major + Seq("tb", "s", "n").foreach { c => + Seq("1e-700000000", "1e-2147483647").foreach { t => + withClue(s"$c '$t': ")(read(c, s""""$t"""") shouldBe int(0)) + } + } + } + it should "check TINYINT and SMALLINT ranges as Elasticsearch does (integral range)" in { // byte: `parser.intValue(coerce)` -- a number truncated through Jackson's getIntValue, a text // through Double.parseDouble -- then the byte range on that int