diff --git a/README.md b/README.md index 8a21d22..0703d2c 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,33 @@ const statements = parseStatements(` console.log(statements.length); // 3 ``` +### Format SQL + +```typescript +import { format } from "@questdb/sql-parser/formatter"; + +format("SELECT * FROM trades WHERE symbol = 'BTC-USD' LATEST ON ts PARTITION BY symbol"); +// SELECT * +// FROM trades +// WHERE symbol = 'BTC-USD' +// LATEST ON ts PARTITION BY symbol +``` + +The formatter is token-based and never throws on SQL content. It preserves every token, comment, and unknown character, changes whitespace only, and leaves unterminated or unbalanced input verbatim from the point of the problem. + +Options: `indent` (default two spaces), `maxWidth` (default 50), and `capitalize` (default `false`). + +```typescript +format( + "create table tab (s symbol index type bitmap, timestamp timestamp) timestamp(timestamp) partition by day wal", + { capitalize: true }, +); +// CREATE TABLE tab (s SYMBOL INDEX TYPE BITMAP, timestamp TIMESTAMP) +// TIMESTAMP(timestamp) PARTITION BY DAY WAL +``` + +With `capitalize`, the formatter parses the statement and raises only the words the grammar read as syntax, so a keyword that names a table, view or column keeps its case, as `timestamp` does above. Most QuestDB keywords are non-reserved and double as names, which is why this needs the grammar. SQL the parser cannot read comes back with its case untouched, and the layout is the same either way. + ### Autocomplete ```typescript @@ -183,6 +210,7 @@ The parser uses Chevrotain's [CST pattern](https://chevrotain.io/docs/guide/conc | `parseOne(sql)` | Parse a single statement, throws on errors or multiple statements | | `parseStatements(sql)` | Parse multiple statements, throws on errors | | `toSql(ast)` | Convert `Statement[]` back to a SQL string | +| `format(sql, options)` | Format SQL text; tolerant of invalid input, whitespace-only edits | ### Low-Level API diff --git a/package.json b/package.json index ec8a1f5..ba6afe0 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,16 @@ "types": "./dist/grammar/index.d.ts", "default": "./dist/grammar/index.cjs" } + }, + "./formatter": { + "import": { + "types": "./dist/formatter/index.d.ts", + "default": "./dist/formatter/index.js" + }, + "require": { + "types": "./dist/formatter/index.d.ts", + "default": "./dist/formatter/index.cjs" + } } }, "packageManager": "yarn@4.9.1", @@ -46,6 +56,7 @@ "questdb", "sql", "parser", + "formatter", "chevrotain", "ast" ], diff --git a/src/formatter/classification.ts b/src/formatter/classification.ts new file mode 100644 index 0000000..379d242 --- /dev/null +++ b/src/formatter/classification.ts @@ -0,0 +1,81 @@ +import { IdentifierKeyword, keywordTokenArray } from "../parser/tokens" + +export type SignificantKind = "word" | "operator" | "delimiter" + +export const operatorTokenNames: ReadonlySet = new Set([ + "Star", + "Plus", + "Minus", + "Divide", + "Modulo", + "Equals", + "NotEquals", + "LessThan", + "GreaterThan", + "LessThanOrEqual", + "GreaterThanOrEqual", + "IPv4ContainedBy", + "IPv4ContainedByOrEqual", + "IPv4Contains", + "IPv4ContainsOrEqual", + "Concat", + "BitAnd", + "BitXor", + "BitOr", + "RegexMatch", + "RegexNotMatch", + "RegexNotEquals", + "DoubleColon", + "ColonEquals", + "Colon", + "AtSign", +]) + +export const delimiterTokenNames: ReadonlySet = new Set([ + "LParen", + "RParen", + "LBracket", + "RBracket", + "Comma", + "Semicolon", + "Dot", +]) + +const literalTokenNames = [ + "GeohashBinaryLiteral", + "GeohashLiteral", + "LongLiteral", + "DecimalLiteral", + "DurationLiteral", + "NumberLiteral", + "StringLiteral", + "QuotedIdentifier", + "Identifier", + "VariableReference", +] + +export const wordTokenNames: ReadonlySet = new Set([ + ...keywordTokenArray.map((token) => token.name), + ...literalTokenNames, +]) + +const nonReservedKeywordNames = keywordTokenArray + .filter((token) => token.CATEGORIES?.includes(IdentifierKeyword)) + .map((token) => token.name) + +export const operandTokenNames: ReadonlySet = new Set([ + ...literalTokenNames, + ...nonReservedKeywordNames, + "Null", + "True", + "False", + "NaN", + "RParen", + "RBracket", +]) + +export const classify = (tokenName: string): SignificantKind => { + if (operatorTokenNames.has(tokenName)) return "operator" + if (delimiterTokenNames.has(tokenName)) return "delimiter" + return "word" +} diff --git a/src/formatter/context.ts b/src/formatter/context.ts new file mode 100644 index 0000000..301d3b8 --- /dev/null +++ b/src/formatter/context.ts @@ -0,0 +1,50 @@ +import { StreamToken } from "./lexer" + +export type StatementKind = + | "select" + | "insert" + | "update" + | "createTable" + | "createMaterializedView" + | "createLiveView" + | "alterTable" + | "alterMaterializedView" + | "other" + +const selectStarters: ReadonlySet = new Set([ + "Select", + "With", + "Declare", + "Identifier", + "QuotedIdentifier", + "LParen", +]) + +export const isSignificant = (token: StreamToken) => + token.kind === "word" || + token.kind === "operator" || + token.kind === "delimiter" + +export const isExplainPrefix = (token: StreamToken) => + token.tokenName === "Explain" + +export const detectStatementKind = (tokens: StreamToken[]): StatementKind => { + const names = tokens.filter(isSignificant).map((token) => token.tokenName) + if (names[0] === "Explain") names.shift() + const [first, second] = names + + if (first === undefined || first === null) return "other" + if (selectStarters.has(first)) return "select" + if (first === "Insert") return "insert" + if (first === "Update") return "update" + if (first === "Create") { + if (second === "Table") return "createTable" + if (second === "Materialized") return "createMaterializedView" + if (second === "Live") return "createLiveView" + } + if (first === "Alter") { + if (second === "Table") return "alterTable" + if (second === "Materialized") return "alterMaterializedView" + } + return "other" +} diff --git a/src/formatter/doc.ts b/src/formatter/doc.ts new file mode 100644 index 0000000..10a9247 --- /dev/null +++ b/src/formatter/doc.ts @@ -0,0 +1,168 @@ +export type Doc = + | { type: "text"; text: string } + | { type: "gap"; text: string } + | { type: "line"; flat: string } + | { type: "hardline" } + | { type: "concat"; parts: Doc[] } + | { type: "group"; doc: Doc } + | { type: "indent"; doc: Doc } + | { type: "verbatim"; text: string } + +export const text = (value: string): Doc => ({ type: "text", text: value }) +export const gap = (value: string): Doc => ({ type: "gap", text: value }) +export const line = (flat: string): Doc => ({ type: "line", flat }) +export const hardline: Doc = { type: "hardline" } +export const concat = (parts: Doc[]): Doc => ({ type: "concat", parts }) +export const group = (doc: Doc): Doc => ({ type: "group", doc }) +export const indent = (doc: Doc): Doc => ({ type: "indent", doc }) +export const verbatim = (value: string): Doc => ({ + type: "verbatim", + text: value, +}) + +export const asLine = (leading: Doc): Doc => + leading.type === "gap" ? line(leading.text) : leading + +export type PrintOptions = { + indent: string + maxWidth: number +} + +type Mode = "flat" | "break" + +type Command = { level: number; mode: Mode; doc: Doc } + +const firstLineLength = (value: string) => { + const newline = value.indexOf("\n") + return newline === -1 ? value.length : newline +} + +const fits = (next: Command, rest: Command[], width: number): boolean => { + let remaining = width + let restIndex = rest.length - 1 + const stack: Command[] = [next] + + while (remaining >= 0) { + const command = stack.pop() + if (command === undefined) { + if (restIndex < 0) return true + stack.push(rest[restIndex--]) + continue + } + const { level, mode, doc } = command + switch (doc.type) { + case "text": + case "gap": + remaining -= doc.text.length + break + case "line": + if (mode === "break") return true + remaining -= doc.flat.length + break + case "hardline": + return mode === "break" + case "verbatim": + remaining -= firstLineLength(doc.text) + if (doc.text.includes("\n")) return remaining >= 0 + break + case "concat": + for (let i = doc.parts.length - 1; i >= 0; i--) { + stack.push({ level, mode, doc: doc.parts[i] }) + } + break + case "group": + case "indent": + stack.push({ level, mode, doc: doc.doc }) + break + } + } + return false +} + +export const printDoc = (doc: Doc, options: PrintOptions): string => { + const out: string[] = [] + let column = 0 + let pendingGap = "" + let lineHasContent = false + let indentIndex = -1 + const commands: Command[] = [{ level: 0, mode: "break", doc }] + + const emit = (value: string) => { + if (pendingGap !== "") { + out.push(pendingGap) + column += pendingGap.length + pendingGap = "" + } + out.push(value) + lineHasContent = true + } + + const newline = (level: number) => { + const indentation = options.indent.repeat(level) + pendingGap = "" + if (!lineHasContent) { + if (indentIndex >= 0) out[indentIndex] = indentation + column = indentation.length + return + } + out.push("\n") + indentIndex = out.push(indentation) - 1 + column = indentation.length + lineHasContent = false + } + + while (commands.length > 0) { + const command = commands.pop()! + const { level, mode, doc: current } = command + switch (current.type) { + case "text": + emit(current.text) + column += current.text.length + break + case "gap": + if (lineHasContent) pendingGap = current.text + break + case "line": + if (mode === "flat") pendingGap = current.flat + else newline(level) + break + case "hardline": + newline(level) + break + case "verbatim": { + emit(current.text) + const lastNewline = current.text.lastIndexOf("\n") + column = + lastNewline === -1 + ? column + current.text.length + : current.text.length - lastNewline - 1 + break + } + case "concat": + for (let i = current.parts.length - 1; i >= 0; i--) { + commands.push({ level, mode, doc: current.parts[i] }) + } + break + case "indent": + commands.push({ level: level + 1, mode, doc: current.doc }) + break + case "group": { + if (mode === "flat") { + commands.push({ level, mode: "flat", doc: current.doc }) + break + } + const flat: Command = { level, mode: "flat", doc: current.doc } + const width = options.maxWidth - column - pendingGap.length + commands.push( + fits(flat, commands, width) + ? flat + : { level, mode: "break", doc: current.doc }, + ) + break + } + } + } + + if (!lineHasContent && indentIndex >= 0) out.length = indentIndex - 1 + return out.join("") +} diff --git a/src/formatter/index.ts b/src/formatter/index.ts new file mode 100644 index 0000000..ee6afa7 --- /dev/null +++ b/src/formatter/index.ts @@ -0,0 +1,44 @@ +import { PrintOptions } from "./doc" +import { formatStatement } from "./layout" +import { scan } from "./lexer" +import { splitStatements } from "./statements" +import { syntaxOffsets } from "./syntax" + +export type FormatOptions = { + indent?: string + maxWidth?: number + /** + * Uppercase the words the parser reads as syntax, leaving the names of + * tables, views and columns as written. Off by default, and a no-op for SQL + * the parser cannot read. + */ + capitalize?: boolean +} + +const DEFAULT_OPTIONS: PrintOptions = { indent: " ", maxWidth: 50 } + +const NOTHING_TO_RAISE: ReadonlySet = new Set() + +const resolveOptions = (options: FormatOptions): PrintOptions => { + const indent = options.indent ?? DEFAULT_OPTIONS.indent + if (typeof indent !== "string" || !/^[ \t]*$/.test(indent)) { + throw new TypeError("indent must be a string of spaces and tabs") + } + const maxWidth = options.maxWidth ?? DEFAULT_OPTIONS.maxWidth + if ( + typeof maxWidth !== "number" || + !Number.isFinite(maxWidth) || + maxWidth <= 0 + ) { + throw new TypeError("maxWidth must be a finite number greater than 0") + } + return { indent, maxWidth } +} + +export const format = (sql: string, options: FormatOptions = {}): string => { + const print = resolveOptions(options) + const uppercase = options.capitalize ? syntaxOffsets(sql) : NOTHING_TO_RAISE + return splitStatements(scan(sql)) + .map((statement) => formatStatement(statement, print, uppercase)) + .join("\n\n") +} diff --git a/src/formatter/layout.ts b/src/formatter/layout.ts new file mode 100644 index 0000000..ea3b43c --- /dev/null +++ b/src/formatter/layout.ts @@ -0,0 +1,630 @@ +import { detectStatementKind, isExplainPrefix, StatementKind } from "./context" +import { + asLine, + concat, + Doc, + gap, + group, + hardline, + indent, + printDoc, + PrintOptions, + text, + verbatim, +} from "./doc" +import { reconstruct, StreamToken } from "./lexer" +import { + clausePhrases, + continuesPhrase, + joinSubClauses, + matchPhrase, + Phrase, + PhraseMatch, + pivotSubClauses, + windowSubClauses, +} from "./phrases" +import { gapBetween, isOperand, isSign, Piece } from "./spacing" +import { Statement } from "./statements" + +type Element = { leading: Doc; doc: Doc } + +type Separator = { + kind: "comma" | "logical" | "subClause" + leading: Doc + doc: Doc +} + +type Sequence = { items: Element[]; separators: Separator[] } + +type GroupExpansion = "none" | "first" | "all" + +type SequenceContext = { + phrase: Phrase | null + clauseLevel: boolean + insideParens: boolean + logical: boolean + subClauses: Phrase[] + commas: boolean + expandableGroups: GroupExpansion + groupsSeen: number +} + +const groupContext = (): SequenceContext => ({ + phrase: null, + clauseLevel: false, + insideParens: true, + logical: false, + subClauses: [], + commas: true, + expandableGroups: "none", + groupsSeen: 0, +}) + +const caseContext = (): SequenceContext => ({ + ...groupContext(), + insideParens: false, + commas: false, +}) + +const isComment = (piece: Piece) => + piece.token.kind === "lineComment" || piece.token.kind === "blockComment" + +const isCloser = (piece: Piece) => + piece.token.tokenName === "RParen" || piece.token.tokenName === "RBracket" + +const startsQuery = (piece: Piece | undefined) => + piece !== undefined && + (piece.token.tokenName === "Select" || + piece.token.tokenName === "With" || + piece.token.tokenName === "Declare") + +const opensBlock = (next: Piece | undefined) => startsQuery(next) + +type Clause = { doc: Doc; headless: boolean } + +const alwaysBreaksList = (phrase: Phrase | null) => + phrase !== null && phrase.names[0] === "Declare" + +const allowsLogical = (phrase: Phrase | null) => + phrase !== null && phrase.names[0] === "Where" + +const subClausesFor = (phrase: Phrase | null): Phrase[] => + phrase !== null && phrase.role === "join" ? joinSubClauses : [] + +const endsOperand = (piece: Piece | null) => + piece !== null && + (piece.token.kind === "word" || + piece.token.tokenName === "RParen" || + piece.token.tokenName === "RBracket") + +const groupExpansionFor = (phrase: Phrase | null): GroupExpansion => { + if (phrase === null) return "none" + if (phrase.names[0] === "Values") return "all" + if (phrase.names.join(" ") === "Create Table") return "first" + return "none" +} + +const withoutTrailingWhitespace = (tokens: StreamToken[]): StreamToken[] => { + let end = tokens.length + while (end > 0 && tokens[end - 1].kind === "whitespace") end-- + return tokens.slice(0, end) +} + +const toPieces = ( + tokens: StreamToken[], +): { pieces: Piece[]; trailingGap: string } => { + const pieces: Piece[] = [] + let gapBefore = "" + for (const token of tokens) { + if (token.kind === "whitespace") { + gapBefore += token.image + continue + } + pieces.push({ token, gapBefore, unary: false, endsLine: false }) + gapBefore = "" + } + return { pieces, trailingGap: gapBefore } +} + +class StatementBuilder { + private index = 0 + private phrases: Phrase[] + private kind: StatementKind + private readonly tokens: StreamToken[] + previous: Piece | null = null + + constructor( + private readonly pieces: Piece[], + kind: StatementKind, + private readonly uppercaseOffsets: ReadonlySet, + ) { + this.kind = kind + this.phrases = clausePhrases[kind] + this.tokens = pieces.map((piece) => piece.token) + } + + private enterQuery() { + this.kind = "select" + this.phrases = clausePhrases.select + } + + private startsQueryBody(match: PhraseMatch | null): boolean { + if (match === null) return false + const first = match.phrase.names[0] + if (first === "Select" || first === "Declare") return true + return ( + first === "With" && + (this.kind === "insert" || this.phrases === clausePhrases.select) + ) + } + + build(): Doc { + const first = this.peek() + if (first === undefined || !isExplainPrefix(first.token)) { + return this.buildClauses(false) + } + const prefix = this.takeText() + return concat([prefix.doc, gap(" "), this.buildClauses(false)]) + } + + private get atEnd() { + return this.index >= this.pieces.length + } + + private peek(offset = 0): Piece | undefined { + return this.pieces[this.index + offset] + } + + private take(): Piece { + const piece = this.pieces[this.index++] + piece.unary = isSign(piece) && !isOperand(this.previous) + this.previous = piece + return piece + } + + private leadingFor(piece: Piece): Doc { + if ( + isComment(piece) && + this.previous !== null && + piece.gapBefore.includes("\n") + ) { + piece.endsLine = true + return hardline + } + return gap(gapBetween(this.previous, piece)) + } + + /** Raises the words the caller identified as syntax; see the capitalize option. */ + private textOf(piece: Piece): string { + return this.uppercaseOffsets.has(piece.token.startOffset) + ? piece.token.image.toUpperCase() + : piece.token.image + } + + private takeText(): Element { + const piece = this.peek()! + const leading = this.leadingFor(piece) + this.take() + return { leading, doc: text(this.textOf(piece)) } + } + + private matchClause(current: Phrase | null): PhraseMatch | null { + const match = matchPhrase(this.tokens, this.index, this.phrases) + if (match === null || continuesPhrase(current, match.phrase)) return null + return match + } + + private buildClauses(insideParens: boolean): Doc { + return concat( + this.collectClauses(insideParens).map((clause, i) => + i === 0 ? clause.doc : concat([hardline, clause.doc]), + ), + ) + } + + private collectClauses(insideParens: boolean): Clause[] { + const clauses: Clause[] = [] + while (!this.atEnd) { + const piece = this.peek()! + if (insideParens && piece.token.tokenName === "RParen") break + if (clauses.length > 0 && this.kind !== "select") { + if (this.startsQueryBody(this.matchClause(null))) { + this.enterQuery() + clauses.push({ + doc: indent(concat([hardline, this.buildClauses(insideParens)])), + headless: false, + }) + break + } + } + const start = this.index + const clause = this.buildClause(insideParens) + if (this.index === start) { + const element = this.takeText() + clauses.push({ + doc: concat([element.leading, element.doc]), + headless: true, + }) + continue + } + clauses.push(clause) + } + return clauses + } + + private buildClause(insideParens: boolean): Clause { + const match = this.matchClause(null) + const phrase = match === null ? null : match.phrase + let head: Doc | null = null + if (match !== null) { + const headParts: Doc[] = [this.takeText().doc] + while (this.index < match.endIndex) { + const element = this.takeText() + headParts.push(element.leading, element.doc) + } + head = concat(headParts) + } + const sequence = this.buildSequence({ + phrase, + clauseLevel: true, + insideParens, + logical: allowsLogical(phrase), + subClauses: subClausesFor(phrase), + commas: true, + expandableGroups: groupExpansionFor(phrase), + groupsSeen: 0, + }) + return { + doc: this.renderClause(head, sequence, alwaysBreaksList(phrase)), + headless: head === null, + } + } + + private renderClause( + head: Doc | null, + sequence: Sequence, + alwaysBreak: boolean, + ): Doc { + const { items } = sequence + if (head === null) return this.renderList(sequence, false) + if (items.length === 0) return head + if (items.length === 1) { + return concat([head, this.renderList(sequence, false)]) + } + const firstInline = sequence.separators.every( + (separator) => separator.kind === "subClause", + ) + const body = concat([ + head, + indent(this.renderList(sequence, true, firstInline)), + ]) + return alwaysBreak ? body : group(body) + } + + private renderList( + sequence: Sequence, + breakable: boolean, + firstInline = false, + ): Doc { + const { items, separators } = sequence + const parts: Doc[] = [] + items.forEach((item, i) => { + const afterComma = i === 0 || separators[i - 1].kind === "comma" + const breaks = breakable && afterComma && !(firstInline && i === 0) + parts.push(breaks ? asLine(item.leading) : item.leading, item.doc) + const separator = separators[i] + if (separator === undefined) return + parts.push( + breakable && separator.kind !== "comma" + ? asLine(separator.leading) + : separator.leading, + separator.doc, + ) + }) + return concat(parts) + } + + private buildSequence(ctx: SequenceContext): Sequence { + const items: Element[] = [] + const separators: Separator[] = [] + let current: Element | null = null + let betweenPending = false + let subClauseStarted = false + + const closeItem = () => { + items.push(current ?? { leading: gap(""), doc: concat([]) }) + current = null + } + + while (!this.atEnd) { + const piece = this.peek()! + const name = piece.token.tokenName + if (ctx.insideParens && isCloser(piece)) break + if (ctx.clauseLevel && this.matchClause(ctx.phrase) !== null) break + if ( + items.length === 0 && + current === null && + ctx.subClauses.length > 0 && + matchPhrase(this.tokens, this.index, ctx.subClauses) !== null + ) { + subClauseStarted = true + } + if (ctx.clauseLevel && name === "As" && startsQuery(this.peek(1))) { + const element = this.takeText() + current = this.append(current, element) + this.phrases = clausePhrases.select + break + } + + if (ctx.commas && !subClauseStarted && name === "Comma") { + closeItem() + separators.push({ + kind: "comma", + leading: this.leadingFor(piece), + doc: text(","), + }) + this.take() + continue + } + + const subClause = + current !== null && endsOperand(this.previous) + ? matchPhrase(this.tokens, this.index, ctx.subClauses) + : null + if (subClause !== null) { + closeItem() + const leading = this.leadingFor(piece) + const words: Doc[] = [this.takeText().doc] + while (this.index < subClause.endIndex) { + const word = this.takeText() + words.push(word.leading, word.doc) + } + separators.push({ kind: "subClause", leading, doc: concat(words) }) + subClauseStarted = true + continue + } + + if (name === "Between") betweenPending = true + const closesBetween = name === "And" && betweenPending + if (closesBetween) betweenPending = false + + if ( + ctx.logical && + !closesBetween && + (name === "And" || name === "Or") && + current !== null + ) { + closeItem() + separators.push({ + kind: "logical", + leading: this.leadingFor(piece), + doc: text(this.textOf(piece)), + }) + this.take() + continue + } + + const element = this.buildElement(ctx) + const lastSeparator = separators[separators.length - 1] + if ( + current === null && + lastSeparator !== undefined && + items.length === separators.length && + piece.token.kind === "lineComment" && + element.leading.type === "gap" + ) { + lastSeparator.doc = concat([ + lastSeparator.doc, + element.leading, + element.doc, + ]) + continue + } + current = this.append(current, element) + } + + if (current !== null) closeItem() + return { items, separators } + } + + private append(current: Element | null, element: Element): Element { + if (current === null) return element + return { + leading: current.leading, + doc: concat([current.doc, element.leading, element.doc]), + } + } + + private buildElement(ctx: SequenceContext): Element { + const piece = this.peek()! + const name = piece.token.tokenName + if (name === "LParen") { + if (opensBlock(this.peek(1)) || this.precedesTableSource(ctx)) { + return this.buildBlock() + } + const afterIn = this.previous?.token.tokenName === "In" + const afterOver = + this.previous?.token.tokenName === "Over" || + (ctx.phrase?.names[0] === "Window" && + this.previous?.token.tokenName === "As") + const afterPivot = + this.previous?.token.tokenName === "Pivot" || + this.previous?.token.tokenName === "Unpivot" + const expandable = + ctx.expandableGroups === "all" || + (ctx.expandableGroups === "first" && ctx.groupsSeen === 0) + ctx.groupsSeen++ + return this.buildGroup( + expandable || afterIn || afterOver || afterPivot, + "RParen", + afterOver ? windowSubClauses : afterPivot ? pivotSubClauses : [], + ) + } + if (name === "LBracket") return this.buildGroup(false, "RBracket") + if (name === "Case") return this.buildCase() + + const element = this.takeText() + if ( + piece.token.kind === "lineComment" || + (piece.token.kind === "blockComment" && this.followedByNewline(piece)) + ) { + piece.endsLine = true + return { leading: element.leading, doc: concat([element.doc, hardline]) } + } + return element + } + + private followedByNewline(piece: Piece) { + const next = this.peek() + return ( + piece.endsLine || (next !== undefined && next.gapBefore.includes("\n")) + ) + } + + private buildGroup( + expandable: boolean, + closer: string, + subClauses: Phrase[] = [], + ): Element { + const open = this.takeText() + const sequence = this.buildSequence({ ...groupContext(), subClauses }) + const close = + this.peek()?.token.tokenName === closer ? this.takeText() : null + const closeParts = close === null ? [] : [close.leading, close.doc] + + if (expandable) { + return { + leading: open.leading, + doc: group( + concat([ + open.doc, + indent(this.renderList(sequence, true)), + ...closeParts.map(asLine), + ]), + ), + } + } + return { + leading: open.leading, + doc: concat([open.doc, this.renderList(sequence, false), ...closeParts]), + } + } + + private precedesTableSource(ctx: SequenceContext): boolean { + const name = this.previous?.token.tokenName + if (name === undefined || name === null) return false + if (name === "From" || name === "Join") return true + const clause = ctx.phrase?.names[0] + if (name === "Comma") return clause === "From" + if (name === "As") + return clause === "With" || this.kind.startsWith("create") + return false + } + + private buildBlock(): Element { + const open = this.peek()! + const base = gapBetween(this.previous, open) + const leading = gap( + base === "" && this.previous?.token.kind === "word" ? " " : base, + ) + this.take() + + const outerPhrases = this.phrases + const outerKind = this.kind + this.enterQuery() + const clauses = this.collectClauses(true) + this.phrases = outerPhrases + this.kind = outerKind + + const close = + this.peek()?.token.tokenName === "RParen" ? this.takeText() : null + + if (clauses.length === 1 && clauses[0].headless) { + const closeParts = close === null ? [] : [close.leading, close.doc] + return { + leading, + doc: concat([text(open.token.image), clauses[0].doc, ...closeParts]), + } + } + + const body = concat( + clauses.map((clause, i) => + i === 0 ? clause.doc : concat([hardline, clause.doc]), + ), + ) + const closeParts = close === null ? [] : [hardline, close.doc] + return { + leading, + doc: concat([ + text(open.token.image), + indent(concat([hardline, body])), + ...closeParts, + ]), + } + } + + private buildCase(): Element { + const start = this.takeText() + const head: Doc[] = [start.doc] + const rows: Element[] = [] + let end: Element | null = null + + while (!this.atEnd) { + const piece = this.peek()! + const name = piece.token.tokenName + if (isCloser(piece)) break + if (name === "End") { + end = this.takeText() + break + } + if (name === "When" || name === "Else") { + rows.push(this.takeText()) + continue + } + const element = this.buildElement(caseContext()) + if (rows.length === 0) { + head.push(element.leading, element.doc) + continue + } + const row = rows[rows.length - 1] + row.doc = concat([row.doc, element.leading, element.doc]) + } + + const parts: Doc[] = [concat(head)] + if (rows.length > 0) { + parts.push( + indent(concat(rows.flatMap((row) => [asLine(row.leading), row.doc]))), + ) + } + if (end !== null) parts.push(asLine(end.leading), end.doc) + return { leading: start.leading, doc: group(concat(parts)) } + } +} + +export const formatStatement = ( + statement: Statement, + options: PrintOptions, + uppercaseOffsets: ReadonlySet, +): string => { + const kind = detectStatementKind(statement.tokens) + const boundary = statement.verbatimFrom ?? statement.tokens.length + const { pieces, trailingGap } = toPieces(statement.tokens.slice(0, boundary)) + const builder = new StatementBuilder(pieces, kind, uppercaseOffsets) + const parts: Doc[] = [builder.build()] + + if (statement.verbatimFrom !== null) { + const tail = statement.tokens.slice(statement.verbatimFrom) + const tailPiece: Piece = { + token: tail[0], + gapBefore: trailingGap, + unary: false, + endsLine: false, + } + parts.push( + gap(gapBetween(builder.previous, tailPiece)), + verbatim(reconstruct(withoutTrailingWhitespace(tail))), + ) + } + + return printDoc(concat(parts), options) +} diff --git a/src/formatter/lexer.ts b/src/formatter/lexer.ts new file mode 100644 index 0000000..33249c9 --- /dev/null +++ b/src/formatter/lexer.ts @@ -0,0 +1,171 @@ +import { createToken, Lexer, TokenType } from "chevrotain" +import { + allTokens, + BlockComment, + LineComment, + QuotedIdentifier, + StringLiteral, + WhiteSpace, +} from "../parser/lexer" +import { classify, SignificantKind } from "./classification" + +export type StreamTokenKind = + | SignificantKind + | "whitespace" + | "lineComment" + | "blockComment" + | "tolerant" + | "opaque" + +export type StreamToken = { + kind: StreamTokenKind + image: string + startOffset: number + endOffset: number + tokenName: string | null +} + +const TRIVIA_GROUP = "trivia" + +const FormatterWhiteSpace = createToken({ + name: "WhiteSpace", + pattern: /\s+/, + group: TRIVIA_GROUP, + line_breaks: true, +}) + +const FormatterLineComment = createToken({ + name: "LineComment", + pattern: /--[^\n\r]*/, + group: TRIVIA_GROUP, +}) + +const FormatterBlockComment = createToken({ + name: "BlockComment", + pattern: /\/\*[\s\S]*?\*\//, + group: TRIVIA_GROUP, + line_breaks: true, +}) + +const matchUnterminatedQuoted = + (quote: string) => + (text: string, offset: number): [string] | null => { + if (text[offset] !== quote) return null + let index = offset + 1 + while (index < text.length) { + if (text[index] === quote) { + if (text[index + 1] === quote) { + index += 2 + continue + } + return null + } + index++ + } + return [text.slice(offset)] + } + +const matchUnterminatedBlockComment = ( + text: string, + offset: number, +): [string] | null => { + if (!text.startsWith("/*", offset)) return null + if (text.indexOf("*/", offset + 2) !== -1) return null + return [text.slice(offset)] +} + +const UnterminatedBlockComment = createToken({ + name: "UnterminatedBlockComment", + pattern: { exec: matchUnterminatedBlockComment }, + line_breaks: true, + start_chars_hint: ["/"], +}) + +const UnterminatedString = createToken({ + name: "UnterminatedString", + pattern: { exec: matchUnterminatedQuoted("'") }, + line_breaks: true, + start_chars_hint: ["'"], +}) + +const UnterminatedQuotedIdentifier = createToken({ + name: "UnterminatedQuotedIdentifier", + pattern: { exec: matchUnterminatedQuoted('"') }, + line_breaks: true, + start_chars_hint: ['"'], +}) + +const tolerantTokenNames: ReadonlySet = new Set([ + UnterminatedBlockComment.name, + UnterminatedString.name, + UnterminatedQuotedIdentifier.name, +]) + +const triviaKinds: Record = { + [FormatterWhiteSpace.name]: "whitespace", + [FormatterLineComment.name]: "lineComment", + [FormatterBlockComment.name]: "blockComment", +} + +export const buildFormatterTokens = (): TokenType[] => + allTokens.flatMap((token) => { + if (token === WhiteSpace) return [FormatterWhiteSpace] + if (token === LineComment) return [FormatterLineComment] + if (token === BlockComment) { + return [FormatterBlockComment, UnterminatedBlockComment] + } + if (token === StringLiteral) return [UnterminatedString, StringLiteral] + if (token === QuotedIdentifier) { + return [UnterminatedQuotedIdentifier, QuotedIdentifier] + } + return [token] + }) + +export const formatterLexer = new Lexer(buildFormatterTokens(), { + positionTracking: "onlyOffset", +}) + +export const scan = (sql: string): StreamToken[] => { + const result = formatterLexer.tokenize(sql) + + const significant = result.tokens.map( + (token): StreamToken => ({ + kind: tolerantTokenNames.has(token.tokenType.name) + ? "tolerant" + : classify(token.tokenType.name), + image: token.image, + startOffset: token.startOffset, + endOffset: token.startOffset + token.image.length, + tokenName: tolerantTokenNames.has(token.tokenType.name) + ? null + : token.tokenType.name, + }), + ) + + const trivia = (result.groups[TRIVIA_GROUP] ?? []).map( + (token): StreamToken => ({ + kind: triviaKinds[token.tokenType.name], + image: token.image, + startOffset: token.startOffset, + endOffset: token.startOffset + token.image.length, + tokenName: null, + }), + ) + + const opaque = result.errors.map( + (error): StreamToken => ({ + kind: "opaque", + image: sql.slice(error.offset, error.offset + error.length), + startOffset: error.offset, + endOffset: error.offset + error.length, + tokenName: null, + }), + ) + + return [...significant, ...trivia, ...opaque].sort( + (a, b) => a.startOffset - b.startOffset, + ) +} + +export const reconstruct = (tokens: StreamToken[]): string => + tokens.map((token) => token.image).join("") diff --git a/src/formatter/phrases.ts b/src/formatter/phrases.ts new file mode 100644 index 0000000..cde04fd --- /dev/null +++ b/src/formatter/phrases.ts @@ -0,0 +1,199 @@ +import { StatementKind } from "./context" +import { StreamToken } from "./lexer" + +export type PhraseRole = "clause" | "join" | "setOp" | "action" + +export type Phrase = { + names: string[] + role: PhraseRole +} + +export type PhraseMatch = { + phrase: Phrase + endIndex: number +} + +const phrases = + (role: PhraseRole) => + (...list: string[][]): Phrase[] => + list.map((names) => ({ names, role })) + +const clause = phrases("clause") +const join = phrases("join") +const setOp = phrases("setOp") +const action = phrases("action") + +const selectPhrases: Phrase[] = [ + ...clause( + ["With"], + ["Declare"], + ["Select"], + ["From"], + ["Where"], + ["Latest", "On"], + ["Latest", "By"], + ["Sample", "By"], + ["Group", "By"], + ["Order", "By"], + ["Limit"], + ["Window"], + ["Pivot"], + ), + ...join( + ["Join"], + ["Inner", "Join"], + ["Left", "Join"], + ["Left", "Outer", "Join"], + ["Right", "Join"], + ["Right", "Outer", "Join"], + ["Full", "Join"], + ["Full", "Outer", "Join"], + ["Cross", "Join"], + ["Asof", "Join"], + ["Lt", "Join"], + ["Splice", "Join"], + ["Window", "Join"], + ["Prevailing", "Join"], + ["Horizon", "Join"], + ), + ...setOp(["Union", "All"], ["Union"], ["Except"], ["Intersect"]), +] + +const insertPhrases: Phrase[] = [ + ...clause( + ["Insert", "Into"], + ["Insert", "Atomic", "Into"], + ["Insert"], + ["Values"], + ), + ...selectPhrases, +] + +const joinPhrases: Phrase[] = selectPhrases.filter( + (phrase) => phrase.role === "join", +) + +const updatePhrases: Phrase[] = [ + ...clause(["Update"], ["Set"], ["From"], ["Where"]), + ...joinPhrases, +] + +const createOptionPhrases: Phrase[] = clause( + ["Dedup"], + ["Ttl"], + ["Expire", "Rows"], + ["Storage", "Policy"], + ["With"], + ["In", "Volume"], + ["Refresh"], + ["Flush"], +) + +const alterActionPhrases: Phrase[] = action( + ["Add", "Column"], + ["Drop", "Column"], + ["Rename", "Column"], + ["Alter", "Column"], + ["Attach", "Partition"], + ["Attach", "Partition", "List"], + ["Detach", "Partition"], + ["Detach", "Partition", "List"], + ["Drop", "Partition"], + ["Drop", "Partition", "List"], + ["Set"], + ["Squash"], + ["Dedup"], + ["Resume", "Wal"], + ["Suspend", "Wal"], + ["Rebase", "Wal"], + ["Convert", "Partition"], + ["Convert", "Partition", "List"], + ["Drop", "Expire"], + ["Drop", "Storage", "Policy"], + ["Enable", "Storage", "Policy"], + ["Disable", "Storage", "Policy"], +) + +export const joinSubClauses: Phrase[] = clause( + ["On"], + ["Range"], + ["List"], + ["Tolerance"], + ["Include", "Prevailing"], + ["Exclude", "Prevailing"], +) + +export const windowSubClauses: Phrase[] = clause( + ["Partition", "By"], + ["Order", "By"], + ["Rows"], + ["Range"], + ["Groups"], + ["Exclude"], + ["Anchor"], +) + +export const pivotSubClauses: Phrase[] = clause(["For"], ["Group", "By"]) + +export const clausePhrases: Record = { + select: selectPhrases, + insert: insertPhrases, + update: updatePhrases, + createTable: [...clause(["Create", "Table"]), ...createOptionPhrases], + createMaterializedView: [ + ...clause(["Create", "Materialized", "View"]), + ...createOptionPhrases, + ], + createLiveView: [ + ...clause(["Create", "Live", "View"]), + ...createOptionPhrases, + ], + alterTable: [...clause(["Alter", "Table"]), ...alterActionPhrases], + alterMaterializedView: [ + ...clause(["Alter", "Materialized", "View"]), + ...alterActionPhrases, + ], + other: [], +} + +const phraseKey = (phrase: Phrase) => phrase.names.join(" ") + +const continuations: Record> = { + "Sample By": new Set(["From", "With"]), +} + +export const continuesPhrase = (current: Phrase | null, next: Phrase) => + current !== null && + (continuations[phraseKey(current)]?.has(phraseKey(next)) ?? false) + +const matchSingle = ( + tokens: StreamToken[], + start: number, + phrase: Phrase, +): PhraseMatch | null => { + let index = start + for (const name of phrase.names) { + while (index < tokens.length && tokens[index].kind === "whitespace") index++ + if (index >= tokens.length || tokens[index].tokenName !== name) return null + index++ + } + return { phrase, endIndex: index } +} + +export const matchPhrase = ( + tokens: StreamToken[], + index: number, + candidates: Phrase[], +): PhraseMatch | null => { + let best: PhraseMatch | null = null + for (const phrase of candidates) { + const match = matchSingle(tokens, index, phrase) + if ( + match && + (best === null || phrase.names.length > best.phrase.names.length) + ) { + best = match + } + } + return best +} diff --git a/src/formatter/spacing.ts b/src/formatter/spacing.ts new file mode 100644 index 0000000..203f193 --- /dev/null +++ b/src/formatter/spacing.ts @@ -0,0 +1,66 @@ +import { operandTokenNames } from "./classification" +import { StreamToken } from "./lexer" + +export type Piece = { + token: StreamToken + gapBefore: string + unary: boolean + endsLine: boolean +} + +const noSpaceBefore: ReadonlySet = new Set([ + "Comma", + "RParen", + "RBracket", + "Semicolon", + "Dot", + "DoubleColon", + "Colon", + "LBracket", +]) + +const noSpaceAfter: ReadonlySet = new Set([ + "LParen", + "LBracket", + "Dot", + "DoubleColon", + "Colon", +]) + +const isComment = (piece: Piece) => + piece.token.kind === "lineComment" || piece.token.kind === "blockComment" + +const isNumber = (piece: Piece) => piece.token.tokenName === "NumberLiteral" + +const isUncertain = (piece: Piece) => + piece.token.kind === "opaque" || piece.token.kind === "tolerant" + +export const isSign = (piece: Piece) => + piece.token.tokenName === "Minus" || piece.token.tokenName === "Plus" + +export const isOperand = (piece: Piece | null) => + piece !== null && + piece.token.tokenName !== null && + operandTokenNames.has(piece.token.tokenName) + +const preserved = (next: Piece) => (next.gapBefore === "" ? "" : " ") + +export const gapBetween = (previous: Piece | null, next: Piece): string => { + if (previous === null) return "" + if (previous.token.kind === "lineComment" || previous.endsLine) return "" + if (isUncertain(previous) || isUncertain(next)) return next.gapBefore + if (previous.token.kind === "operator" && next.token.kind === "operator") { + return preserved(next) + } + if (previous.unary && !isComment(next)) return "" + const previousName = previous.token.tokenName + const nextName = next.token.tokenName + if (nextName === "Dot" && isNumber(previous)) return " " + if (previousName === "Dot" && isNumber(next)) return " " + if (nextName !== null && noSpaceBefore.has(nextName)) return "" + if (previousName !== null && noSpaceAfter.has(previousName)) return "" + if (nextName === "LParen" && previous.token.kind === "word") { + return preserved(next) + } + return " " +} diff --git a/src/formatter/statements.ts b/src/formatter/statements.ts new file mode 100644 index 0000000..5491256 --- /dev/null +++ b/src/formatter/statements.ts @@ -0,0 +1,73 @@ +import { StreamToken } from "./lexer" + +export type Statement = { + tokens: StreamToken[] + verbatimFrom: number | null +} + +const closerFor: Record = { + LParen: "RParen", + LBracket: "RBracket", +} + +const isOpener = (token: StreamToken) => + token.tokenName !== null && token.tokenName in closerFor + +const isCloser = (token: StreamToken) => + token.tokenName === "RParen" || token.tokenName === "RBracket" + +const hasContent = (tokens: StreamToken[]) => + tokens.some((token) => token.kind !== "whitespace") + +export const splitStatements = (tokens: StreamToken[]): Statement[] => { + const statements: Statement[] = [] + let current: StreamToken[] = [] + let openers: number[] = [] + + const flush = (verbatimFrom: number | null) => { + if (hasContent(current)) statements.push({ tokens: current, verbatimFrom }) + current = [] + openers = [] + } + + const preserveRest = (from: number, verbatimFrom: number) => { + current.push(...tokens.slice(from)) + flush(verbatimFrom) + } + + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index] + + if (token.kind === "tolerant") { + preserveRest(index, current.length) + return statements + } + + if (isOpener(token)) { + openers.push(current.length) + current.push(token) + continue + } + + if (isCloser(token)) { + const opener = + openers.length > 0 ? current[openers[openers.length - 1]] : null + if (opener !== null && closerFor[opener.tokenName!] !== token.tokenName) { + preserveRest(index, openers[0]) + return statements + } + if (opener !== null) openers.pop() + current.push(token) + continue + } + + current.push(token) + + if (token.tokenName === "Semicolon" && openers.length === 0) { + flush(null) + } + } + + flush(openers.length > 0 ? openers[0] : null) + return statements +} diff --git a/src/formatter/syntax.ts b/src/formatter/syntax.ts new file mode 100644 index 0000000..4b1348d --- /dev/null +++ b/src/formatter/syntax.ts @@ -0,0 +1,46 @@ +import { parse } from "../parser/parser" +import { keywordTokenArray } from "../parser/tokens" + +const keywordNames = new Set(keywordTokenArray.map((token) => token.name)) + +/** The grammar rule through which any word becomes a name. */ +const IDENTIFIER_RULE = "identifier" + +type CstNode = { name?: string; children?: Record } +type CstToken = { startOffset?: number; tokenType?: { name: string } } + +const collect = ( + node: unknown, + insideName: boolean, + offsets: Set, +): void => { + if (node === null || typeof node !== "object") return + + const token = node as CstToken + if (token.startOffset !== undefined && token.tokenType !== undefined) { + if (!insideName && keywordNames.has(token.tokenType.name)) { + offsets.add(token.startOffset) + } + return + } + + const rule = node as CstNode + const namesSomething = insideName || rule.name === IDENTIFIER_RULE + for (const children of Object.values(rule.children ?? {})) { + for (const child of children) collect(child, namesSomething, offsets) + } +} + +/** + * The offset of every keyword the grammar consumed as syntax. A keyword the + * grammar took through its identifier rule names a table, view or column, and + * is left out. SQL that does not parse has no syntax to report. + */ +export const syntaxOffsets = (sql: string): ReadonlySet => { + const { cst, lexErrors, parseErrors } = parse(sql) + if (lexErrors.length > 0 || parseErrors.length > 0) return new Set() + + const offsets = new Set() + collect(cst, false, offsets) + return offsets +} diff --git a/src/grammar/keywords.ts b/src/grammar/keywords.ts index a6fda07..60016b1 100644 --- a/src/grammar/keywords.ts +++ b/src/grammar/keywords.ts @@ -20,6 +20,8 @@ export const keywords: string[] = [ "batch", "between", "bloom_filter", + "bloom_filter_columns", + "bloom_filter_fpp", "by", "bypass", "cache", diff --git a/src/index.ts b/src/index.ts index ec8ee25..486e77c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -61,6 +61,8 @@ export { functions, operators, } from "./grammar/index" +export { format } from "./formatter/index" +export type { FormatOptions } from "./formatter/index" /** * Parse SQL string to AST diff --git a/src/parser/ast.ts b/src/parser/ast.ts index 375e7e1..6f1eabc 100644 --- a/src/parser/ast.ts +++ b/src/parser/ast.ts @@ -276,6 +276,9 @@ export interface CreateLiveViewStatement extends AstNode { value?: string } query: SelectStatement + ownedBy?: string + /** Whether the query was written as AS (query) rather than AS query */ + asParens?: boolean } export interface DropLiveViewStatement extends AstNode { @@ -407,6 +410,8 @@ export interface AlterMaterializedViewAddIndex { actionType: "addIndex" column: string capacity?: number + indexType?: "posting" | "posting_delta" | "posting_ef" | "bitmap" | "none" + indexInclude?: string[] } export interface AlterMaterializedViewSymbolCapacity { @@ -494,6 +499,7 @@ export type AlterUserAction = | AlterUserPasswordAction | AlterUserCreateTokenAction | AlterUserDropTokenAction + | AlterUserMemoryLimitAction export interface AlterUserEnableAction { actionType: "enable" @@ -525,6 +531,12 @@ export interface AlterUserDropTokenAction { token?: string } +/** SET MEMORY LIMIT 1G | 512M | 0 | UNLIMITED */ +export interface AlterUserMemoryLimitAction { + actionType: "setMemoryLimit" + limit: string +} + export interface AlterServiceAccountStatement extends AstNode { type: "alterServiceAccount" account: QualifiedName @@ -685,6 +697,8 @@ export interface ConvertPartitionAction { partitions?: string[] target: string where?: Expression + /** WITH (bloom_filter_columns = '...', bloom_filter_fpp = 0.01) */ + withParams?: TableParam[] } export interface DropTableStatement extends AstNode { @@ -970,8 +984,10 @@ export interface SwitchStatement extends AstNode { export interface AlterGroupStatement extends AstNode { type: "alterGroup" group: QualifiedName - action: "setAlias" | "dropAlias" - externalAlias: string + action: "setAlias" | "dropAlias" | "setMemoryLimit" + externalAlias?: string + /** SET MEMORY LIMIT value, e.g. "2G", "512M", "0", "UNLIMITED" */ + memoryLimit?: string } export interface CompileViewStatement extends AstNode { @@ -1087,6 +1103,8 @@ export interface TableRef extends AstNode { columnAliases?: string[] joins?: JoinClause[] timestampDesignation?: string + /** TIMESTAMP(col) was written after the alias: FROM t alias TIMESTAMP(col) */ + timestampAfterAlias?: boolean } export interface JoinClause extends AstNode { @@ -1094,6 +1112,8 @@ export interface JoinClause extends AstNode { joinType?: | "inner" | "left" + | "right" + | "full" | "cross" | "asof" | "lt" diff --git a/src/parser/cst-types.d.ts b/src/parser/cst-types.d.ts index 2b1e5a3..fa4e3ff 100644 --- a/src/parser/cst-types.d.ts +++ b/src/parser/cst-types.d.ts @@ -289,6 +289,19 @@ export type TableRefCstChildren = { columnRef?: ColumnRefCstNode[]; As?: IToken[]; identifier?: (IdentifierCstNode)[]; + aliasTimestampDesignation?: AliasTimestampDesignationCstNode[]; +}; + +export interface AliasTimestampDesignationCstNode extends CstNode { + name: "aliasTimestampDesignation"; + children: AliasTimestampDesignationCstChildren; +} + +export type AliasTimestampDesignationCstChildren = { + Timestamp: IToken[]; + LParen: IToken[]; + columnRef: ColumnRefCstNode[]; + RParen: IToken[]; }; export interface TableFunctionCallCstNode extends CstNode { @@ -418,12 +431,14 @@ export interface StandardJoinCstNode extends CstNode { export type StandardJoinCstChildren = { Left?: IToken[]; - Outer?: IToken[]; + Outer?: (IToken)[]; + Right?: IToken[]; + Full?: IToken[]; Inner?: IToken[]; Cross?: IToken[]; Join: IToken[]; Lateral?: IToken[]; - tableRef: TableRefCstNode[]; + fromSource: FromSourceCstNode[]; On?: IToken[]; expression?: ExpressionCstNode[]; }; @@ -516,7 +531,9 @@ export interface FillValueCstNode extends CstNode { export type FillValueCstChildren = { Null?: IToken[]; NumberLiteral?: IToken[]; - identifier?: IdentifierCstNode[]; + identifier?: (IdentifierCstNode)[]; + LParen?: IToken[]; + RParen?: IToken[]; }; export interface AlignToClauseCstNode extends CstNode { @@ -1234,7 +1251,7 @@ export type CreateLiveViewBodyCstChildren = { In?: IToken[]; Memory?: IToken[]; Partition?: IToken[]; - By?: IToken[]; + By?: (IToken)[]; partitionPeriod?: PartitionPeriodCstNode[]; Start?: IToken[]; From?: IToken[]; @@ -1245,6 +1262,8 @@ export type CreateLiveViewBodyCstChildren = { LParen?: IToken[]; selectStatement?: (SelectStatementCstNode)[]; RParen?: IToken[]; + Owned?: IToken[]; + stringOrIdentifier?: StringOrIdentifierCstNode[]; }; export interface DropLiveViewStatementCstNode extends CstNode { @@ -1365,6 +1384,10 @@ export type AlterGroupStatementCstChildren = { Alias?: (IToken)[]; StringLiteral?: (IToken)[]; Drop?: IToken[]; + Set?: IToken[]; + Memory?: IToken[]; + Limit?: IToken[]; + memoryLimit?: MemoryLimitCstNode[]; }; export interface AlterViewStatementCstNode extends CstNode { @@ -1412,6 +1435,10 @@ export interface AlterUserActionCstNode extends CstNode { export type AlterUserActionCstChildren = { Enable?: IToken[]; Disable?: IToken[]; + Set?: IToken[]; + Memory?: IToken[]; + Limit?: IToken[]; + memoryLimit?: MemoryLimitCstNode[]; With?: (IToken)[]; No?: IToken[]; Password?: (IToken)[]; @@ -1431,6 +1458,18 @@ export type AlterUserActionCstChildren = { Drop?: IToken[]; }; +export interface MemoryLimitCstNode extends CstNode { + name: "memoryLimit"; + children: MemoryLimitCstChildren; +} + +export type MemoryLimitCstChildren = { + Unlimited?: IToken[]; + DurationLiteral?: IToken[]; + NumberLiteral?: IToken[]; + Identifier?: IToken[]; +}; + export interface AlterTableStatementCstNode extends CstNode { name: "alterTableStatement"; children: AlterTableStatementCstChildren; @@ -1519,12 +1558,16 @@ export interface ConvertPartitionTargetCstNode extends CstNode { export type ConvertPartitionTargetCstChildren = { List?: IToken[]; StringLiteral?: (IToken)[]; - Comma?: IToken[]; + Comma?: (IToken)[]; To: IToken[]; Table?: IToken[]; identifier?: IdentifierCstNode[]; Where?: IToken[]; expression?: ExpressionCstNode[]; + With?: IToken[]; + LParen?: IToken[]; + tableParam?: (TableParamCstNode)[]; + RParen?: IToken[]; }; export interface AlterMaterializedViewStatementCstNode extends CstNode { @@ -1550,10 +1593,11 @@ export type AlterMaterializedViewActionCstChildren = { columnRef?: ColumnRefCstNode[]; Add?: IToken[]; Index?: (IToken)[]; - Capacity?: (IToken)[]; - NumberLiteral?: (IToken)[]; + indexTypeOptions?: IndexTypeOptionsCstNode[]; Drop?: (IToken)[]; Symbol?: IToken[]; + Capacity?: IToken[]; + NumberLiteral?: (IToken)[]; Set?: IToken[]; Ttl?: IToken[]; DurationLiteral?: (IToken)[]; @@ -1953,6 +1997,8 @@ export type CopyOptionCstChildren = { ParquetVersion?: IToken[]; NumberLiteral?: IToken[]; RawArrayEncoding?: IToken[]; + BloomFilterColumns?: IToken[]; + BloomFilterFpp?: IToken[]; }; export interface CheckpointStatementCstNode extends CstNode { @@ -2913,6 +2959,7 @@ export interface ICstNodeVisitor extends ICstVisitor { implicitSelectBody(children: ImplicitSelectBodyCstChildren, param?: IN): OUT; implicitSelectStatement(children: ImplicitSelectStatementCstChildren, param?: IN): OUT; tableRef(children: TableRefCstChildren, param?: IN): OUT; + aliasTimestampDesignation(children: AliasTimestampDesignationCstChildren, param?: IN): OUT; tableFunctionCall(children: TableFunctionCallCstChildren, param?: IN): OUT; tableFunctionName(children: TableFunctionNameCstChildren, param?: IN): OUT; joinClause(children: JoinClauseCstChildren, param?: IN): OUT; @@ -2984,6 +3031,7 @@ export interface ICstNodeVisitor extends ICstVisitor { alterUserStatement(children: AlterUserStatementCstChildren, param?: IN): OUT; alterServiceAccountStatement(children: AlterServiceAccountStatementCstChildren, param?: IN): OUT; alterUserAction(children: AlterUserActionCstChildren, param?: IN): OUT; + memoryLimit(children: MemoryLimitCstChildren, param?: IN): OUT; alterTableStatement(children: AlterTableStatementCstChildren, param?: IN): OUT; alterTableAction(children: AlterTableActionCstChildren, param?: IN): OUT; convertPartitionTarget(children: ConvertPartitionTargetCstChildren, param?: IN): OUT; diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts index 50ddb39..999ff8e 100644 --- a/src/parser/lexer.ts +++ b/src/parser/lexer.ts @@ -28,6 +28,8 @@ import { Binary, Boolean, BloomFilter, + BloomFilterColumns, + BloomFilterFpp, By, Bypass, Byte, @@ -265,6 +267,7 @@ import { Unbounded, Union, Unlock, + Unlimited, Unnest, Unpivot, Update, @@ -358,6 +361,8 @@ export { Binary, Boolean, BloomFilter, + BloomFilterColumns, + BloomFilterFpp, By, Bypass, Byte, @@ -595,6 +600,7 @@ export { Unbounded, Union, Unlock, + Unlimited, Unnest, Unpivot, Update, diff --git a/src/parser/parser.ts b/src/parser/parser.ts index d20cf53..998bde9 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -355,11 +355,14 @@ import { Alias, Compile, Lateral, + Unlimited, Unnest, Ordinality, // PARQUET clause tokens Parquet, BloomFilter, + BloomFilterColumns, + BloomFilterFpp, Plain, RleDictionary, DeltaBinaryPacked, @@ -928,6 +931,8 @@ class QuestDBParser extends CstParser { const la2 = this.LA(2).tokenType return la2 !== On && la2 !== By } + // Don't consume OWNED as alias when followed by BY (OWNED BY clause) + if (la1 === Owned) return this.LA(2).tokenType !== By // If next token is AS, always allow (explicit alias) if (la1 === As) return true // Don't consume identifier as alias if followed by LParen — @@ -948,8 +953,20 @@ class QuestDBParser extends CstParser { ]) }, }) + // TIMESTAMP designation after the alias: FROM t alias TIMESTAMP(col) + this.OPTION5(() => this.SUBRULE(this.aliasTimestampDesignation)) }) + private aliasTimestampDesignation = this.RULE( + "aliasTimestampDesignation", + () => { + this.CONSUME(Timestamp) + this.CONSUME(LParen) + this.SUBRULE(this.columnRef) + this.CONSUME(RParen) + }, + ) + private tableFunctionCall = this.RULE("tableFunctionCall", () => { this.SUBRULE(this.tableFunctionName) this.CONSUME(LParen) @@ -1106,7 +1123,7 @@ class QuestDBParser extends CstParser { ]) }) - // Standard joins: (INNER | LEFT [OUTER] | CROSS)? JOIN [LATERAL] + ON + // Standard joins: (INNER | LEFT|RIGHT|FULL [OUTER] | CROSS)? JOIN [LATERAL] + ON private standardJoin = this.RULE("standardJoin", () => { this.OPTION(() => { this.OR([ @@ -1116,13 +1133,25 @@ class QuestDBParser extends CstParser { this.OPTION1(() => this.CONSUME(Outer)) }, }, + { + ALT: () => { + this.CONSUME(Right) + this.OPTION4(() => this.CONSUME1(Outer)) + }, + }, + { + ALT: () => { + this.CONSUME(Full) + this.OPTION5(() => this.CONSUME2(Outer)) + }, + }, { ALT: () => this.CONSUME(Inner) }, { ALT: () => this.CONSUME(Cross) }, ]) }) this.CONSUME(Join) this.OPTION3(() => this.CONSUME(Lateral)) - this.SUBRULE(this.tableRef) + this.SUBRULE(this.fromSource) this.OPTION2(() => { this.CONSUME(On) this.SUBRULE(this.expression) @@ -1278,7 +1307,16 @@ class QuestDBParser extends CstParser { this.OR([ { ALT: () => this.CONSUME(Null) }, { ALT: () => this.CONSUME(NumberLiteral) }, - { ALT: () => this.SUBRULE(this.identifier) }, + { + ALT: () => { + this.SUBRULE(this.identifier) + this.OPTION(() => { + this.CONSUME(LParen) + this.SUBRULE1(this.identifier) + this.CONSUME(RParen) + }) + }, + }, ]) }) @@ -2383,6 +2421,11 @@ class QuestDBParser extends CstParser { }, { ALT: () => this.SUBRULE1(this.selectStatement) }, ]) + this.OPTION1(() => { + this.CONSUME(Owned) + this.CONSUME1(By) + this.SUBRULE(this.stringOrIdentifier) + }) }) private dropLiveViewStatement = this.RULE("dropLiveViewStatement", () => { @@ -2528,6 +2571,14 @@ class QuestDBParser extends CstParser { this.CONSUME1(StringLiteral) }, }, + { + ALT: () => { + this.CONSUME(Set) + this.CONSUME(Memory) + this.CONSUME(Limit) + this.SUBRULE(this.memoryLimit) + }, + }, ]) }) @@ -2567,6 +2618,14 @@ class QuestDBParser extends CstParser { this.OR([ { ALT: () => this.CONSUME(Enable) }, { ALT: () => this.CONSUME(Disable) }, + { + ALT: () => { + this.CONSUME(Set) + this.CONSUME(Memory) + this.CONSUME(Limit) + this.SUBRULE(this.memoryLimit) + }, + }, { ALT: () => { this.CONSUME(With) @@ -2648,6 +2707,19 @@ class QuestDBParser extends CstParser { ]) }) + private memoryLimit = this.RULE("memoryLimit", () => { + this.OR([ + { ALT: () => this.CONSUME(Unlimited) }, + { ALT: () => this.CONSUME(DurationLiteral) }, + { + ALT: () => { + this.CONSUME(NumberLiteral) + this.OPTION(() => this.CONSUME(Identifier)) + }, + }, + ]) + }) + private alterTableStatement = this.RULE("alterTableStatement", () => { this.CONSUME(Table) this.SUBRULE(this.tableNameOrString) @@ -2992,6 +3064,17 @@ class QuestDBParser extends CstParser { this.CONSUME(Where) this.SUBRULE(this.expression) }) + // Optional WITH (bloom_filter_columns = '...', bloom_filter_fpp = 0.01) + this.OPTION2(() => { + this.CONSUME(With) + this.CONSUME(LParen) + this.SUBRULE(this.tableParam) + this.MANY1(() => { + this.CONSUME2(Comma) + this.SUBRULE1(this.tableParam) + }) + this.CONSUME(RParen) + }) }) // ========================================================================== @@ -3022,10 +3105,7 @@ class QuestDBParser extends CstParser { ALT: () => { this.CONSUME(Add) this.CONSUME(Index) - this.OPTION(() => { - this.CONSUME(Capacity) - this.CONSUME(NumberLiteral) - }) + this.SUBRULE(this.indexTypeOptions) }, }, { @@ -3717,6 +3797,18 @@ class QuestDBParser extends CstParser { this.SUBRULE2(this.booleanLiteral) }, }, + { + ALT: () => { + this.CONSUME(BloomFilterColumns) + this.SUBRULE3(this.stringOrIdentifier) + }, + }, + { + ALT: () => { + this.CONSUME(BloomFilterFpp) + this.SUBRULE3(this.expression) + }, + }, ]) }) diff --git a/src/parser/toSql.ts b/src/parser/toSql.ts index e0fda05..28d9141 100644 --- a/src/parser/toSql.ts +++ b/src/parser/toSql.ts @@ -362,9 +362,11 @@ function tableRefToSql(ref: AST.TableRef): string { sql = "LATERAL " + sql } - if (ref.timestampDesignation) { - sql += ` TIMESTAMP(${escapeIdentifier(ref.timestampDesignation)})` - } + const designation = ref.timestampDesignation + ? ` TIMESTAMP(${escapeIdentifier(ref.timestampDesignation)})` + : "" + + if (!ref.timestampAfterAlias) sql += designation if (ref.alias) { sql += ` AS ${escapeIdentifier(ref.alias)}` @@ -374,6 +376,8 @@ function tableRefToSql(ref: AST.TableRef): string { sql += `(${ref.columnAliases.map(escapeIdentifier).join(", ")})` } + if (ref.timestampAfterAlias) sql += designation + if (ref.joins) { for (const join of ref.joins) { sql += " " + joinToSql(join) @@ -724,16 +728,16 @@ function createTableToSql(stmt: AST.CreateTableStatement): string { parts.push(storagePolicyToSql(stmt.storagePolicy)) } - if (stmt.tableFormat) { - parts.push(`FORMAT ${stmt.tableFormat.toUpperCase()}`) - } - if (stmt.bypassWal) { parts.push("BYPASS WAL") } else if (stmt.wal) { parts.push("WAL") } + if (stmt.tableFormat) { + parts.push(`FORMAT ${stmt.tableFormat.toUpperCase()}`) + } + if (stmt.withParams && stmt.withParams.length > 0) { const paramParts = stmt.withParams.map((p) => { if (p.value) { @@ -810,7 +814,12 @@ function createLiveViewToSql(stmt: AST.CreateLiveViewStatement): string { parts.push("START FROM BEGINNING") else parts.push(`START FROM ${escapeString(stmt.startFrom.value!)}`) } - parts.push(`AS (${selectToSql(stmt.query)})`) + parts.push( + stmt.asParens + ? `AS (${selectToSql(stmt.query)})` + : `AS ${selectToSql(stmt.query)}`, + ) + if (stmt.ownedBy) parts.push(`OWNED BY ${escapeIdentifier(stmt.ownedBy)}`) return parts.join(" ") } @@ -1026,6 +1035,12 @@ function alterTableToSql(stmt: AST.AlterTableStatement): string { parts.push("WHERE") parts.push(expressionToSql(action.where)) } + if (action.withParams && action.withParams.length > 0) { + const params = action.withParams.map((p) => + p.value ? `${p.name} = ${expressionToSql(p.value)}` : p.name, + ) + parts.push(`WITH (${params.join(", ")})`) + } break } case "setStoragePolicy": @@ -1286,12 +1301,16 @@ function alterMaterializedViewToSql( ] const action = stmt.action switch (action.actionType) { - case "addIndex": { - let s = `ALTER COLUMN ${escapeIdentifier(action.column)} ADD INDEX` - if (action.capacity) s += ` CAPACITY ${action.capacity}` - parts.push(s) + case "addIndex": + parts.push( + `ALTER COLUMN ${escapeIdentifier(action.column)} ADD INDEX` + + indexOptionsToSql( + action.indexType, + action.indexInclude, + action.capacity, + ), + ) break - } case "symbolCapacity": parts.push( `ALTER COLUMN ${escapeIdentifier(action.column)} SYMBOL CAPACITY ${action.capacity}`, @@ -1423,10 +1442,12 @@ function createGroupToSql(stmt: AST.CreateGroupStatement): string { function alterGroupToSql(stmt: AST.AlterGroupStatement): string { const parts: string[] = ["ALTER GROUP", qualifiedNameToSql(stmt.group)] - if (stmt.action === "setAlias") { - parts.push(`WITH EXTERNAL ALIAS ${escapeString(stmt.externalAlias)}`) + if (stmt.action === "setMemoryLimit") { + parts.push(`SET MEMORY LIMIT ${stmt.memoryLimit}`) + } else if (stmt.action === "setAlias") { + parts.push(`WITH EXTERNAL ALIAS ${escapeString(stmt.externalAlias!)}`) } else { - parts.push(`DROP EXTERNAL ALIAS ${escapeString(stmt.externalAlias)}`) + parts.push(`DROP EXTERNAL ALIAS ${escapeString(stmt.externalAlias!)}`) } return parts.join(" ") } @@ -1471,6 +1492,8 @@ function alterUserActionToSql(action: AST.AlterUserAction): string { case "password": if (action.noPassword) return "WITH NO PASSWORD" return `WITH PASSWORD ${escapeString(action.password!)}` + case "setMemoryLimit": + return `SET MEMORY LIMIT ${action.limit}` case "createToken": { const parts = [`CREATE TOKEN TYPE ${action.tokenType}`] if (action.publicKeyX != null && action.publicKeyY != null) { diff --git a/src/parser/tokens.ts b/src/parser/tokens.ts index 55ad1f6..47342bd 100644 --- a/src/parser/tokens.ts +++ b/src/parser/tokens.ts @@ -368,6 +368,8 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Lateral", "Ordinality", "BloomFilter", + "BloomFilterColumns", + "BloomFilterFpp", "Rebase", "Stats", "Switch", @@ -384,6 +386,17 @@ export const IDENTIFIER_KEYWORD_NAMES = new globalThis.Set([ "Anchor", "Beginning", "Memory", + "Unlimited", + // Resource group words (#33): column names in query_activity() and friends + "MemoryLimit", + "CpuWeight", + "MaxActiveQueries", + "MaxQueuedQueries", + "QueueTimeout", + "Mapping", + "Priority", + "Resource", + "Unset", "Daily", "Expression", "Posting", @@ -464,6 +477,8 @@ export const Base = getToken("Base") export const Batch = getToken("Batch") export const Between = getToken("Between") export const BloomFilter = getToken("BloomFilter") +export const BloomFilterColumns = getToken("BloomFilterColumns") +export const BloomFilterFpp = getToken("BloomFilterFpp") export const By = getToken("By") export const Bypass = getToken("Bypass") export const Cache = getToken("Cache") @@ -657,6 +672,7 @@ export const Type = getToken("Type") export const Unbounded = getToken("Unbounded") export const Union = getToken("Union") export const Unlock = getToken("Unlock") +export const Unlimited = getToken("Unlimited") export const Unnest = getToken("Unnest") export const Unpivot = getToken("Unpivot") export const Update = getToken("Update") diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts index e5500eb..18337e6 100644 --- a/src/parser/visitor.ts +++ b/src/parser/visitor.ts @@ -85,6 +85,8 @@ import type { ExpressionCstChildren, FillClauseCstChildren, FillValueCstChildren, + AliasTimestampDesignationCstChildren, + MemoryLimitCstChildren, FromClauseCstChildren, FromSourceCstChildren, FromToClauseCstChildren, @@ -206,6 +208,7 @@ type ConvertPartitionTargetResult = { partitions?: string[] target: string where?: AST.Expression + withParams?: AST.TableParam[] } type PivotBodyResult = { aggregations: AST.PivotAggregation[] @@ -753,9 +756,21 @@ class QuestDBVisitor extends BaseVisitor { result.timestampDesignation = colRef.name.parts.join(".") } + if (ctx.aliasTimestampDesignation) { + result.timestampDesignation = this.visit( + ctx.aliasTimestampDesignation, + ) as string + result.timestampAfterAlias = true + } + return result } + aliasTimestampDesignation(ctx: AliasTimestampDesignationCstChildren): string { + const colRef = this.visit(ctx.columnRef) as AST.ColumnRef + return colRef.name.parts.join(".") + } + tableFunctionCall(ctx: TableFunctionCallCstChildren): AST.TableFunctionCall { let name: string if (ctx.tableFunctionName) { @@ -905,10 +920,12 @@ class QuestDBVisitor extends BaseVisitor { standardJoin(ctx: StandardJoinCstChildren): AST.JoinClause { const result: AST.JoinClause = { type: "join", - table: this.visit(ctx.tableRef) as AST.TableRef, + table: this.visit(ctx.fromSource) as AST.TableRef, } if (ctx.Inner) result.joinType = "inner" else if (ctx.Left) result.joinType = "left" + else if (ctx.Right) result.joinType = "right" + else if (ctx.Full) result.joinType = "full" else if (ctx.Cross) result.joinType = "cross" if (ctx.Outer) result.outer = true if (ctx.Lateral) result.lateral = true @@ -993,8 +1010,14 @@ class QuestDBVisitor extends BaseVisitor { if (ctx.Null) return "NULL" if (ctx.NumberLiteral) return this.tokenImage(ctx.NumberLiteral[0]) if (ctx.identifier) { - const name = this.extractIdentifierName(ctx.identifier[0].children) - return name.toUpperCase() + const name = this.extractIdentifierName( + ctx.identifier[0].children, + ).toUpperCase() + if (ctx.LParen && ctx.identifier[1]) { + const column = this.extractIdentifierName(ctx.identifier[1].children) + return `${name}(${column})` + } + return name } return "" } @@ -1928,6 +1951,10 @@ class QuestDBVisitor extends BaseVisitor { result.startFrom = { kind: "now" } } } + if (ctx.Owned && ctx.stringOrIdentifier) { + result.ownedBy = this.visit(ctx.stringOrIdentifier) as string + } + if (ctx.LParen) result.asParens = true return result } @@ -2086,19 +2113,20 @@ class QuestDBVisitor extends BaseVisitor { alterGroupStatement( ctx: AlterGroupStatementCstChildren, ): AST.AlterGroupStatement { - const alias = ctx.StringLiteral![0].image.slice(1, -1) - if (ctx.With) { + const group = this.visit(ctx.qualifiedName) as AST.QualifiedName + if (ctx.memoryLimit) { return { type: "alterGroup", - group: this.visit(ctx.qualifiedName) as AST.QualifiedName, - action: "setAlias", - externalAlias: alias, + group, + action: "setMemoryLimit", + memoryLimit: this.visit(ctx.memoryLimit) as string, } } + const alias = ctx.StringLiteral![0].image.slice(1, -1) return { type: "alterGroup", - group: this.visit(ctx.qualifiedName) as AST.QualifiedName, - action: "dropAlias", + group, + action: ctx.With ? "setAlias" : "dropAlias", externalAlias: alias, } } @@ -2166,8 +2194,15 @@ class QuestDBVisitor extends BaseVisitor { actionType: "addIndex", column: colRef.name.parts[colRef.name.parts.length - 1], } - if (ctx.Capacity && ctx.NumberLiteral) { - result.capacity = tokenInt(ctx.NumberLiteral[0].image) + if (ctx.indexTypeOptions) { + const opts = this.visit(ctx.indexTypeOptions[0]) as { + capacity?: number + indexType?: AST.AlterMaterializedViewAddIndex["indexType"] + include?: string[] + } + if (opts.capacity !== undefined) result.capacity = opts.capacity + if (opts.indexType) result.indexType = opts.indexType + if (opts.include) result.indexInclude = opts.include } return result } @@ -2263,6 +2298,12 @@ class QuestDBVisitor extends BaseVisitor { } alterUserAction(ctx: AlterUserActionCstChildren): AST.AlterUserAction { + if (ctx.memoryLimit) { + return { + actionType: "setMemoryLimit", + limit: this.visit(ctx.memoryLimit) as string, + } + } if (ctx.Enable) { return { actionType: "enable" } } @@ -2309,6 +2350,12 @@ class QuestDBVisitor extends BaseVisitor { } } + memoryLimit(ctx: MemoryLimitCstChildren): string { + if (ctx.Unlimited) return "UNLIMITED" + if (ctx.DurationLiteral) return ctx.DurationLiteral[0].image + return ctx.NumberLiteral![0].image + (ctx.Identifier?.[0]?.image ?? "") + } + alterTableAction(ctx: AlterTableActionCstChildren): AST.AlterTableAction { // SET STORAGE POLICY(...) — must come before generic SET branches below if (ctx.Set && ctx.storagePolicy) { @@ -3078,6 +3125,8 @@ class QuestDBVisitor extends BaseVisitor { ctx.StatisticsEnabled?.[0] ?? ctx.ParquetVersion?.[0] ?? ctx.RawArrayEncoding?.[0] ?? + ctx.BloomFilterColumns?.[0] ?? + ctx.BloomFilterFpp?.[0] ?? (ctx.On ? ctx.On[0] : undefined) let key = keyToken?.image ?? "OPTION" @@ -3227,16 +3276,10 @@ class QuestDBVisitor extends BaseVisitor { } } - convertPartitionTarget(ctx: ConvertPartitionTargetCstChildren): { - partitions?: string[] - target: string - where?: AST.Expression - } { - const result: { - partitions?: string[] - target: string - where?: AST.Expression - } = { + convertPartitionTarget( + ctx: ConvertPartitionTargetCstChildren, + ): ConvertPartitionTargetResult { + const result: ConvertPartitionTargetResult = { target: "TABLE", } @@ -3261,6 +3304,12 @@ class QuestDBVisitor extends BaseVisitor { result.where = this.visit(ctx.expression[0]) as AST.Expression } + if (ctx.tableParam) { + result.withParams = ctx.tableParam.map( + (p: CstNode) => this.visit(p) as AST.TableParam, + ) + } + return result } diff --git a/tests/content-assist.test.ts b/tests/content-assist.test.ts index 02e72b9..4013d44 100644 --- a/tests/content-assist.test.ts +++ b/tests/content-assist.test.ts @@ -64,8 +64,8 @@ describe("Content Assist", () => { expect(tokens).toContain("Inner") expect(tokens).toContain("Left") expect(tokens).toContain("Cross") - expect(tokens).not.toContain("Right") - expect(tokens).not.toContain("Full") + expect(tokens).toContain("Right") + expect(tokens).toContain("Full") }) it("should suggest WHERE, ORDER BY, etc. after FROM clause", () => { diff --git a/tests/docs-roundtrip.test.ts b/tests/docs-roundtrip.test.ts index 06d7189..9a8c079 100644 --- a/tests/docs-roundtrip.test.ts +++ b/tests/docs-roundtrip.test.ts @@ -66,10 +66,12 @@ function normalizeSql(sql: string): string { s = s.replace(/\s+\]/g, "]") // N2: Normalize arithmetic operator spacing: a+b, a + b → a + b s = s.replace(/\s*([+\-*/%])\s*/g, " $1 ") + // N2b: Normalize DECLARE assignment spacing: @x:=1, @x := 1 → @x := 1 + s = s.replace(/\s*:=\s*/g, " := ") // N3: Normalize VALUES spacing: VALUES( → VALUES ( s = s.replace(/VALUES\s*\(/g, "VALUES (") - // N4: Normalize keyword-paren spacing: KEYS( → KEYS (, JOIN( → JOIN ( - s = s.replace(/([A-Z])\(/g, "$1 (") + // N4: Normalize keyword-paren spacing: KEYS( → KEYS (, JOIN( → JOIN (, T1( → T1 ( + s = s.replace(/([A-Z0-9_])\(/g, "$1 (") // N5: Normalize quoted identifiers: "FOO" → FOO, 'FOO' → FOO (quoting is stylistic) // QuestDB accepts both single and double quotes for identifiers. s = s.replace(/"([^"]+)"/g, "$1") @@ -94,6 +96,14 @@ function normalizeSql(sql: string): string { s = s.replace(/\bPASSWORD\s+'([^']+)'/g, "PASSWORD $1") // N9: Normalize TTL 0 — "TTL 0" and "TTL 0 DAYS" are the same (0 means disabled) s = s.replace(/\bTTL 0 [A-Z]+\b/g, "TTL 0") + // N9b: DROP EXPIRE and DROP EXPIRE ROWS are the same + s = s.replace(/\bDROP EXPIRE ROWS\b/g, "DROP EXPIRE") + // N9c: CREATE TABLE accepts FORMAT in any position among the trailing options; + // compare with FORMAT moved to the end of the statement + s = s.replace( + /^(CREATE TABLE\b.*?) FORMAT (PARQUET|NATIVE)\b(.*)$/, + "$1$3 FORMAT $2", + ) // N10: Normalize singular/plural time units: HOUR → HOURS, DAY → DAYS, etc. s = s.replace(/\b(\d+)\s+(HOUR|DAY|WEEK|MONTH|YEAR)\b/g, "$1 $2S") // N11: Normalize FORMAT/CODEC value quoting: FORMAT 'PARQUET' → FORMAT PARQUET diff --git a/tests/fixtures/docs-queries.json b/tests/fixtures/docs-queries.json index a4cddc1..f59df5d 100644 --- a/tests/fixtures/docs-queries.json +++ b/tests/fixtures/docs-queries.json @@ -4082,10 +4082,10 @@ "query": "CREATE VIEW mixed_params AS (\n DECLARE @fixed := 5, OVERRIDABLE @adjustable := 10\n SELECT * FROM data WHERE a >= @fixed AND b <= @adjustable\n)" }, { - "query": "CREATE VIEW \u65e5\u672c\u8a9e\u30d3\u30e5\u30fc AS (SELECT * FROM trades)" + "query": "CREATE VIEW 日本語ビュー AS (SELECT * FROM trades)" }, { - "query": "CREATE VIEW R\u00e9szv\u00e9ny_\u00e1rak AS (SELECT * FROM prices)" + "query": "CREATE VIEW Részvény_árak AS (SELECT * FROM prices)" }, { "query": "CREATE VIEW with_timestamp AS (\n (SELECT ts, value FROM my_view ORDER BY ts) timestamp(ts)\n)" @@ -4436,7 +4436,7 @@ "skipAutocomplete": true }, { - "query": "-- This would fail if combinations \u00d7 aggregates > 5000\ntrades PIVOT (\n avg(price)\n FOR symbol IN (SELECT DISTINCT symbol FROM trades) -- many symbols\n side IN ('buy', 'sell') -- \u00d7 2\n)", + "query": "-- This would fail if combinations × aggregates > 5000\ntrades PIVOT (\n avg(price)\n FOR symbol IN (SELECT DISTINCT symbol FROM trades) -- many symbols\n side IN ('buy', 'sell') -- × 2\n)", "skipAutocomplete": true }, { @@ -5458,5 +5458,2525 @@ }, { "query": "GRANT CONVERT PARTITION TO PARQUET ON trades TO alice" + }, + { + "query": "CREATE TABLE prices (\n ts TIMESTAMP,\n ticker SYMBOL,\n price DOUBLE\n) TIMESTAMP(ts) PARTITION BY DAY\nDEDUP UPSERT KEYS(ts, ticker)" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING,\n exchange SYMBOL,\n price DOUBLE,\n quantity DOUBLE\n) TIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL,\n exchange SYMBOL,\n price DOUBLE,\n quantity DOUBLE\n), INDEX(symbol TYPE POSTING)\nTIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING INCLUDE (exchange, price),\n exchange SYMBOL,\n price DOUBLE,\n quantity DOUBLE\n) TIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "ALTER TABLE trades\n ALTER COLUMN symbol ADD INDEX TYPE POSTING INCLUDE (exchange, price)" + }, + { + "query": "-- Default adaptive encoding (recommended for most workloads)\nCREATE TABLE t1 (ts TIMESTAMP, s SYMBOL INDEX TYPE POSTING)\n TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "-- Force Elias-Fano only (benchmarking)\nCREATE TABLE t2 (ts TIMESTAMP, s SYMBOL INDEX TYPE POSTING EF)\n TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "-- Force delta + Frame-of-Reference only (benchmarking)\nCREATE TABLE t3 (ts TIMESTAMP, s SYMBOL INDEX TYPE POSTING DELTA)\n TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "-- If your typical queries look like this:\nSELECT timestamp, price, quantity FROM trades WHERE symbol = 'AAPL'" + }, + { + "query": "-- Then include those columns (timestamp is auto-included as designated timestamp):\nCREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING INCLUDE (price, quantity),\n exchange SYMBOL,\n price DOUBLE,\n quantity DOUBLE,\n -- other columns not needed in hot queries\n raw_data VARCHAR,\n metadata VARCHAR\n) TIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "EXPLAIN SELECT timestamp, price FROM trades WHERE symbol = 'AAPL'" + }, + { + "query": "-- Reads from sidecar if price is in INCLUDE\nSELECT price FROM trades WHERE symbol = 'AAPL'" + }, + { + "query": "-- Covering index + filter on covered column\nSELECT price FROM trades WHERE symbol = 'AAPL' AND price > 100" + }, + { + "query": "-- Multiple keys, still uses covering index\nSELECT price FROM trades WHERE symbol IN ('AAPL', 'GOOGL', 'MSFT')" + }, + { + "query": "-- Latest row per symbol, reads from sidecar\nSELECT timestamp, symbol, price\nFROM trades\nWHERE symbol = 'AAPL'\nLATEST ON timestamp PARTITION BY symbol" + }, + { + "query": "-- Enumerates keys from index metadata, O(keys x partitions) instead of full scan\nSELECT DISTINCT symbol FROM trades" + }, + { + "query": "-- Also works with timestamp filters\nSELECT DISTINCT symbol FROM trades WHERE timestamp > '2024-01-01'" + }, + { + "query": "-- Plan: Count over CoveringIndex, no column data read\nSELECT COUNT(*) FROM trades WHERE symbol = 'AAPL'" + }, + { + "query": "-- Aggregates over a covered column read from the sidecar instead of\n-- the column file\nSELECT count(*), min(price), max(price)\nFROM trades\nWHERE symbol = 'AAPL'" + }, + { + "query": "SELECT timestamp, symbol, avg(price)\nFROM trades\nWHERE timestamp IN '$today'\nSAMPLE BY 1h" + }, + { + "query": "SELECT * FROM trades WHERE timestamp IN '$now - 1h..$now'" + }, + { + "query": "SELECT * FROM trades WHERE timestamp IN '$today'" + }, + { + "query": "SELECT * FROM trades WHERE timestamp IN '[2025-01]#XNYS'" + }, + { + "query": "SELECT t.timestamp, t.symbol, t.price, p.bid_price, p.ask_price\nFROM fx_trades AS t\nASOF JOIN core_price AS p ON (symbol)\nWHERE t.timestamp IN '$now-1h..$now'" + }, + { + "query": "SELECT\n h.offset / 1_000_000_000 AS horizon_sec,\n t.symbol,\n avg((m.best_bid + m.best_ask) / 2 - t.price) AS avg_markout\nFROM fx_trades AS t\nHORIZON JOIN market_data AS m ON (symbol)\nLIST (0, 5s, 30s, 1m) AS h\nWHERE t.timestamp IN '$now-1h..$now'\nORDER BY t.symbol, horizon_sec" + }, + { + "query": "SELECT symbol, side, sum(price)\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "(\n SELECT symbol, side, sum(price) AS total_price\n FROM trades WHERE timestamp IN '$today'\n)\nWHERE total_price > 10_000_000" + }, + { + "query": "DECLARE\n @symbol := 'BTC-USDT',\n @window := '$now - 1d..$now'\nSELECT timestamp, price FROM trades\nWHERE symbol = @symbol AND timestamp IN @window" + }, + { + "query": "CREATE VIEW expensive_trades AS (\n DECLARE OVERRIDABLE @min_price := 100\n SELECT * FROM trades WHERE price >= @min_price\n)" + }, + { + "query": "-- Override at query time\nDECLARE @min_price := 500 SELECT * FROM expensive_trades" + }, + { + "query": "CREATE TABLE trades (\n ts TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING INCLUDE (price, amount),\n price DOUBLE,\n amount DOUBLE\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "SELECT value FROM UNNEST(ARRAY[1.0, 2.0, 3.0])" + }, + { + "query": "SELECT u.trade_id, u.price, u.size, u.side, u.time\nFROM UNNEST(\n '[{\"trade_id\":994619709,\"side\":\"sell\",\"size\":\"0.00000100\",\"price\":\"69839.36\",\"time\":\"2026-04-06T10:32:55.517183Z\"},\n {\"trade_id\":994619708,\"side\":\"buy\",\"size\":\"0.00000006\",\"price\":\"69839.35\",\"time\":\"2026-04-06T10:32:55.418434Z\"}]'::VARCHAR\n COLUMNS(trade_id LONG, price DOUBLE, size DOUBLE, side VARCHAR, time TIMESTAMP)\n) u" + }, + { + "query": "SELECT id, order_ts, sym, price, sec_offs, order_ts + usec_offs AS ts\nFROM orders CROSS JOIN offsets\nORDER BY order_ts + usec_offs" + }, + { + "query": "SELECT /*+ markout_horizon(orders offsets) */ sum(price)\nFROM (SELECT * FROM (\n SELECT price, ts + usec_offs AS timestamp\n FROM orders CROSS JOIN offsets\n ORDER BY ts + usec_offs\n) TIMESTAMP(timestamp))" + }, + { + "query": "CREATE TABLE trades (\n ts TIMESTAMP,\n symbol SYMBOL,\n side SYMBOL,\n price DOUBLE,\n qty DOUBLE\n) TIMESTAMP(ts) PARTITION BY DAY\nDEDUP UPSERT KEYS(ts, symbol, side)" + }, + { + "query": "-- NYSE trading hours on workdays for January (22 intervals, one query)\nSELECT * FROM trades\nWHERE ts IN '[2024-01]T09:30@America/New_York#workday;6h30m'" + }, + { + "query": "-- Subquery loses designated timestamp\n-- Solution: Apply TIMESTAMP() to the subquery result\nWITH recent AS (\n (SELECT * FROM trades WHERE timestamp > dateadd('d', -7, now()))\n TIMESTAMP(timestamp)\n)\nSELECT * FROM recent SAMPLE BY 1h" + }, + { + "query": "CREATE LIVE VIEW trades_ma\nFLUSH EVERY 1s\nIN MEMORY 5s\nSTART FROM NOW\nAS\nSELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS 300 PRECEDING\n ) AS moving_avg\nFROM trades" + }, + { + "query": "SELECT * FROM trades_ma" + }, + { + "query": "SHOW CREATE LIVE VIEW trades_ma" + }, + { + "query": "CREATE LIVE VIEW trades_daily_volume\nFLUSH EVERY 1s\nSTART FROM NOW\nAS\nSELECT\n timestamp,\n symbol,\n sum(amount) OVER w AS cumulative_volume\nFROM trades\nWINDOW w AS (\n PARTITION BY symbol\n ORDER BY timestamp\n ANCHOR DAILY '00:00'\n)" + }, + { + "query": "CREATE LIVE VIEW trades_ma\nFLUSH EVERY 1s\nSTART FROM BEGINNING\nAS\nSELECT\n timestamp,\n symbol,\n avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING)\n AS moving_avg\nFROM trades" + }, + { + "query": "SELECT view_name, base_table_name, view_status, lag_seqtxn, lag_micros\nFROM live_views()" + }, + { + "query": "REFRESH MATERIALIZED VIEW my_view" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL,\n price DOUBLE,\n amount DOUBLE\n) TIMESTAMP(timestamp)\nPARTITION BY DAY\nFORMAT PARQUET" + }, + { + "query": "ALTER TABLE market_data SET STORAGE POLICY(TO PARQUET 1h)" + }, + { + "query": "ALTER TABLE trades CONVERT PARTITION TO PARQUET\nWHERE timestamp < '2025-08-31'\nWITH (bloom_filter_columns = 'symbol,side', bloom_filter_fpp = 0.01)" + }, + { + "query": "COPY market_data TO 'market_data_single' WITH FORMAT PARQUET PARTITION_BY NONE" + }, + { + "query": "COPY market_data TO 'market_data_monthly' WITH FORMAT PARQUET PARTITION_BY MONTH" + }, + { + "query": "COPY (SELECT * FROM market_data WHERE timestamp IN '2024')\nTO 'market_data_2024'\nWITH FORMAT PARQUET PARTITION_BY MONTH" + }, + { + "query": "-- Instead of:\n-- ALTER TABLE trades SET TTL 30 DAYS;\n\n-- Use:\nALTER TABLE trades SET STORAGE POLICY(DROP LOCAL 30d)" + }, + { + "query": "CREATE TABLE trades (\n ts TIMESTAMP,\n symbol SYMBOL,\n price DOUBLE\n) TIMESTAMP(ts) PARTITION BY DAY\n STORAGE POLICY(TO PARQUET 3d, DROP LOCAL 1M)" + }, + { + "query": "ALTER TABLE trades SET STORAGE POLICY(\n TO PARQUET 3 DAYS,\n DROP LOCAL 1 MONTH\n)" + }, + { + "query": "SELECT * FROM storage_policies" + }, + { + "query": "GRANT SET STORAGE POLICY ON trades TO analyst" + }, + { + "query": "GRANT REMOVE STORAGE POLICY ON trades TO admin" + }, + { + "query": "SELECT table_dir_name, to_parquet, drop_local, status\nFROM storage_policies\nWHERE table_dir_name LIKE 'trades%'" + }, + { + "query": "CREATE TABLE 'trades' (\n ts TIMESTAMP,\n symbol SYMBOL CAPACITY 256 CACHE,\n price DOUBLE\n) timestamp(ts) PARTITION BY DAY\nSTORAGE POLICY(TO PARQUET 1 DAY) WAL" + }, + { + "query": "-- Posting index with covering columns — reads from compact sidecar files\nCREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING INCLUDE (price),\n price DOUBLE\n) TIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "-- London business hours (09:00-17:00) for January workdays\nSELECT * FROM trades\nWHERE ts IN '[2024-01]T09:00@Europe/London#wd;8h'" + }, + { + "query": "-- NYSE trading hours (09:30-16:00 Eastern)\nSELECT * FROM trades\nWHERE ts IN '[2024-01]T09:30@America/New_York#wd;6h30m'" + }, + { + "query": "-- On an existing view\nALTER MATERIALIZED VIEW trades_hourly SET TTL 7 DAYS" + }, + { + "query": "ALTER TABLE trades SET TTL 0h" + }, + { + "query": "SELECT * FROM my_view" + }, + { + "query": "SELECT ts, price FROM my_view WHERE symbol = 'AAPL'" + }, + { + "query": "SELECT v1.ts, v2.value\nFROM view1 v1\nJOIN view2 v2 ON v1.id = v2.id" + }, + { + "query": "-- View definition\nCREATE VIEW trades_view AS (\n SELECT ts, symbol, price, quantity FROM trades WHERE price > 0\n)" + }, + { + "query": "-- This query is optimized as if written inline\nSELECT ts, price FROM trades_view WHERE symbol = 'AAPL' ORDER BY ts\n-- Optimizer sees: SELECT ts, price FROM trades WHERE price > 0 AND symbol = 'AAPL' ORDER BY ts\n-- Only ts and price columns are read, filters applied at scan, ordering uses index" + }, + { + "query": "CREATE VIEW price_range AS (\n DECLARE OVERRIDABLE @lo := 100, OVERRIDABLE @hi := 1000\n SELECT ts, symbol, price FROM trades WHERE price >= @lo AND price <= @hi\n)" + }, + { + "query": "-- Query with custom range\nDECLARE @lo := 50, @hi := 200 SELECT * FROM price_range" + }, + { + "query": "CREATE VIEW mixed_params AS (\n DECLARE @fixed_filter := 'active', OVERRIDABLE @limit := 100\n SELECT * FROM data WHERE status = @fixed_filter LIMIT @limit\n)" + }, + { + "query": "-- @limit can be overridden, @fixed_filter cannot\nDECLARE @limit := 50 SELECT * FROM mixed_params" + }, + { + "query": "-- Level 1: Raw data filtering\nCREATE VIEW valid_trades AS (\n SELECT * FROM trades WHERE price > 0 AND quantity > 0\n)" + }, + { + "query": "-- Level 2: Aggregation\nCREATE VIEW hourly_stats AS (\n SELECT ts, symbol, sum(quantity) as volume\n FROM valid_trades\n SAMPLE BY 1h\n)" + }, + { + "query": "-- Level 3: Derived metrics\nCREATE VIEW hourly_vwap AS (\n SELECT ts, symbol, volume, turnover / volume as vwap\n FROM hourly_stats\n WHERE volume > 0\n)" + }, + { + "query": "DROP VIEW my_view" + }, + { + "query": "-- Or safely:\nDROP VIEW IF EXISTS my_view" + }, + { + "query": "SELECT symbol, avg(price) AS avg_price\nFROM trades\nWHERE timestamp IN '2026-09-14'\nSAMPLE BY 1m" + }, + { + "query": "SELECT query_id, memory_used, memory_limit, query\nFROM query_activity()\nWHERE query LIKE 'SELECT symbol, avg(price)%'" + }, + { + "query": "-- List all tables\nSHOW TABLES" + }, + { + "query": "-- Meta-query: full table metadata including designated timestamp\nSELECT * FROM tables()" + }, + { + "query": "CREATE TABLE IF NOT EXISTS 'trades' (\n symbol SYMBOL capacity 256 CACHE,\n side SYMBOL capacity 256 CACHE,\n price DOUBLE,\n amount DOUBLE,\n my_ts TIMESTAMP\n) timestamp (my_ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n trade_id LONG,\n symbol SYMBOL,\n price DOUBLE,\n volume LONG\n) TIMESTAMP(timestamp) PARTITION BY DAY\nDEDUP UPSERT KEYS(timestamp, trade_id)" + }, + { + "query": "SELECT count() AS out_of_order_rows\nFROM (\n SELECT price AS current_price,\n lag(price) OVER (ORDER BY amount) AS previous_price\n FROM trades\n WHERE symbol = 'BTC-USDT' AND timestamp IN '$today'\n)\nWHERE current_price < previous_price" + }, + { + "query": "WITH column_and_prev AS (\n SELECT row_number() OVER (ORDER BY amount) AS rownum,\n amount,\n price AS current_price,\n lag(price) OVER (ORDER BY amount) AS previous_price\n FROM trades\n WHERE symbol = 'BTC-USDT' AND timestamp IN '$today'\n)\nSELECT rownum, amount, current_price, previous_price\nFROM column_and_prev\nWHERE current_price < previous_price\nORDER BY rownum\nLIMIT 1" + }, + { + "query": "WITH\nmax_min AS (\nSELECT max(price), avg(price), min(price)\nFROM trades WHERE timestamp IN '2024-12-08'\n)\nSELECT max(count_sec), max_min.* FROM (\n SELECT count() as count_sec FROM trades\n WHERE timestamp IN '2024-12-08'\n SAMPLE BY 1s\n) CROSS JOIN max_min" + }, + { + "query": "CREATE VIEW lmax_trades AS (\n SELECT timestamp, symbol, side, price, quantity, counterparty\n FROM fx_trades\n WHERE ecn = 'LMAX'\n)" + }, + { + "query": "CREATE GROUP lmax_desk" + }, + { + "query": "GRANT SELECT ON lmax_trades TO lmax_desk" + }, + { + "query": "SELECT * FROM lmax_trades" + }, + { + "query": "-- works, LMAX rows only\nSELECT * FROM fx_trades" + }, + { + "query": "CREATE VIEW desk_trades AS (\n DECLARE OVERRIDABLE @ecn := ''\n SELECT timestamp, symbol, side, price, quantity, counterparty\n FROM fx_trades\n WHERE ecn = @ecn\n)" + }, + { + "query": "CREATE VIEW lmax_trades AS (\n DECLARE @ecn := 'LMAX'\n SELECT * FROM desk_trades\n)" + }, + { + "query": "CREATE VIEW ebs_trades AS (\n DECLARE @ecn := 'EBS'\n SELECT * FROM desk_trades\n)" + }, + { + "query": "CREATE GROUP ebs_desk" + }, + { + "query": "GRANT SELECT ON ebs_trades TO ebs_desk" + }, + { + "query": "ADD USER trader_jane TO lmax_desk" + }, + { + "query": "CREATE SERVICE ACCOUNT lmax_dashboard OWNED BY lmax_desk" + }, + { + "query": "GRANT SELECT ON lmax_trades TO lmax_dashboard" + }, + { + "query": "ALTER TABLE fx_trades\n ALTER COLUMN ecn ADD INDEX TYPE POSTING\n INCLUDE (symbol, side, price, quantity, counterparty)" + }, + { + "query": "CREATE TABLE events (\n visitor_id SYMBOL,\n pathname SYMBOL,\n timestamp TIMESTAMP,\n metric_name SYMBOL\n) TIMESTAMP(timestamp) PARTITION BY MONTH" + }, + { + "query": "WITH PrevEvents AS (\n SELECT\n visitor_id,\n pathname,\n timestamp,\n lag(timestamp) OVER (PARTITION BY visitor_id ORDER BY timestamp) AS prev_ts\n FROM\n events WHERE timestamp IN '$now - 7d..$now'\n AND metric_name = 'page_view'\n), VisitorSessions AS (\n SELECT *,\n SUM(CASE WHEN datediff('h', timestamp, prev_ts) > 1 THEN 1 END)\n OVER(\n PARTITION BY visitor_id\n ORDER BY timestamp\n ) as local_session_id FROM PrevEvents\n\n), GlobalSessions AS (\n SELECT visitor_id, pathname, timestamp, prev_ts,\n concat(visitor_id, '#', coalesce(local_session_id,0)::int) AS session_id\n FROM VisitorSessions\n), EventSequences AS (\n SELECT *, row_number() OVER (\n PARTITION BY session_id ORDER BY timestamp\n ) as session_sequence,\n row_number() OVER (\n PARTITION BY session_id ORDER BY timestamp DESC\n ) as reverse_session_sequence,\n first_value(timestamp::long) OVER (\n PARTITION BY session_id ORDER BY timestamp\n ) as session_ts\n FROM GlobalSessions\n), EventsFullInfo AS (\n SELECT e1.session_id, e1.session_ts::timestamp as session_ts, e1.visitor_id,\n e1.timestamp, e1.pathname, e1.session_sequence,\n CASE WHEN e1.session_sequence = 1 THEN true END is_entry_page,\n e2.pathname as next_pathname, datediff('T', e1.timestamp, e1.prev_ts)::double as elapsed,\n e2.reverse_session_sequence,\n CASE WHEN e2.reverse_session_sequence = 1 THEN true END is_exit_page\n FROM EventSequences e1\n LEFT JOIN EventSequences e2 ON (e1.session_id = e2.session_id)\n WHERE e2.session_sequence - e1.session_sequence = 1\n)\nSELECT * FROM EventsFullInfo" + }, + { + "query": "WITH pivoted AS (\n SELECT\n timestamp,\n symbol,\n CASE WHEN side = 'buy' THEN price END as buy,\n CASE WHEN side = 'sell' THEN price END as sell\n FROM trades\n WHERE timestamp IN '$now - 5m..$now'\n AND symbol = 'ETH-USDT'\n),\nunpivoted AS (\n SELECT timestamp, symbol, 'buy' as side, buy as price\n FROM pivoted\n\n UNION ALL\n\n SELECT timestamp, symbol, 'sell' as side, sell as price\n FROM pivoted\n)\nSELECT * FROM unpivoted\nWHERE price IS NOT NULL\nORDER BY timestamp" + }, + { + "query": "WITH sensor_data AS (\n SELECT\n timestamp,\n sensor_id,\n temperature,\n humidity,\n pressure\n FROM sensors\n WHERE timestamp IN '$now - 1h..$now'\n)\nSELECT timestamp, sensor_id, 'temperature' as metric, temperature as value FROM sensor_data\nWHERE temperature IS NOT NULL\n\nUNION ALL\n\nSELECT timestamp, sensor_id, 'humidity' as metric, humidity as value FROM sensor_data\nWHERE humidity IS NOT NULL\n\nUNION ALL\n\nSELECT timestamp, sensor_id, 'pressure' as metric, pressure as value FROM sensor_data\nWHERE pressure IS NOT NULL\n\nORDER BY timestamp, sensor_id, metric" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH with_prev AS (\n SELECT\n timestamp,\n symbol,\n high,\n low,\n close,\n lag(close) OVER (PARTITION BY symbol ORDER BY timestamp) AS prev_close\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n),\ntrue_range AS (\n SELECT\n timestamp,\n symbol,\n high,\n low,\n close,\n greatest(\n high - low,\n abs(high - prev_close),\n abs(low - prev_close)\n ) AS tr\n FROM with_prev\n WHERE prev_close IS NOT NULL\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(tr, 6) AS true_range,\n round(avg(tr, 'period', 14) OVER (PARTITION BY symbol ORDER BY timestamp), 6) AS atr\nFROM true_range\nORDER BY timestamp" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1h..$now'\n\nSELECT\n timestamp,\n symbol,\n round(bid_price, 5) AS bid,\n round(ask_price, 5) AS ask,\n round(ask_price - bid_price, 6) AS spread_absolute,\n round((ask_price - bid_price) / ((bid_price + ask_price) / 2) * 10000, 2) AS spread_bps,\n round((bid_price + ask_price) / 2, 5) AS mid_price\nFROM core_price\nWHERE symbol = @symbol\n AND timestamp IN @lookback\nORDER BY timestamp" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1d..$now'\n\nSELECT\n timestamp,\n symbol,\n round(avg((ask_price - bid_price) / ((bid_price + ask_price) / 2) * 10000), 2) AS avg_spread_bps,\n round(min((ask_price - bid_price) / ((bid_price + ask_price) / 2) * 10000), 2) AS min_spread_bps,\n round(max((ask_price - bid_price) / ((bid_price + ask_price) / 2) * 10000), 2) AS max_spread_bps,\n count() AS quote_count\nFROM core_price\nWHERE symbol = @symbol\n AND timestamp IN @lookback\nSAMPLE BY 1h\nORDER BY timestamp" + }, + { + "query": "WITH OHLC AS (\n SELECT\n timestamp, symbol,\n first(price) AS open,\n max(price) as high,\n min(price) as low,\n last(price) AS close,\n sum(quantity) AS volume\n FROM fx_trades\n WHERE symbol = 'EURUSD' AND timestamp IN '$yesterday'\n SAMPLE BY 15m\n), stats AS (\n SELECT\n timestamp,\n close,\n AVG(close) OVER w AS sma20,\n AVG(close * close) OVER w AS avg_close_sq\n FROM OHLC\n WINDOW w AS (ORDER BY timestamp ROWS 19 PRECEDING)\n)\nSELECT\n timestamp,\n close,\n sma20,\n sqrt(avg_close_sq - (sma20 * sma20)) as stdev20,\n sma20 + 2 * sqrt(avg_close_sq - (sma20 * sma20)) as upper_band,\n sma20 - 2 * sqrt(avg_close_sq - (sma20 * sma20)) as lower_band\nFROM stats\nORDER BY timestamp" + }, + { + "query": "WITH OHLC AS (\n SELECT\n timestamp, symbol,\n first(price) AS open,\n last(price) AS close,\n sum(quantity) AS volume\n FROM fx_trades\n WHERE symbol IN ('EURUSD', 'GBPUSD')\n AND timestamp IN '$yesterday'\n SAMPLE BY 15m\n), stats AS (\n SELECT\n timestamp,\n symbol,\n close,\n AVG(close) OVER w AS sma20,\n AVG(close * close) OVER w AS avg_close_sq\n FROM OHLC\n WINDOW w AS (PARTITION BY symbol ORDER BY timestamp ROWS 19 PRECEDING)\n)\nSELECT\n timestamp,\n symbol,\n close,\n sma20,\n sma20 + 2 * sqrt(avg_close_sq - (sma20 * sma20)) as upper_band,\n sma20 - 2 * sqrt(avg_close_sq - (sma20 * sma20)) as lower_band\nFROM stats\nORDER BY symbol, timestamp" + }, + { + "query": "DECLARE\n @symbol := 'BTC-USDT',\n @history := '$now - 6M..$now',\n @display := '$now - 1M..$now'\n\nWITH daily_ohlc AS (\n SELECT\n timestamp,\n symbol,\n first(open) AS open,\n max(high) AS high,\n min(low) AS low,\n last(close) AS close\n FROM trades_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @history\n SAMPLE BY 1d\n),\nbands AS (\n SELECT\n timestamp,\n symbol,\n close,\n AVG(close) OVER w AS sma20,\n AVG(close * close) OVER w AS avg_close_sq\n FROM daily_ohlc\n WINDOW w AS (PARTITION BY symbol ORDER BY timestamp ROWS 19 PRECEDING)\n),\nbollinger AS (\n SELECT\n timestamp,\n symbol,\n close,\n sma20,\n sma20 + 2 * sqrt(avg_close_sq - (sma20 * sma20)) AS upper_band,\n sma20 - 2 * sqrt(avg_close_sq - (sma20 * sma20)) AS lower_band\n FROM bands\n),\nwith_bandwidth AS (\n SELECT\n timestamp,\n symbol,\n close,\n sma20,\n upper_band,\n lower_band,\n (upper_band - lower_band) / sma20 * 100 AS bandwidth\n FROM bollinger\n),\nwith_range AS (\n SELECT\n timestamp,\n symbol,\n close,\n sma20,\n upper_band,\n lower_band,\n bandwidth,\n min(bandwidth) OVER w AS min_bw,\n max(bandwidth) OVER w AS max_bw\n FROM with_bandwidth\n WINDOW w AS (PARTITION BY symbol)\n)\nSELECT\n timestamp,\n symbol,\n round(close, 2) AS close,\n round(sma20, 2) AS sma20,\n round(upper_band, 2) AS upper_band,\n round(lower_band, 2) AS lower_band,\n round(bandwidth, 4) AS bandwidth,\n round((bandwidth - min_bw) / (max_bw - min_bw) * 100, 1) AS range_position\nFROM with_range\nWHERE timestamp IN @display\nORDER BY timestamp" + }, + { + "query": "WITH\nyear_series AS (\n DECLARE @year:=2000,\n @rate := 0.1,\n @principal := 1000.0\n SELECT @year as start_year, @year + (x - 1) AS timestamp,\n @rate AS interest_rate, @principal as initial_principal\n FROM long_sequence(5) -- number of years\n),\ncompounded_values AS (\n SELECT\n timestamp,\n initial_principal,\n interest_rate,\n initial_principal *\n POWER(\n 1 + interest_rate,\n timestamp - start_year + 1\n ) AS compounding\n FROM\n year_series\n), compounding_year_before AS (\nSELECT\n timestamp,\n initial_principal,\n interest_rate,\n LAG(cv.compounding) OVER (ORDER BY timestamp) AS year_principal,\n cv.compounding as compounding_amount\nFROM\n compounded_values cv\nORDER BY\n timestamp\n )\nselect timestamp, initial_principal, interest_rate,\ncoalesce(year_principal, initial_principal) as year_principal,\ncompounding_amount\nfrom compounding_year_before" + }, + { + "query": "-- For hourly returns\nWITH ln_values AS (\n SELECT\n timestamp,\n return,\n SUM(ln(1 + return)) OVER (ORDER BY timestamp) AS ln_value\n FROM hourly_returns\n)\nSELECT timestamp, 100 * exp(ln_value) AS price FROM ln_values" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH channels AS (\n SELECT\n timestamp,\n symbol,\n close,\n max(high) OVER w AS upper_channel,\n min(low) OVER w AS lower_channel\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n WINDOW w AS (\n PARTITION BY symbol ORDER BY timestamp\n ROWS BETWEEN 19 PRECEDING AND CURRENT ROW\n )\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(upper_channel, 5) AS upper_channel,\n round(lower_channel, 5) AS lower_channel,\n round((upper_channel + lower_channel) / 2, 5) AS middle_channel\nFROM channels\nORDER BY timestamp" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n count() AS fill_count,\n sum(t.quantity) AS total_volume,\n avg(t.quantity) AS avg_fill_size,\n avg((m.best_ask - m.best_bid)\n / ((m.best_bid + m.best_ask) / 2) * 10000) AS avg_spread_bps,\n avg(((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000) AS avg_slippage_bps,\n avg((m.best_ask - t.price)\n / t.price * 10000) AS avg_slippage_vs_ask_bps,\n avg(CASE WHEN t.passive THEN 1.0 ELSE 0.0 END) AS passive_ratio\nFROM fx_trades t\nASOF JOIN market_data m ON (symbol)\nWHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\nGROUP BY t.symbol, t.ecn\nORDER BY t.symbol, avg_slippage_bps" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n avg(((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000) AS avg_markout_bps,\n sum(((m.best_bid + m.best_ask) / 2 - t.price)\n * t.quantity) AS total_pnl\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n RANGE FROM 0s TO 5m STEP 5s AS h\nWHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\nGROUP BY t.symbol, t.ecn, horizon_sec\nORDER BY t.symbol, t.ecn, horizon_sec" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n hour(t.timestamp) AS hour_utc,\n h.offset,\n count() AS n,\n avg(((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000) AS markout_5s_bps,\n avg((m.best_ask - m.best_bid)\n / ((m.best_bid + m.best_ask) / 2) * 10000) AS avg_spread_bps\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n LIST (5s) AS h\nWHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\nGROUP BY t.symbol, t.ecn, hour(t.timestamp), h.offset\nORDER BY t.symbol, t.ecn, hour_utc" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n t.passive,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n avg(((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000) AS avg_markout_bps\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n LIST (0, 1s, 5s, 10s, 1m) AS h\nWHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\nGROUP BY t.symbol, t.ecn, t.passive, horizon_sec\nORDER BY t.symbol, t.ecn, t.passive, horizon_sec" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n h.offset,\n count() AS fill_count,\n sum(t.quantity) AS total_volume,\n sum(((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000 * t.quantity)\n / sum(t.quantity) AS vw_markout_5s_bps,\n avg(CASE\n WHEN (m.best_bid + m.best_ask) / 2 < t.price THEN 1.0\n ELSE 0.0\n END) AS adverse_fill_ratio\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n LIST (5s) AS h\nWHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\nGROUP BY t.symbol, t.ecn, h.offset\nORDER BY t.symbol, vw_markout_5s_bps" + }, + { + "query": "WITH markouts AS (\n SELECT\n t.symbol,\n t.ecn,\n t.price,\n t.quantity,\n h.offset,\n m.best_bid,\n m.best_ask\n FROM fx_trades t\n HORIZON JOIN market_data m ON (symbol)\n LIST (0, 5s, 1m) AS h\n WHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\n)\nSELECT * FROM markouts\nPIVOT (\n count() AS fills,\n avg(quantity) AS avg_size,\n sum(quantity) AS volume,\n avg(((best_bid + best_ask) / 2 - price) / price * 10000) AS markout_bps\n FOR offset IN (0 AS at_fill, 5000000000 AS t_5s, 60000000000 AS t_1m)\n GROUP BY symbol, ecn\n)\nORDER BY t_5s_markout_bps" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1d..$now'\n\nWITH sampled AS (\n SELECT\n timestamp,\n symbol,\n last((bid_price + ask_price) / 2) AS close_mid,\n avg((ask_price - bid_price) /\n ((bid_price + ask_price) / 2)) AS avg_rel_spread\n FROM core_price\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n SAMPLE BY 1m ALIGN TO CALENDAR\n),\nreturns AS (\n SELECT\n timestamp,\n symbol,\n close_mid,\n avg_rel_spread,\n LN(close_mid / LAG(close_mid)\n OVER (PARTITION BY symbol ORDER BY timestamp))\n AS log_ret\n FROM sampled\n)\nSELECT\n timestamp,\n symbol,\n round(AVG(avg_rel_spread) OVER w * 10000, 2)\n AS rel_spread_1h_bps,\n round(SQRT(\n (AVG(log_ret * log_ret) OVER w -\n AVG(log_ret) OVER w * AVG(log_ret) OVER w) * 1440 * 365\n ) * 100, 2) AS realized_vol_1h_ann,\n CASE\n WHEN AVG(avg_rel_spread) OVER w > 0 THEN\n round(SQRT(\n (AVG(log_ret * log_ret) OVER w -\n AVG(log_ret) OVER w\n * AVG(log_ret) OVER w) * 1440\n ) / AVG(avg_rel_spread) OVER w, 2)\n ELSE NULL\n END AS vol_spread_ratio\nFROM returns\nWHERE log_ret IS NOT NULL\nWINDOW w AS (\n PARTITION BY symbol ORDER BY timestamp ROWS 59 PRECEDING\n)" + }, + { + "query": "WITH fills_enriched AS (\n SELECT\n f.order_id,\n f.symbol,\n f.side,\n f.price,\n f.quantity,\n f.timestamp,\n (m.best_bid + m.best_ask) / 2 AS mid_at_fill\n FROM fx_trades f\n ASOF JOIN market_data m ON (symbol)\n WHERE f.timestamp IN '$yesterday'\n),\norder_summary AS (\n SELECT\n order_id,\n symbol,\n side,\n first(mid_at_fill) AS arrival_mid,\n sum(price * quantity) / sum(quantity) AS avg_exec_price,\n sum(quantity) AS total_qty,\n count() AS n_fills,\n min(timestamp) AS first_fill_ts,\n max(timestamp) AS last_fill_ts\n FROM fills_enriched\n GROUP BY order_id, symbol, side\n)\nSELECT\n order_id,\n symbol,\n side,\n n_fills,\n total_qty,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (avg_exec_price - arrival_mid)\n / arrival_mid * 10000 AS total_is_bps\nFROM order_summary\nORDER BY total_is_bps DESC" + }, + { + "query": "WITH fills_enriched AS (\n SELECT\n f.order_id,\n f.symbol,\n f.side,\n f.price,\n f.quantity,\n f.timestamp,\n (m.best_bid + m.best_ask) / 2 AS mid_at_fill\n FROM fx_trades f\n ASOF JOIN market_data m ON (symbol)\n WHERE f.timestamp IN '$yesterday'\n),\norder_bounds AS (\n SELECT\n order_id,\n symbol,\n side,\n first(mid_at_fill) AS arrival_mid,\n last(mid_at_fill) AS mid_at_last_fill,\n min(timestamp) AS first_fill_ts,\n max(timestamp) AS last_fill_ts\n FROM fills_enriched\n GROUP BY order_id, symbol, side\n)\nSELECT\n order_id,\n symbol,\n side,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (mid_at_last_fill - arrival_mid)\n / arrival_mid * 10000 AS execution_drift_bps,\n last_fill_ts - first_fill_ts AS execution_duration\nFROM order_bounds\nORDER BY execution_drift_bps DESC" + }, + { + "query": "WITH fills_enriched AS (\n SELECT\n f.order_id,\n f.symbol,\n f.side,\n f.price,\n f.quantity,\n m.best_ask - m.best_bid AS spread_at_fill\n FROM fx_trades f\n ASOF JOIN market_data m ON (symbol)\n WHERE f.timestamp IN '$yesterday'\n)\nSELECT\n order_id,\n symbol,\n sum(0.5 * spread_at_fill * quantity)\n / sum(quantity) AS avg_halfspread,\n sum(0.5 * spread_at_fill / price * 10000 * quantity)\n / sum(quantity) AS spread_cost_bps,\n sum(quantity) AS total_qty\nFROM fills_enriched\nGROUP BY order_id, symbol\nORDER BY spread_cost_bps DESC" + }, + { + "query": "WITH order_markouts AS (\n SELECT\n f.order_id,\n f.symbol,\n f.side,\n h.offset,\n sum((m.best_bid + m.best_ask) / 2 * f.quantity)\n / sum(f.quantity) AS weighted_mid,\n sum(f.price * f.quantity) / sum(f.quantity) AS avg_exec_price,\n sum(f.quantity) AS total_qty\n FROM fx_trades f\n HORIZON JOIN market_data m ON (f.symbol = m.symbol)\n LIST (0s, 30m) AS h\n WHERE f.timestamp IN '$yesterday'\n),\npivoted AS (\n SELECT * FROM order_markouts\n PIVOT (\n first(weighted_mid) AS mid\n FOR offset IN (\n 0 AS at_fill,\n 1800000000000 AS at_30m\n )\n GROUP BY order_id, symbol, side, avg_exec_price, total_qty\n )\n)\nSELECT\n order_id,\n symbol,\n side,\n total_qty,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (avg_exec_price - at_fill_mid)\n / at_fill_mid * 10000 AS total_is_bps,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (at_30m_mid - at_fill_mid)\n / at_fill_mid * 10000 AS permanent_bps,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (avg_exec_price - at_30m_mid)\n / at_fill_mid * 10000 AS temporary_bps\nFROM pivoted\nORDER BY total_is_bps DESC" + }, + { + "query": "WITH markouts AS (\n SELECT\n f.symbol,\n f.price,\n f.quantity,\n f.side,\n h.offset,\n (m.best_bid + m.best_ask) / 2 AS mid\n FROM fx_trades f\n HORIZON JOIN market_data m ON (f.symbol = m.symbol)\n LIST (0, 1800s) AS h\n WHERE f.timestamp IN '$yesterday'\n),\npivoted AS (\n SELECT * FROM markouts\n PIVOT (\n avg(mid) AS mid,\n avg(price) AS px,\n sum(quantity) AS vol\n FOR offset IN (\n 0 AS at_fill,\n 1800000000000 AS at_30m\n )\n GROUP BY symbol, side\n )\n)\nSELECT\n symbol,\n side,\n at_fill_vol AS total_volume,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (at_fill_px - at_fill_mid) / at_fill_mid * 10000 AS effective_spread_bps,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (at_30m_mid - at_fill_mid) / at_fill_mid * 10000 AS permanent_bps,\n CASE WHEN side = 'buy' THEN 1 ELSE -1 END\n * (at_fill_px - at_30m_mid) / at_fill_mid * 10000 AS temporary_bps\nFROM pivoted\nORDER BY symbol, side" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH with_prev AS (\n SELECT\n timestamp,\n symbol,\n high,\n low,\n close,\n lag(close) OVER (PARTITION BY symbol ORDER BY timestamp) AS prev_close\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n),\nwith_tr AS (\n SELECT\n timestamp,\n symbol,\n high,\n low,\n close,\n greatest(\n high - low,\n abs(high - prev_close),\n abs(low - prev_close)\n ) AS tr\n FROM with_prev\n WHERE prev_close IS NOT NULL\n),\nwith_indicators AS (\n SELECT\n timestamp,\n symbol,\n close,\n avg(close, 'period', 20) OVER w AS ema20,\n avg(tr, 'period', 20) OVER w AS atr\n FROM with_tr\n WINDOW w AS (PARTITION BY symbol ORDER BY timestamp)\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(ema20, 5) AS middle,\n round(ema20 + 2 * atr, 5) AS upper,\n round(ema20 - 2 * atr, 5) AS lower\nFROM with_indicators\nORDER BY timestamp" + }, + { + "query": "SELECT\n t.symbol,\n t.counterparty,\n t.passive,\n h.offset / 1000000 AS horizon_ms,\n count() AS n,\n avg(\n CASE t.side\n WHEN 'buy' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000\n WHEN 'sell' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / t.price * 10000\n END\n ) AS avg_markout_bps\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n LIST (0, 1T, 5T, 10T, 50T, 100T,\n 500T, 1000T, 5000T) AS h\nWHERE t.timestamp IN '$yesterday'\nGROUP BY t.symbol, t.counterparty, t.passive, horizon_ms\nORDER BY t.symbol, t.counterparty, horizon_ms" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1m..$now'\n\nSELECT\n timestamp,\n symbol,\n round((bid_price + ask_price) / 2, 5) AS mid_price,\n LN(\n (bid_price + ask_price) / 2 /\n LAG((bid_price + ask_price) / 2)\n OVER (PARTITION BY symbol ORDER BY timestamp)\n ) AS log_return\nFROM core_price\nWHERE symbol = @symbol\n AND ecn = 'LMAX'\n AND timestamp IN @lookback" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1d..$now'\n\nWITH sampled AS (\n SELECT\n timestamp,\n symbol,\n last((bid_price + ask_price) / 2) AS close_mid\n FROM core_price\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n SAMPLE BY 1m ALIGN TO CALENDAR\n)\nSELECT\n timestamp,\n symbol,\n round(close_mid, 5) AS close_mid,\n LN(close_mid / LAG(close_mid)\n OVER (PARTITION BY symbol ORDER BY timestamp))\n AS log_return\nFROM sampled" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH ema AS (\n SELECT\n timestamp,\n symbol,\n close,\n avg(close, 'period', 12) OVER w AS ema12,\n avg(close, 'period', 26) OVER w AS ema26\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n WINDOW w AS (PARTITION BY symbol ORDER BY timestamp)\n),\nmacd_line AS (\n SELECT\n timestamp,\n symbol,\n close,\n ema12,\n ema26,\n ema12 - ema26 AS macd\n FROM ema\n),\nwith_signal AS (\n SELECT\n timestamp,\n symbol,\n close,\n macd,\n avg(macd, 'period', 9) OVER (PARTITION BY symbol ORDER BY timestamp) AS signal\n FROM macd_line\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(macd, 6) AS macd,\n round(signal, 6) AS signal,\n round(macd - signal, 6) AS histogram\nFROM with_signal\nORDER BY timestamp" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n t.counterparty,\n t.passive,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n avg(\n CASE t.side\n WHEN 'buy' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000\n WHEN 'sell' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / t.price * 10000\n END\n ) AS avg_markout_bps,\n sum(\n CASE t.side\n WHEN 'buy' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n * t.quantity\n WHEN 'sell' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n * t.quantity\n END\n ) AS total_pnl\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n RANGE FROM 0s TO 30s STEP 5s AS h\nWHERE t.timestamp IN '$now-1h..$now'\nGROUP BY t.symbol, t.ecn, t.counterparty, t.passive, horizon_sec\nORDER BY t.symbol, t.ecn, t.counterparty, t.passive, horizon_sec" + }, + { + "query": "SELECT\n t.ecn,\n t.passive,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n round(avg(\n CASE t.side\n WHEN 'buy' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000\n WHEN 'sell' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / t.price * 10000\n END\n ), 3) AS avg_markout_bps\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n LIST (-30s, -5s, 0, 5s, 30s) AS h\nWHERE t.timestamp IN '$now-1h..$now'\nGROUP BY t.ecn, t.passive, horizon_sec\nORDER BY t.ecn, t.passive, horizon_sec" + }, + { + "query": "SELECT\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n round(avg(\n CASE t.side\n WHEN 'buy' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000\n WHEN 'sell' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / t.price * 10000\n END\n ), 3) AS avg_markout_bps\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n RANGE FROM -30s TO 30s STEP 1s AS h\nWHERE t.timestamp IN '$now-1h..$now'\nGROUP BY horizon_sec\nORDER BY horizon_sec" + }, + { + "query": "SELECT\n t.ecn,\n t.side,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n round(avg(\n CASE t.side\n WHEN 'buy' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000\n WHEN 'sell' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / t.price * 10000\n END\n ), 3) AS avg_markout_bps\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n LIST (-30s, -5s, 0, 5s, 30s) AS h\nWHERE t.timestamp IN '$now-1h..$now'\nGROUP BY t.ecn, t.side, horizon_sec\nORDER BY t.ecn, t.side, horizon_sec" + }, + { + "query": "SELECT\n t.symbol,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n avg(((m.best_bid + m.best_ask) / 2 - t.price) / t.price * 10000) AS avg_markout_bps,\n sum(((m.best_bid + m.best_ask) / 2 - t.price) * t.quantity) AS total_pnl\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n RANGE FROM 0s TO 10m STEP 10s AS h\nWHERE t.side = 'buy'\n AND t.timestamp IN '$now-1h..$now'\nGROUP BY t.symbol, horizon_sec\nORDER BY t.symbol, horizon_sec" + }, + { + "query": "SELECT\n t.symbol,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n avg((t.price - (m.best_bid + m.best_ask) / 2) / t.price * 10000) AS avg_markout_bps,\n sum((t.price - (m.best_bid + m.best_ask) / 2) * t.quantity) AS total_pnl\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n RANGE FROM 0s TO 10m STEP 10s AS h\nWHERE t.side = 'sell'\n AND t.timestamp IN '$now-1h..$now'\nGROUP BY t.symbol, horizon_sec\nORDER BY t.symbol, horizon_sec" + }, + { + "query": "SELECT\n t.symbol,\n t.counterparty,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n avg(((m.best_bid + m.best_ask) / 2 - t.price) / t.price * 10000) AS avg_markout_bps,\n sum(t.quantity) AS total_volume\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n LIST (0, 1s, 5s, 10s, 30s, 1m, 5m) AS h\nWHERE t.side = 'buy'\n AND t.timestamp IN '$now-1h..$now'\nGROUP BY t.symbol, t.counterparty, horizon_sec\nORDER BY t.symbol, t.counterparty, horizon_sec" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n t.passive,\n h.offset / 1000000000 AS horizon_sec,\n count() AS n,\n avg(((m.best_bid + m.best_ask) / 2 - t.price)\n / t.price * 10000) AS avg_markout_bps,\n avg((m.best_ask - m.best_bid)\n / ((m.best_bid + m.best_ask) / 2) * 10000) / 2 AS avg_half_spread_bps\nFROM fx_trades t\nHORIZON JOIN market_data m ON (symbol)\n RANGE FROM 0s TO 5m STEP 5s AS h\nWHERE t.side = 'buy'\n AND t.timestamp IN '$now-1h..$now'\nGROUP BY t.symbol, t.ecn, t.passive, horizon_sec\nORDER BY t.symbol, t.ecn, t.passive, horizon_sec" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1M..$now'\n\nWITH ohlc AS (\n SELECT\n timestamp,\n symbol,\n last(price) AS close\n FROM fx_trades\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n SAMPLE BY 15m ALIGN TO CALENDAR\n),\nwith_peak AS (\n SELECT\n timestamp,\n symbol,\n close,\n max(close) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) AS running_peak\n FROM ohlc\n),\nwith_drawdown AS (\n SELECT\n timestamp,\n symbol,\n close,\n running_peak,\n (close - running_peak) / running_peak * 100 AS drawdown\n FROM with_peak\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(running_peak, 5) AS peak,\n round(drawdown, 4) AS drawdown_pct,\n round(min(drawdown) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ), 4) AS max_drawdown_pct\nFROM with_drawdown\nORDER BY timestamp" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1M..$now'\n\nWITH ohlc AS (\n SELECT timestamp, symbol, last(price) AS close\n FROM fx_trades\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n SAMPLE BY 15m ALIGN TO CALENDAR\n),\nwith_peak AS (\n SELECT timestamp, symbol, close,\n max(close) OVER (PARTITION BY symbol ORDER BY timestamp ROWS UNBOUNDED PRECEDING) AS running_peak\n FROM ohlc\n),\nwith_drawdown AS (\n SELECT timestamp, symbol, close, running_peak,\n (close - running_peak) / running_peak * 100 AS drawdown\n FROM with_peak\n)\nSELECT timestamp, symbol, round(close, 5) AS close, round(drawdown, 2) AS drawdown_pct\nFROM with_drawdown\nWHERE drawdown < -1 -- Drawdowns greater than 1%\nORDER BY drawdown\nLIMIT 10" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1M..$now'\n\nWITH ohlc AS (\n SELECT\n timestamp,\n symbol,\n last(price) AS close,\n sum(quantity) AS volume\n FROM fx_trades\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n SAMPLE BY 15m ALIGN TO CALENDAR\n),\nwith_direction AS (\n SELECT\n timestamp,\n symbol,\n close,\n volume,\n CASE\n WHEN close > lag(close) OVER w THEN volume\n WHEN close < lag(close) OVER w THEN -volume\n ELSE 0\n END AS directed_volume\n FROM ohlc\n WINDOW w AS (PARTITION BY symbol ORDER BY timestamp)\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(volume, 0) AS volume,\n round(sum(directed_volume) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ), 0) AS obv\nFROM with_direction\nORDER BY timestamp" + }, + { + "query": "WITH quote_lag AS (\n SELECT\n timestamp,\n symbol,\n ecn,\n bid_price, bid_volume, ask_price, ask_volume,\n lag(bid_price) OVER w AS prev_bid_price,\n lag(bid_volume) OVER w AS prev_bid_volume,\n lag(ask_price) OVER w AS prev_ask_price,\n lag(ask_volume) OVER w AS prev_ask_volume\n FROM core_price\n WHERE symbol = 'EURUSD'\n AND timestamp IN '$yesterday'\n WINDOW w AS (PARTITION BY symbol, ecn ORDER BY timestamp)\n),\ncontributions AS (\n SELECT\n timestamp,\n symbol,\n ecn,\n CASE\n WHEN bid_price > prev_bid_price THEN bid_volume\n WHEN bid_price < prev_bid_price THEN -prev_bid_volume\n ELSE bid_volume - prev_bid_volume\n END\n -\n CASE\n WHEN ask_price < prev_ask_price THEN ask_volume\n WHEN ask_price > prev_ask_price THEN -prev_ask_volume\n ELSE ask_volume - prev_ask_volume\n END AS ofi_event\n FROM quote_lag\n WHERE prev_bid_price IS NOT NULL\n)\nSELECT\n timestamp,\n symbol,\n ecn,\n sum(ofi_event) AS ofi\nFROM contributions\nSAMPLE BY 1s" + }, + { + "query": "WITH quote_lag AS (\n SELECT\n timestamp,\n symbol,\n ecn,\n bid_price, bid_volume, ask_price, ask_volume,\n lag(bid_price) OVER w AS prev_bid_price,\n lag(bid_volume) OVER w AS prev_bid_volume,\n lag(ask_price) OVER w AS prev_ask_price,\n lag(ask_volume) OVER w AS prev_ask_volume\n FROM core_price\n WHERE symbol = 'EURUSD'\n AND timestamp IN '$yesterday'\n WINDOW w AS (PARTITION BY symbol, ecn ORDER BY timestamp)\n),\ncontributions AS (\n SELECT\n timestamp,\n symbol,\n CASE\n WHEN bid_price > prev_bid_price THEN bid_volume\n WHEN bid_price < prev_bid_price THEN -prev_bid_volume\n ELSE bid_volume - prev_bid_volume\n END\n -\n CASE\n WHEN ask_price < prev_ask_price THEN ask_volume\n WHEN ask_price > prev_ask_price THEN -prev_ask_volume\n ELSE ask_volume - prev_ask_volume\n END AS ofi_event\n FROM quote_lag\n WHERE prev_bid_price IS NOT NULL\n)\nSELECT timestamp, symbol, sum(ofi_event) AS ofi\nFROM contributions\nSAMPLE BY 1s" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH with_lag AS (\n SELECT\n timestamp,\n symbol,\n close,\n lag(close, 12) OVER (PARTITION BY symbol ORDER BY timestamp) AS close_12_ago\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(close_12_ago, 5) AS close_12_ago,\n round((close - close_12_ago) / close_12_ago * 100, 4) AS roc\nFROM with_lag\nWHERE close_12_ago IS NOT NULL\nORDER BY timestamp" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH returns AS (\n SELECT\n timestamp,\n symbol,\n close,\n ln(close / lag(close)\n OVER (PARTITION BY symbol ORDER BY timestamp))\n AS log_return\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n),\nwith_stats AS (\n SELECT\n timestamp,\n symbol,\n close,\n log_return,\n avg(log_return) OVER w AS mean_return,\n avg(log_return * log_return) OVER w AS mean_sq_return\n FROM returns\n WHERE log_return IS NOT NULL\n WINDOW w AS (\n PARTITION BY symbol ORDER BY timestamp\n ROWS BETWEEN 19 PRECEDING AND CURRENT ROW\n )\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(log_return * 100, 4) AS return_pct,\n round(\n sqrt(mean_sq_return - mean_return * mean_return)\n * sqrt(365 * 96) * 100,\n 2) AS realized_vol_annualized\nFROM with_stats\nORDER BY timestamp" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 1M..$now'\n\nWITH ohlc AS (\n SELECT\n timestamp,\n symbol,\n last(price) AS close\n FROM fx_trades\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n SAMPLE BY 5m ALIGN TO CALENDAR\n),\nreturns AS (\n SELECT\n timestamp,\n symbol,\n close,\n ln(close / lag(close)\n OVER (PARTITION BY symbol ORDER BY timestamp))\n AS log_return\n FROM ohlc\n),\nwith_stats AS (\n SELECT\n timestamp,\n symbol,\n close,\n log_return,\n avg(log_return) OVER w AS mean_return,\n avg(log_return * log_return) OVER w AS mean_sq_return\n FROM returns\n WHERE log_return IS NOT NULL\n WINDOW w AS (\n PARTITION BY symbol ORDER BY timestamp\n ROWS BETWEEN 11 PRECEDING AND CURRENT ROW\n )\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(log_return * 100, 4) AS return_pct,\n round(\n sqrt(mean_sq_return - mean_return * mean_return)\n * sqrt(288 * 365) * 100,\n 2) AS realized_vol_annualized\nFROM with_stats\nORDER BY timestamp" + }, + { + "query": "WITH stats AS (\n SELECT\n timestamp,\n symbol,\n price,\n AVG(price) OVER w AS rolling_avg,\n AVG(price * price) OVER w AS rolling_avg_sq\n FROM fx_trades\n WHERE timestamp IN '$yesterday' AND symbol = 'EURUSD'\n WINDOW w AS (PARTITION BY symbol ORDER BY timestamp)\n)\nSELECT\n timestamp,\n symbol,\n price,\n rolling_avg,\n SQRT(rolling_avg_sq - rolling_avg * rolling_avg) AS rolling_stddev\nFROM stats\nLIMIT 10" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH changes AS (\n SELECT\n timestamp,\n symbol,\n close,\n close - lag(close) OVER (PARTITION BY symbol ORDER BY timestamp) AS change\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n),\ngains_losses AS (\n SELECT\n timestamp,\n symbol,\n close,\n CASE WHEN change > 0 THEN change ELSE 0 END AS gain,\n CASE WHEN change < 0 THEN -change ELSE 0 END AS loss\n FROM changes\n),\nsmoothed AS (\n SELECT\n timestamp,\n symbol,\n close,\n avg(gain, 'period', 14) OVER w AS avg_gain,\n avg(loss, 'period', 14) OVER w AS avg_loss\n FROM gains_losses\n WINDOW w AS (PARTITION BY symbol ORDER BY timestamp)\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(100 - (100 / (1 + avg_gain / avg_loss)), 2) AS rsi\nFROM smoothed\nORDER BY timestamp" + }, + { + "query": "SELECT\n t.symbol,\n t.ecn,\n t.counterparty,\n t.passive,\n count() AS trade_count,\n sum(t.quantity) AS total_qty,\n avg(\n CASE t.side\n WHEN 'buy' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n WHEN 'sell' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n END\n ) AS avg_slippage_vs_mid_bps,\n avg(\n CASE t.side\n WHEN 'buy' THEN (t.price - m.best_ask) / m.best_ask * 10000\n WHEN 'sell' THEN (m.best_bid - t.price) / m.best_bid * 10000\n END\n ) AS avg_slippage_vs_tob_bps,\n avg(\n (m.best_ask - m.best_bid)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n ) AS avg_spread_bps\nFROM fx_trades t\nASOF JOIN market_data m ON (symbol)\nWHERE t.timestamp IN '$yesterday'\nGROUP BY t.symbol, t.ecn, t.counterparty, t.passive\nORDER BY avg_slippage_vs_mid_bps DESC" + }, + { + "query": "SELECT\n t.ecn,\n t.passive,\n count() AS trade_count,\n round(avg(\n CASE t.side\n WHEN 'buy' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n WHEN 'sell' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n END\n ), 3) AS avg_slippage_bps\nFROM fx_trades t\nASOF JOIN market_data m ON (symbol)\nWHERE t.timestamp IN '$yesterday'\nGROUP BY t.ecn, t.passive\nORDER BY t.ecn, t.passive" + }, + { + "query": "SELECT\n t.timestamp,\n t.ecn,\n count() AS trade_count,\n round(avg(\n CASE t.side\n WHEN 'buy' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n WHEN 'sell' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n END\n ), 3) AS avg_slippage_bps\nFROM fx_trades t\nASOF JOIN market_data m ON (symbol)\nWHERE t.timestamp IN '$yesterday'\nSAMPLE BY 1h" + }, + { + "query": "WITH fills AS (\n SELECT\n t.symbol,\n t.price,\n t.quantity,\n h.offset,\n (m.best_bid + m.best_ask) / 2 AS mid,\n m.best_ask - m.best_bid AS spread,\n CASE\n WHEN t.quantity < 100000 THEN 'S'\n WHEN t.quantity < 1000000 THEN 'M'\n WHEN t.quantity < 10000000 THEN 'L'\n ELSE 'XL'\n END AS size_bucket\n FROM fx_trades t\n HORIZON JOIN market_data m ON (symbol)\n LIST (0, 5s, 1m) AS h\n WHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\n)\nSELECT * FROM fills\nPIVOT (\n count() AS n,\n avg((mid - price) / price * 10000) AS markout_bps,\n avg(spread / mid * 10000) AS spread_bps\n FOR offset IN (0 AS at_fill, 5000000000 AS t_5s, 60000000000 AS t_1m)\n GROUP BY symbol, size_bucket\n)\nORDER BY symbol, size_bucket" + }, + { + "query": "WITH cp_costs AS (\n SELECT\n t.symbol,\n t.counterparty,\n t.ecn,\n t.passive,\n t.price,\n t.quantity,\n h.offset,\n m.best_bid,\n m.best_ask,\n (m.best_bid + m.best_ask) / 2 AS mid\n FROM fx_trades t\n HORIZON JOIN market_data m ON (symbol)\n LIST (0, 5s, 1m) AS h\n WHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\n)\nSELECT * FROM cp_costs\nPIVOT (\n count() AS fills,\n sum(quantity) AS volume,\n avg((mid - price) / price * 10000) AS markout_bps\n FOR offset IN (0 AS at_fill, 5000000000 AS t_5s, 60000000000 AS t_1m)\n GROUP BY symbol, counterparty, ecn, passive\n)\nORDER BY t_1m_markout_bps" + }, + { + "query": "WITH hourly AS (\n SELECT\n t.symbol,\n t.price,\n t.quantity,\n hour(t.timestamp) AS hour_utc,\n h.offset,\n m.best_bid,\n m.best_ask,\n (m.best_bid + m.best_ask) / 2 AS mid\n FROM fx_trades t\n HORIZON JOIN market_data m ON (symbol)\n LIST (0, 5s, 1m) AS h\n WHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\n)\nSELECT * FROM hourly\nPIVOT (\n count() AS n,\n avg((mid - price) / price * 10000) AS markout_bps,\n avg((best_ask - best_bid) / mid * 10000) AS spread_bps\n FOR offset IN (0 AS at_fill, 5000000000 AS t_5s, 60000000000 AS t_1m)\n GROUP BY symbol, hour_utc\n)\nORDER BY symbol, hour_utc" + }, + { + "query": "WITH daily AS (\n SELECT\n t.symbol,\n t.ecn,\n t.price,\n t.quantity,\n t.timestamp::date AS trade_date,\n h.offset,\n (m.best_bid + m.best_ask) / 2 AS mid\n FROM fx_trades t\n HORIZON JOIN market_data m ON (symbol)\n LIST (0, 1m, 5m) AS h\n WHERE t.side = 'buy'\n AND t.timestamp IN '$yesterday'\n)\nSELECT * FROM daily\nPIVOT (\n count() AS fills,\n sum(quantity) AS volume,\n sum((mid - price) * quantity) AS pnl\n FOR offset IN (0 AS at_fill, 60000000000 AS t_1m, 300000000000 AS t_5m)\n GROUP BY trade_date, symbol, ecn\n)\nORDER BY trade_date, symbol, ecn" + }, + { + "query": "SELECT\n t.timestamp,\n t.symbol,\n t.ecn,\n t.counterparty,\n t.side,\n t.passive,\n t.price,\n t.quantity,\n m.best_bid,\n m.best_ask,\n (m.best_bid + m.best_ask) / 2 AS mid,\n (m.best_ask - m.best_bid) AS spread,\n CASE t.side\n WHEN 'buy' THEN (t.price - (m.best_bid + m.best_ask) / 2)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n WHEN 'sell' THEN ((m.best_bid + m.best_ask) / 2 - t.price)\n / ((m.best_bid + m.best_ask) / 2) * 10000\n END AS slippage_bps,\n CASE t.side\n WHEN 'buy' THEN (t.price - m.best_ask) / m.best_ask * 10000\n WHEN 'sell' THEN (m.best_bid - t.price) / m.best_bid * 10000\n END AS slippage_vs_tob_bps\nFROM fx_trades t\nASOF JOIN market_data m ON (symbol)\nWHERE t.timestamp IN '$yesterday'\nORDER BY t.timestamp" + }, + { + "query": "DECLARE\n @symbol := 'EURUSD',\n @lookback := '$now - 2d..$now'\n\nWITH ranges AS (\n SELECT\n timestamp,\n symbol,\n close,\n min(low) OVER w AS lowest_low,\n max(high) OVER w AS highest_high\n FROM market_data_ohlc_15m\n WHERE symbol = @symbol\n AND timestamp IN @lookback\n WINDOW w AS (\n PARTITION BY symbol ORDER BY timestamp\n ROWS BETWEEN 13 PRECEDING AND CURRENT ROW\n )\n),\nwith_k AS (\n SELECT\n timestamp,\n symbol,\n close,\n (close - lowest_low) / (highest_high - lowest_low) * 100 AS pct_k\n FROM ranges\n WHERE highest_high > lowest_low\n)\nSELECT\n timestamp,\n symbol,\n round(close, 5) AS close,\n round(pct_k, 2) AS pct_k,\n round(avg(pct_k) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 2 PRECEDING AND CURRENT ROW\n ), 2) AS pct_d\nFROM with_k\nORDER BY timestamp" + }, + { + "query": "SELECT twap(price, timestamp) AS twap\nFROM trades\nWHERE symbol = 'ETH-USDT'\n AND timestamp IN '$yesterday'" + }, + { + "query": "SELECT symbol, twap(price, timestamp) AS twap\nFROM trades\nWHERE symbol IN ('BTC-USDT', 'ETH-USDT', 'SOL-USDT')\n AND timestamp IN '$yesterday'" + }, + { + "query": "SELECT timestamp, symbol, twap(price, timestamp) AS twap\nFROM trades\nWHERE symbol = 'BTC-USDT'\n AND timestamp IN '$yesterday'\nSAMPLE BY 1h" + }, + { + "query": "SELECT\n timestamp,\n symbol,\n twap(price, timestamp) AS twap,\n sum(price * amount) / sum(amount) AS vwap,\n twap(price, timestamp) - sum(price * amount) / sum(amount) AS twap_vwap_diff\nFROM trades\nWHERE symbol = 'BTC-USDT'\n AND timestamp IN '$yesterday'\nSAMPLE BY 1h" + }, + { + "query": "WITH benchmark AS (\n SELECT twap(price, timestamp) AS twap_price\n FROM trades\n WHERE symbol = 'ETH-USDT'\n AND timestamp IN '$yesterday'\n)\nSELECT\n t.timestamp,\n t.price AS fill_price,\n b.twap_price,\n t.price - b.twap_price AS slippage\nFROM trades t\nCROSS JOIN benchmark b\nWHERE t.symbol = 'ETH-USDT'\n AND t.side = 'buy'\n AND t.timestamp IN '$yesterday'\nORDER BY t.timestamp\nLIMIT 20" + }, + { + "query": "DECLARE @tick_size := 0.01\nSELECT\n floor(price / @tick_size) * @tick_size AS price_bin,\n round(SUM(quantity), 2) AS volume\nFROM fx_trades\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$today'\nORDER BY price_bin" + }, + { + "query": "WITH bucketed AS (\n SELECT\n t.timestamp,\n t.symbol,\n t.side,\n t.price,\n t.quantity,\n floor(\n sum(t.quantity) OVER (PARTITION BY symbol ORDER BY timestamp)\n / 1000000\n ) AS vol_bucket\n FROM fx_trades t\n WHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$yesterday'\n),\nbucket_stats AS (\n SELECT\n symbol,\n vol_bucket,\n min(timestamp) AS bucket_start,\n max(timestamp) AS bucket_end,\n count() AS trade_count,\n sum(quantity) AS total_vol,\n sum(CASE WHEN side = 'buy' THEN quantity ELSE 0.0 END) AS buy_vol,\n sum(CASE WHEN side = 'sell' THEN quantity ELSE 0.0 END) AS sell_vol,\n abs(\n sum(CASE WHEN side = 'buy' THEN quantity ELSE 0.0 END)\n - sum(CASE WHEN side = 'sell' THEN quantity ELSE 0.0 END)\n ) / sum(quantity) AS bucket_imbalance\n FROM bucketed\n GROUP BY symbol, vol_bucket\n)\nSELECT\n symbol,\n vol_bucket,\n bucket_start,\n bucket_end,\n total_vol,\n buy_vol,\n sell_vol,\n bucket_imbalance,\n avg(bucket_imbalance) OVER (\n PARTITION BY symbol\n ORDER BY vol_bucket\n ROWS BETWEEN 49 PRECEDING AND CURRENT ROW\n ) AS vpin\nFROM bucket_stats\nORDER BY vol_bucket" + }, + { + "query": "WITH bucketed AS (\n SELECT\n t.timestamp,\n t.symbol,\n t.ecn,\n t.side,\n t.quantity,\n floor(\n sum(t.quantity) OVER (PARTITION BY symbol, ecn ORDER BY timestamp)\n / 1000000\n ) AS vol_bucket\n FROM fx_trades t\n WHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$yesterday'\n),\nbucket_stats AS (\n SELECT\n symbol,\n ecn,\n vol_bucket,\n min(timestamp) AS bucket_start,\n max(timestamp) AS bucket_end,\n sum(quantity) AS total_vol,\n abs(\n sum(CASE WHEN side = 'buy' THEN quantity ELSE 0.0 END)\n - sum(CASE WHEN side = 'sell' THEN quantity ELSE 0.0 END)\n ) / sum(quantity) AS bucket_imbalance\n FROM bucketed\n GROUP BY symbol, ecn, vol_bucket\n)\nSELECT\n symbol,\n ecn,\n vol_bucket,\n bucket_start,\n bucket_end,\n total_vol,\n bucket_imbalance,\n avg(bucket_imbalance) OVER (\n PARTITION BY symbol, ecn\n ORDER BY vol_bucket\n ROWS BETWEEN 49 PRECEDING AND CURRENT ROW\n ) AS vpin\nFROM bucket_stats\nORDER BY ecn, vol_bucket" + }, + { + "query": "WITH sampled AS (\n SELECT\n timestamp, symbol,\n total_volume,\n ((high + low + close) / 3) * total_volume AS traded_value\n FROM fx_trades_ohlc_1m\n WHERE timestamp IN '$yesterday' AND symbol = 'EURUSD'\n)\nSELECT\n timestamp, symbol,\n SUM(traded_value) OVER w / SUM(total_volume) OVER w AS vwap\nFROM sampled\nWINDOW w AS (ORDER BY timestamp)" + }, + { + "query": "WITH sampled AS (\n SELECT\n timestamp, symbol,\n total_volume,\n ((high + low + close) / 3) * total_volume AS traded_value\n FROM fx_trades_ohlc_1m\n WHERE timestamp IN '$yesterday'\n AND symbol IN ('EURUSD', 'GBPUSD', 'USDJPY')\n)\nSELECT\n timestamp, symbol,\n SUM(traded_value) OVER w / SUM(total_volume) OVER w AS vwap\nFROM sampled\nWINDOW w AS (PARTITION BY symbol ORDER BY timestamp)" + }, + { + "query": "SELECT count() AS out_of_order_rows\nFROM (\n SELECT timestamp AS current_ts, lag(timestamp) OVER () AS previous_ts\n FROM trades\n WHERE timestamp IN '$today'\n)\nWHERE current_ts < previous_ts" + }, + { + "query": "WITH column_and_prev AS (\n SELECT row_number() OVER () AS rownum,\n timestamp AS current_ts,\n lag(timestamp) OVER () AS previous_ts\n FROM trades\n WHERE timestamp IN '$today'\n)\nSELECT rownum, current_ts, previous_ts\nFROM column_and_prev\nWHERE current_ts < previous_ts\nLIMIT 1" + }, + { + "query": "WITH column_and_prev AS (\n SELECT row_number() OVER () AS rownum,\n price AS current_price,\n lag(price) OVER () AS previous_price\n FROM trades\n WHERE symbol = 'BTC-USDT' AND timestamp IN '$today'\n)\nSELECT rownum, current_price, previous_price\nFROM column_and_prev\nWHERE current_price < previous_price\nLIMIT 1" + }, + { + "query": "SELECT count() AS out_of_order_rows\nFROM (\n SELECT timestamp AS current_ts, lag(timestamp) OVER () AS previous_ts\n FROM read_parquet('trades.parquet')\n)\nWHERE current_ts < previous_ts" + }, + { + "query": "SELECT timestamp, symbol, avg(bid_price) as bid_price, avg(ask_price) as ask_price\nFROM core_price\nWHERE symbol = 'EURUSD' AND timestamp IN '$today'\nSAMPLE BY 100T FILL(PREV(ask_price), PREV)" + }, + { + "query": "WITH sampled AS (\n SELECT timestamp, symbol, avg(bid_price) as bid_price, avg(ask_price) as ask_price\n FROM core_price\n WHERE symbol = 'EURUSD' AND timestamp IN '$today'\n SAMPLE BY 100T FILL(null)\n), with_previous_vals AS (\n SELECT *,\n last_value(ask_price) IGNORE NULLS OVER(PARTITION BY symbol ORDER BY timestamp) as filler\n FROM sampled\n)\nSELECT timestamp, symbol, coalesce(bid_price, filler) as bid_price,\n coalesce(ask_price, filler) as ask_price\nFROM with_previous_vals" + }, + { + "query": "WITH sampled AS (\n SELECT timestamp, symbol, avg(bid_price) as bid_price, avg(ask_price) as ask_price\n FROM core_price\n WHERE symbol = 'EURUSD' AND timestamp IN '$today'\n SAMPLE BY 100T FILL(null)\n), with_previous_vals AS (\n SELECT *,\n last_value(ask_price) IGNORE NULLS OVER(PARTITION BY symbol ORDER BY timestamp) as filler\n FROM sampled\n)\nSELECT timestamp, symbol, coalesce(bid_price, filler) as bid_price,\n coalesce(ask_price, filler) as ask_price,\n case when bid_price is NULL then true END as filled\nFROM with_previous_vals" + }, + { + "query": "DECLARE\n @start_ts := dateadd('m', -2, now()),\n @end_ts := dateadd('m', 2, now())\nWITH\nsandwich AS (\n SELECT * FROM (\n SELECT @start_ts AS timestamp, null AS symbol, null AS open, null AS high, null AS close, null AS low\n UNION ALL\n SELECT timestamp, symbol, open_mid AS open, high_mid AS high, close_mid AS close, low_mid AS low\n FROM core_price_1s\n WHERE timestamp BETWEEN @start_ts AND @end_ts\n UNION ALL\n SELECT @end_ts AS timestamp, null AS symbol, null AS open, null AS high, null AS close, null AS low\n ) ORDER BY timestamp\n),\nsampled AS (\n SELECT\n timestamp,\n symbol,\n first(open) AS open,\n first(high) AS high,\n first(low) AS low,\n first(close) AS close\n FROM sandwich\n SAMPLE BY 30s\n FILL(PREV, PREV, PREV, PREV, 0)\n)\nSELECT * FROM sampled WHERE open IS NOT NULL AND symbol IN ('EURUSD', 'GBPUSD')" + }, + { + "query": "DECLARE\n @start_ts := dateadd('s', -3, now()),\n @end_ts := now()\nWITH\nfiller_row AS (\n SELECT timestamp, open_mid AS open, high_mid AS high, close_mid AS close, low_mid AS low\n FROM core_price_1s\n WHERE timestamp < @start_ts\n LIMIT -1\n),\nsandwich AS (\n SELECT * FROM (\n SELECT * FROM filler_row\n UNION ALL\n SELECT timestamp, open_mid AS open, high_mid AS high, close_mid AS close, low_mid AS low\n FROM core_price_1s\n WHERE timestamp BETWEEN @start_ts AND @end_ts\n ) ORDER BY timestamp\n),\nsampled AS (\n SELECT\n timestamp,\n first(open) AS open,\n first(high) AS high,\n first(low) AS low,\n first(close) AS close\n FROM sandwich\n SAMPLE BY 100T\n FILL(PREV, PREV, PREV, PREV, 0)\n)\nSELECT * FROM sampled WHERE timestamp >= @start_ts" + }, + { + "query": "DECLARE\n @year := '2025',\n @week := 24,\n @first_monday := dateadd('d', -1 * day_of_week(@year) + 1, @year),\n @week_start := dateadd('w', @week - 1, @first_monday),\n @week_end := dateadd('w', @week, @first_monday)\nSELECT * FROM trades\nWHERE timestamp >= @week_start\n AND timestamp < @week_end" + }, + { + "query": "WITH t AS (\n (\n SELECT\n TO_TIMESTAMP(timestamp::STRING, 'yyyy-MM-ddTHH:mm:ss.SSSUUUZ') time,\n symbol,\n ecn,\n bid_price\n FROM\n core_price\n WHERE timestamp IN '$now - 1h..$now'\n ORDER BY time\n ) TIMESTAMP (time)\n)\nSELECT * FROM t LATEST BY symbol" + }, + { + "query": "WITH ranked AS (\n SELECT\n *,\n row_number() OVER (PARTITION BY symbol ORDER BY timestamp DESC) as rn\n FROM trades\n WHERE timestamp IN '$today'\n)\nSELECT timestamp, symbol, side, price, amount\nFROM ranked\nWHERE rn <= 5\nORDER BY symbol, timestamp DESC" + }, + { + "query": "DECLARE @limit := 10\n\nWITH ranked AS (\n SELECT *, row_number() OVER (PARTITION BY symbol ORDER BY timestamp DESC) as rn\n FROM trades\n WHERE timestamp IN '$now - 1d..$now'\n)\nSELECT * FROM ranked WHERE rn <= @limit" + }, + { + "query": "WITH ranked AS (\n SELECT\n *,\n row_number() OVER (PARTITION BY symbol ORDER BY timestamp DESC) as rn\n FROM trades\n WHERE timestamp IN '$today'\n AND side = 'buy' -- Additional filter before ranking\n)\nSELECT timestamp, symbol, side, price, amount\nFROM ranked\nWHERE rn <= 5" + }, + { + "query": "WITH ranked AS (\n SELECT *, row_number() OVER (PARTITION BY symbol ORDER BY timestamp DESC) as rn\n FROM trades\n WHERE timestamp IN '$today'\n)\nSELECT timestamp, symbol, price, rn as rank\nFROM ranked\nWHERE rn <= 5" + }, + { + "query": "SELECT table_name, walEnabled\nFROM tables()\nORDER BY table_name" + }, + { + "query": "SELECT status, progress_percent, start_ts, end_ts, backup_error\nFROM backups()\nORDER BY start_ts DESC\nLIMIT 1" + }, + { + "query": "SELECT backup_instance_name()" + }, + { + "query": "ALTER TABLE trades SET STORAGE POLICY(\n TO PARQUET 7 DAYS,\n TO REMOTE 14 DAYS,\n DROP LOCAL 30 DAYS\n)" + }, + { + "query": "SELECT table_name, walEnabled, partitionBy, designatedTimestamp\nFROM tables()\nWHERE table_name = 'trades'" + }, + { + "query": "SELECT * FROM storage_policies\nWHERE table_dir_name LIKE 'trades~%'" + }, + { + "query": "SELECT * FROM table_cold_partitions('trades')" + }, + { + "query": "SELECT timestamp, symbol, price\nFROM trades\nWHERE timestamp IN '2026-02-10'" + }, + { + "query": "SWITCH COLD STORAGE ROLE TO REFRESHER" + }, + { + "query": "SWITCH COLD STORAGE STATUS" + }, + { + "query": "SWITCH COLD STORAGE ROLE TO MANAGER" + }, + { + "query": "SWITCH COLD STORAGE ROLE TO MANAGER FORCE" + }, + { + "query": "SELECT timestamp, state, seq_txn, size, last_modified, partition_path\nFROM table_cold_partitions('trades')\nWHERE state = 'pending'" + }, + { + "query": "SELECT name, errorTag, errorMessage FROM wal_tables() WHERE suspended" + }, + { + "query": "SELECT timestamp, state, partition_path\nFROM table_cold_partitions('trades')\nWHERE state = 'pending'" + }, + { + "query": "-- variable-length array filled with -1\nSELECT array_build(1, x::int, -1) FROM long_sequence(3)" + }, + { + "query": "-- array filled with the max of an existing array (market_data is on demo.questdb.com)\nSELECT array_build(1, bids[1], array_max(bids[1])) FROM market_data LIMIT 1" + }, + { + "query": "INSERT INTO geo_data VALUES(#u33d8, ##10101111100101111111101101101)\n-- Querying by geohash" + }, + { + "query": "SELECT * FROM geo_data WHERE g5c = #u33d8" + }, + { + "query": "-- insert a 5-bit geohash into a 4 bit column\nINSERT INTO my_geo_data VALUES(#a/4)\n-- insert a 20-bit geohash into an 18 bit column" + }, + { + "query": "INSERT INTO my_geo_data VALUES(#u33d/18)" + }, + { + "query": "INSERT INTO my_geo_data VALUES(#u33, #u33d8b12)\n-- equivalent to" + }, + { + "query": "INSERT INTO my_geo_data VALUES('u33', 'u33d8b12')" + }, + { + "query": "SELECT approx_count_distinct(symbol, 5) FROM fx_trades" + }, + { + "query": "SELECT symbol, approx_median(price) FROM trades\nWHERE timestamp IN '$today'\nGROUP BY symbol" + }, + { + "query": "SELECT symbol, approx_median(price, 3) FROM trades\nWHERE timestamp IN '$today'\nGROUP BY symbol" + }, + { + "query": "SELECT approx_percentile(price, 0.99, 3) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT arg_max(timestamp, price) AS peak_time FROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, arg_max(timestamp, price) AS peak_time\nFROM trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, arg_max(price, amount) AS price_at_peak_volume\nFROM trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT arg_min(timestamp, price) AS bottom_time FROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, arg_min(timestamp, price) AS trough_time\nFROM trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, arg_min(price, amount) AS price_at_min_volume\nFROM trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, array_agg(price) AS prices\nFROM fx_trades\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$now - 3s..$now'\nGROUP BY symbol" + }, + { + "query": "SELECT timestamp, array_agg(price) AS prices\nFROM trades\nWHERE symbol = 'BTC-USDT'\n AND timestamp IN '$now - 5s..$now'\nSAMPLE BY 1s" + }, + { + "query": "SELECT timestamp, array_agg(bids[1]) AS all_bids\nFROM market_data\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$now - 1s..$now'\nSAMPLE BY 100ms" + }, + { + "query": "SELECT symbol,\n array_agg(price) AS prices,\n array_cum_sum(array_agg(price)) AS cumulative_prices\nFROM fx_trades\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$now - 3s..$now'\nGROUP BY symbol" + }, + { + "query": "SELECT avg(price) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, avg(price) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT corr(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, corr(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT count() FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, count() FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, count(price) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, count(distinct ecn) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT count_distinct(side) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, count_distinct(ecn) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT covar_pop(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, covar_pop(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT covar_samp(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, covar_samp(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT kurtosis_samp(value)\nFROM UNNEST(ARRAY[-10.0, -20.0, 100.0, 1000.0, 1000.0])" + }, + { + "query": "SELECT kurtosis_pop(value)\nFROM UNNEST(ARRAY[-10.0, -20.0, 100.0, 1000.0, 1000.0])" + }, + { + "query": "SELECT max(price) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, max(price) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT min(price) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, min(price) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, mode(side)\nFROM trades\nWHERE timestamp IN '$today'\nORDER BY symbol ASC" + }, + { + "query": "SELECT skewness_samp(value)\nFROM UNNEST(ARRAY[-10.0, -20.0, 100.0, 1000.0, 1000.0])" + }, + { + "query": "SELECT skewness_pop(value)\nFROM UNNEST(ARRAY[-10.0, -20.0, 100.0, 1000.0, 1000.0])" + }, + { + "query": "SELECT string_distinct_agg(symbol, ',') AS distinct_symbols\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT side, string_distinct_agg(symbol, ',') AS distinct_symbols\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT sum(quantity) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, sum(quantity) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT twap(price, timestamp)\nFROM trades\nWHERE symbol = 'BTC-USDT'\n AND timestamp IN '$yesterday'" + }, + { + "query": "SELECT symbol, twap(price, timestamp)\nFROM trades\nWHERE timestamp IN '$yesterday'" + }, + { + "query": "SELECT timestamp, symbol, twap(price, timestamp)\nFROM trades\nWHERE symbol IN ('BTC-USDT', 'ETH-USDT')\n AND timestamp IN '$yesterday'\nSAMPLE BY 1h" + }, + { + "query": "SELECT weighted_avg(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT weighted_stddev_freq(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, weighted_stddev_freq(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT weighted_stddev(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT symbol, weighted_stddev(price, quantity) FROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, array_avg(bids[1]) AS avg_bid_price\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT array_build(1, 3, 0) FROM long_sequence(1)" + }, + { + "query": "SELECT x, array_build(1, x::int, -1) FROM long_sequence(3)" + }, + { + "query": "SELECT array_build(1, bids[1], bids[1])\nFROM market_data\nLIMIT 1" + }, + { + "query": "SELECT array_build(1, 5, ARRAY[10.0, 20.0, 30.0]) FROM long_sequence(1)" + }, + { + "query": "SELECT array_build(2, 3, 1.0, 0.0) FROM long_sequence(1)" + }, + { + "query": "SELECT array_build(2, bids[1], bids[1], asks[1])\nFROM market_data\nLIMIT 1" + }, + { + "query": "SELECT array_build(4, bids[1], bids[1], bids[2], asks[1], asks[2])\nFROM market_data\nLIMIT 1" + }, + { + "query": "SELECT symbol, array_count(bids[1]) AS bid_levels, array_count(asks[1]) AS ask_levels\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, array_cum_sum(bids[2]) AS cumulative_bid_volume\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT array_elem_avg(ARRAY[10.0, 20.0, 30.0], ARRAY[30.0, 40.0, 50.0])" + }, + { + "query": "SELECT timestamp, symbol, array_elem_avg(bids[2]) AS avg_bid_volume\nFROM market_data\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$today'\nSAMPLE BY 1h" + }, + { + "query": "SELECT array_elem_max(ARRAY[1.0, 5.0, 3.0], ARRAY[4.0, 2.0, 6.0])" + }, + { + "query": "SELECT array_elem_max(\n ARRAY[[1.0, 8.0, 3.0], [5.0, 2.0, 9.0]],\n ARRAY[[4.0, 6.0, 7.0], [3.0, 8.0, 1.0]]\n)" + }, + { + "query": "SELECT timestamp, symbol, array_elem_max(bids[1]) AS best_bid_per_level\nFROM market_data\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$today'\nSAMPLE BY 1h" + }, + { + "query": "SELECT array_elem_min(ARRAY[1.0, 5.0, 3.0], ARRAY[4.0, 2.0, 6.0])" + }, + { + "query": "SELECT array_elem_min(\n ARRAY[100.0, 200.0, 150.0],\n ARRAY[120.0, 180.0, 160.0, 90.0]\n)" + }, + { + "query": "SELECT array_elem_min(ARRAY[100.0, null], ARRAY[null, 200.0])" + }, + { + "query": "SELECT timestamp, symbol, array_elem_min(bids[1]) AS worst_bid_per_level\nFROM market_data\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$today'\nSAMPLE BY 1h" + }, + { + "query": "SELECT array_elem_min(\n ARRAY[[1.0, 8.0, 3.0], [5.0, 2.0, 9.0]],\n ARRAY[[4.0, 6.0, 7.0], [3.0, 8.0, 1.0]]\n)" + }, + { + "query": "SELECT array_elem_sum(\n ARRAY[1.0, 2.0, 3.0],\n ARRAY[10.0, 20.0, 30.0]\n)" + }, + { + "query": "SELECT timestamp, symbol, array_elem_sum(bids[2]) AS total_bid_volume_per_level\nFROM market_data\nWHERE symbol = 'EURUSD'\n AND timestamp IN '$today'\nSAMPLE BY 1h" + }, + { + "query": "SELECT symbol, array_max(bids[1]) AS best_bid, array_min(asks[1]) AS best_ask\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, array_min(bids[1]) AS worst_bid, array_max(asks[1]) AS worst_ask\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, array_position(bids[1], best_bid) AS best_bid_position\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT array_reverse(ARRAY[1.0, 2.0, 3.0])" + }, + { + "query": "SELECT symbol, array_reverse(bids[1]) AS bids_worst_to_best\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT array_reverse(ARRAY[[1.0, 2.0], [3.0, 4.0]])" + }, + { + "query": "SELECT symbol, array_sort(asks[1], true) AS asks_desc\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT array_sort(ARRAY[3.0, 1.0, 2.0], true)" + }, + { + "query": "SELECT\n array_sort(ARRAY[1.0, null, 2.0]) AS default_nulls,\n array_sort(ARRAY[1.0, null, 2.0], false, true) AS nulls_first" + }, + { + "query": "SELECT array_sort(ARRAY[[3.0, 1.0, 2.0], [6.0, 4.0, 5.0]])" + }, + { + "query": "SELECT symbol, array_stddev(bids[1]) AS bid_price_dispersion\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, array_stddev_pop(asks[2]) AS ask_volume_stddev\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, array_stddev_samp(bids[1]) AS bid_price_stddev\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, array_sum(bids[2]) AS total_bid_volume\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT\n dim_length(bids, 1) AS num_sub_arrays,\n dim_length(bids, 2) AS levels_per_sub_array\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol,\n dot_product(bids[1], bids[2]) / array_sum(bids[2]) AS vwap_bid\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, flatten(bids) AS bids_flat\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, insertion_point(asks[1], best_bid) AS bid_in_ask_book\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT symbol, bids[1] - shift(bids[1], 1, 0.0) AS bid_level_diffs\nFROM market_data\nWHERE symbol = 'EURUSD'\nLIMIT -3" + }, + { + "query": "SELECT isOrdered(numeric_sequence) is_num_ordered,\n isOrdered(ts) is_ts_ordered\nFROM my_table" + }, + { + "query": "SELECT isOrdered(numeric_sequence) FROM my_table" + }, + { + "query": "SELECT * FROM trades WHERE ts IN '$today'" + }, + { + "query": "SELECT * FROM trades WHERE ts IN '$now - 1h..$now'" + }, + { + "query": "SELECT dateadd('h', 2, ts) as shifted_time FROM trades" + }, + { + "query": "SELECT year(ts), month(ts) FROM trades" + }, + { + "query": "SELECT\n is_end_of_month('2024-02-29T00:00:00.000000Z'::timestamp) feb29_leap,\n is_end_of_month('2023-02-28T00:00:00.000000Z'::timestamp) feb28_nonleap,\n is_end_of_month('2024-02-28T00:00:00.000000Z'::timestamp) feb28_leap,\n is_end_of_month(null) nul" + }, + { + "query": "SELECT timestamp, symbol, price\nFROM trades\nWHERE is_end_of_month(timestamp)\nLIMIT 10" + }, + { + "query": "SELECT regr_r2(close, open) AS r2\nFROM market_data_ohlc_1d\nWHERE symbol = 'EURUSD'" + }, + { + "query": "SELECT symbol, regr_r2(close, open) AS r2\nFROM market_data_ohlc_1d\nWHERE symbol IN ('USDCHF', 'EURUSD', 'GBPUSD', 'EURGBP', 'USDHKD')\nORDER BY r2 DESC" + }, + { + "query": "json_extract('{\"name\": \"Lisa\"}', '$.name') -- Lisa" + }, + { + "query": "SELECT current_data_id()" + }, + { + "query": "-- Get the authenticated user of the current session\nSELECT session_user()" + }, + { + "query": "SELECT node_role()" + }, + { + "query": "SELECT query_id, username, state, memory_used, memory_limit, query\nFROM query_activity()" + }, + { + "query": "SELECT * FROM sleep(1)" + }, + { + "query": "SELECT wait_wal_table('trades')" + }, + { + "query": "SELECT wait_wal_table('trades', 42)" + }, + { + "query": "SELECT timestamp, symbol, side,\n avg(price) AS avg_price,\n avg(amount) AS avg_amount\nFROM read_parquet('trades.parquet') TIMESTAMP(timestamp)\nWHERE side = 'buy'\nSAMPLE BY 1m" + }, + { + "query": "SELECT timestamp, symbol, side,\n avg(price) AS avg_price,\n avg(amount) AS avg_amount\nFROM (\n SELECT * FROM read_parquet('trades.parquet')\n WHERE side = 'buy'\n) TIMESTAMP(timestamp)\nSAMPLE BY 1m" + }, + { + "query": "WITH buys AS (\n SELECT * FROM read_parquet('trades.parquet')\n WHERE side = 'buy'\n)\nSELECT timestamp, symbol, side,\n avg(price) AS avg_price,\n avg(amount) AS avg_amount\nFROM buys TIMESTAMP(timestamp)\nSAMPLE BY 1m" + }, + { + "query": "SELECT p.timestamp, p.symbol,\n p.price AS archived_price,\n t.price AS live_price\nFROM read_parquet('trades.parquet') p TIMESTAMP(timestamp)\nASOF JOIN trades t ON (symbol)" + }, + { + "query": "SELECT timestamp, symbol, avg(price) AS avg_price\nFROM (\n SELECT * FROM read_parquet('trades.parquet')\n ORDER BY timestamp\n) TIMESTAMP(timestamp)\nSAMPLE BY 1h" + }, + { + "query": "INSERT INTO trades\nSELECT symbol, side, price, amount, timestamp\nFROM read_parquet('trades.parquet')" + }, + { + "query": "INSERT INTO trades_1m\nSELECT timestamp, symbol, side,\n avg(price) AS avg_price,\n avg(amount) AS avg_amount\nFROM (\n SELECT * FROM read_parquet('trades.parquet')\n WHERE side = 'buy'\n) TIMESTAMP(timestamp)\nSAMPLE BY 1m" + }, + { + "query": "CREATE TABLE trades AS (\n SELECT * FROM read_parquet('trades.parquet')\n) TIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE trades (\n symbol SYMBOL CAPACITY 256,\n side SYMBOL,\n price DOUBLE,\n amount DOUBLE,\n timestamp TIMESTAMP\n) TIMESTAMP(timestamp) PARTITION BY DAY\nDEDUP UPSERT KEYS(timestamp, symbol)" + }, + { + "query": "SELECT rnd_int() FROM long_sequence(5)" + }, + { + "query": "SELECT rnd_int(1,4,0) FROM long_sequence(5)" + }, + { + "query": "SELECT rnd_timestamp_ns(\n to_timestamp('2015', 'yyyy'),\n to_timestamp('2016', 'yyyy'),\n 0)\nFROM long_sequence(5)" + }, + { + "query": "SELECT x, timestamp_sequence_ns(\n to_timestamp_ns('2019-10-17T00:00:00', 'yyyy-MM-ddTHH:mm:ss'),\n 100L -- 100 nanoseconds\n) AS ts\nFROM long_sequence(5)" + }, + { + "query": "SELECT timestamp, symbol,\n round(sum(amount), 2) total,\n bar(sum(amount), 0, 50, 30)\nFROM trades\nWHERE symbol IN ('BTC-USDT', 'ETH-USDT')\nSAMPLE BY 1m\nLIMIT -10" + }, + { + "query": "SELECT timestamp, symbol, round(total, 2) total,\n bar(total, min(total) OVER (PARTITION BY symbol),\n max(total) OVER (PARTITION BY symbol), 30)\nFROM (\n SELECT timestamp, symbol, sum(amount) total\n FROM trades\n WHERE symbol IN ('BTC-USDT', 'ETH-USDT')\n SAMPLE BY 1m\n)\nLIMIT -10" + }, + { + "query": "SELECT timestamp, symbol, round(total, 2) total,\n bar(total, min(total) OVER (),\n max(total) OVER (), 30)\nFROM (\n SELECT timestamp, symbol, sum(amount) total\n FROM trades\n WHERE symbol IN ('BTC-USDT', 'ETH-USDT')\n SAMPLE BY 1m\n)\nLIMIT -10" + }, + { + "query": "SELECT symbol, price,\n bar(price, 0, 100000, 25)\nFROM trades\nLATEST ON timestamp PARTITION BY symbol" + }, + { + "query": "SELECT timestamp, symbol,\n round(avg(price), 0) avg_price,\n sparkline(price, NULL, NULL, 20)\nFROM trades\nWHERE symbol IN ('BTC-USDT', 'ETH-USDT')\n AND timestamp IN '2026-03-07'\nSAMPLE BY 1h\nLIMIT 10" + }, + { + "query": "SELECT symbol, sparkline(price)\nFROM trades\nWHERE timestamp IN '2026-03-07'\nSAMPLE BY 1h" + }, + { + "query": "SELECT symbol, sparkline(amount, 0, 1000000, 24)\nFROM trades\nSAMPLE BY 1d\nLIMIT -5" + }, + { + "query": "SELECT symbol, sparkline(price, 0, NULL, 24)\nFROM trades\nSAMPLE BY 1d\nLIMIT -5" + }, + { + "query": "DECLARE @symbol := 'BTC-USDT'\n\nWITH ohlc AS (\n SELECT\n timestamp AS ts,\n symbol,\n first(price) AS open,\n max(price) AS high,\n min(price) AS low,\n last(price) AS close,\n sum(amount) AS volume\n FROM trades\n WHERE timestamp IN '2024-05-22' AND symbol = @symbol\n SAMPLE BY 1m\n)\nSELECT\n ts,\n symbol,\n open, high, low, close, volume,\n sum((high + low + close) / 3 * volume) OVER w / sum(volume) OVER w AS vwap\nFROM ohlc\nWINDOW w AS (ORDER BY ts CUMULATIVE)\nORDER BY ts" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price) OVER w AS symbol_avg,\n price - avg(price) OVER w AS diff_from_avg\nFROM trades\nWHERE timestamp IN '[$today]'\nWINDOW w AS (PARTITION BY symbol)" + }, + { + "query": "SELECT\n timestamp,\n price,\n lag(price) OVER w AS prev_price,\n price - lag(price) OVER w AS price_change\nFROM trades\nWHERE timestamp IN '[$today]' AND symbol = 'BTC-USDT'\nWINDOW w AS (ORDER BY timestamp)" + }, + { + "query": "WITH prices AS (\n SELECT\n symbol,\n price,\n avg(price) OVER (ORDER BY timestamp) AS moving_avg\n FROM trades\n WHERE timestamp IN '[$today]'\n)\nSELECT * FROM prices\nWHERE moving_avg > 100" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 3 PRECEDING AND CURRENT ROW\n ) AS moving_avg\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price, 'alpha', 0.2) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS ema_alpha\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price, 'period', 10) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS ema_10\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price, 'minute', 5) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS ema_5min\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price, 'alpha', 0.1, amount) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS vwema_alpha\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price, 'period', 10, amount) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS vwema_10\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n avg(price, 'hour', 1, amount) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS vwema_1h\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n ts,\n robot_id,\n corr(motor_temp, joint_velocity) OVER (\n PARTITION BY robot_id\n ORDER BY ts\n ROWS BETWEEN 99 PRECEDING AND CURRENT ROW\n ) AS rolling_corr\nFROM telemetry" + }, + { + "query": "SELECT robot_id, device_corr,\n avg(device_corr) OVER () AS fleet_avg_corr\nFROM (\n SELECT robot_id,\n corr(motor_temp, joint_velocity) AS device_corr\n FROM telemetry\n WHERE ts > dateadd('d', -1, now())\n GROUP BY robot_id\n)" + }, + { + "query": "SELECT\n symbol,\n timestamp,\n count(*) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n RANGE BETWEEN '1' SECOND PRECEDING AND CURRENT ROW\n ) AS trades_last_second\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n ts,\n robot_id,\n covar_pop(motor_temp, joint_velocity) OVER (\n PARTITION BY robot_id\n ORDER BY ts\n ROWS BETWEEN 99 PRECEDING AND CURRENT ROW\n ) AS temp_vel_covariance\nFROM telemetry" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n first_value(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS first_price,\n first_value(price) IGNORE NULLS OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS first_non_null_price\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n ksum(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) AS cumulative_price\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n ksum(price) OVER (\n ORDER BY timestamp\n ROWS BETWEEN 3 PRECEDING AND CURRENT ROW\n ) AS rolling_sum\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n timestamp,\n price,\n last_value(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 2 PRECEDING AND CURRENT ROW\n ) AS last_price,\n last_value(price) IGNORE NULLS OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS last_non_null_price\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n max(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 3 PRECEDING AND CURRENT ROW\n ) AS highest_price\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n min(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 3 PRECEDING AND CURRENT ROW\n ) AS lowest_price\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n nth_value(price, 3) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS 4 PRECEDING\n ) AS third_price\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n first_value(price) OVER w AS first_price,\n nth_value(price, 1) OVER w AS nth_1,\n nth_value(price, 2) OVER w AS nth_2,\n nth_value(price, 3) OVER w AS nth_3\nFROM trades\nWHERE timestamp IN '$today' AND symbol = 'BTC-USDT'\nWINDOW w AS (ORDER BY timestamp ROWS 2 PRECEDING)" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n stddev_pop(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 19 PRECEDING AND CURRENT ROW\n ) AS volatility_20\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT robot_id, robot_avg,\n (robot_avg - avg(robot_avg) OVER ()) / stddev(robot_avg) OVER () AS z_score\nFROM (\n SELECT robot_id, avg(motor_temp) AS robot_avg\n FROM telemetry\n WHERE ts > dateadd('d', -1, now())\n GROUP BY robot_id\n)\nORDER BY z_score DESC" + }, + { + "query": "SELECT\n symbol,\n amount,\n timestamp,\n sum(amount) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW\n ) AS cumulative_amount\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n var_pop(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 9 PRECEDING AND CURRENT ROW\n ) AS price_variance\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n cume_dist() OVER (\n PARTITION BY symbol\n ORDER BY price\n ) AS price_cdf\nFROM trades\nWHERE timestamp IN '$today' AND symbol = 'BTC-USDT'\nORDER BY price DESC" + }, + { + "query": "SELECT ts, val,\n cume_dist() OVER (ORDER BY val) AS cd\nFROM tab" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n dense_rank() OVER (\n PARTITION BY symbol\n ORDER BY price DESC\n ) AS price_rank\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n ntile(4) OVER (\n PARTITION BY symbol\n ORDER BY price\n ) AS price_quartile\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT ts, val,\n ntile(3) OVER (ORDER BY ts) AS bucket\nFROM tab" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n percent_rank() OVER (\n PARTITION BY symbol\n ORDER BY price DESC\n ) AS price_percentile\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n rank() OVER (ORDER BY price DESC) AS rank,\n percent_rank() OVER (ORDER BY price DESC) AS percent_rank\nFROM trades\nWHERE timestamp IN '$today'\n AND symbol = 'BTC-USDT'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n rank() OVER (\n PARTITION BY symbol\n ORDER BY price DESC\n ) AS price_rank\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n symbol,\n price,\n timestamp,\n row_number() OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS trade_number\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n timestamp,\n price,\n lag(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS previous_price,\n lag(price, 2, 0.0) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS price_two_rows_back\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT\n timestamp,\n price,\n lead(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS next_price,\n lead(price, 2, 0.0) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ) AS price_after_next\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "DECLARE @best_bid := bids[1,1]\nSELECT\n timestamp,\n symbol,\n @best_bid AS best_bid,\n avg(@best_bid) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS BETWEEN 3 PRECEDING AND CURRENT ROW\n ) AS bid_moving_avg\nFROM market_data\nWHERE timestamp IN '$today'" + }, + { + "query": "DECLARE\n @best_bid := bids[1,1],\n @volume_l1 := bids[2,1]\nSELECT\n timestamp, symbol,\n @best_bid AS bid_price_l1,\n @volume_l1 AS bid_volume_l1,\n sum(@volume_l1) OVER (\n PARTITION BY symbol ORDER BY timestamp\n ROWS BETWEEN 5 PRECEDING AND CURRENT ROW\n ) AS bid_volume_l1_5rows\nFROM market_data\nWHERE timestamp IN '$today'" + }, + { + "query": "DECLARE\n @best_bid := bids[1,1],\n @volume_l1 := bids[2,1]\nSELECT\n timestamp,\n sum(@volume_l1) OVER (\n ORDER BY timestamp\n RANGE BETWEEN '1' MINUTE PRECEDING AND CURRENT ROW\n ) AS bid_volume_1min\nFROM market_data\nWHERE timestamp IN '$today' AND symbol = 'GBPUSD'" + }, + { + "query": "SELECT\n timestamp,\n symbol,\n COUNT(*) OVER w AS updates_per_min,\n COUNT(CASE WHEN side = 'buy' THEN 1 END) OVER w AS buys_per_minute,\n COUNT(CASE WHEN side = 'sell' THEN 1 END) OVER w AS sells_per_minute\nFROM trades\nWHERE timestamp IN '$today' AND symbol = 'BTC-USDT'\nWINDOW w AS (ORDER BY timestamp RANGE BETWEEN 60000000 PRECEDING AND CURRENT ROW)" + }, + { + "query": "DECLARE @symbol := 'BTC-USDT'\n\nWITH ohlc AS (\n SELECT\n timestamp AS ts,\n first(price) AS open,\n max(price) AS high,\n min(price) AS low,\n last(price) AS close,\n sum(amount) AS volume\n FROM trades\n WHERE timestamp IN '2024-05-22' AND symbol = @symbol\n SAMPLE BY 1m\n)\nSELECT\n ts, open, high, low, close, volume,\n sum((high + low + close) / 3 * volume) OVER w / sum(volume) OVER w AS vwap\nFROM ohlc\nWINDOW w AS (ORDER BY ts CUMULATIVE)" + }, + { + "query": "SELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER w AS avg_price,\n min(price) OVER w AS min_price,\n max(price) OVER w AS max_price\nFROM trades\nWHERE timestamp IN '[$today]' AND symbol = 'BTC-USDT'\nWINDOW w AS (ORDER BY timestamp ROWS BETWEEN 9 PRECEDING AND CURRENT ROW)\nLIMIT 100" + }, + { + "query": "SELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER short_window AS avg_10,\n avg(price) OVER long_window AS avg_50\nFROM trades\nWHERE timestamp IN '[$today]' AND symbol = 'BTC-USDT'\nWINDOW\n short_window AS (ORDER BY timestamp ROWS BETWEEN 9 PRECEDING AND CURRENT ROW),\n long_window AS (ORDER BY timestamp ROWS BETWEEN 49 PRECEDING AND CURRENT ROW)\nLIMIT 100" + }, + { + "query": "SELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER w AS moving_avg,\n row_number() OVER (PARTITION BY symbol ORDER BY timestamp) AS seq\nFROM trades\nWHERE timestamp IN '[$today]' AND symbol = 'BTC-USDT'\nWINDOW w AS (ORDER BY timestamp ROWS BETWEEN 9 PRECEDING AND CURRENT ROW)\nLIMIT 100" + }, + { + "query": "WITH price_stats AS (\n SELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER w AS moving_avg,\n price - avg(price) OVER w AS deviation\n FROM trades\n WHERE timestamp IN '[$today]' AND symbol = 'BTC-USDT'\n WINDOW w AS (ORDER BY timestamp ROWS BETWEEN 19 PRECEDING AND CURRENT ROW)\n)\nSELECT * FROM price_stats\nWHERE deviation > 10\nLIMIT 100" + }, + { + "query": "SELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER w1 AS symbol_avg,\n avg(price) OVER w2 AS moving_avg\nFROM trades\nWHERE timestamp IN '[$today]' AND symbol = 'BTC-USDT'\nWINDOW\n w1 AS (ORDER BY timestamp),\n w2 AS (w1 ROWS BETWEEN 9 PRECEDING AND CURRENT ROW)\nLIMIT 100" + }, + { + "query": "SELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER w3 AS moving_avg\nFROM trades\nWHERE timestamp IN '[$today]'\nWINDOW\n w1 AS (PARTITION BY symbol),\n w2 AS (w1 ORDER BY timestamp),\n w3 AS (w2 ROWS BETWEEN 9 PRECEDING AND CURRENT ROW)\nLIMIT 100" + }, + { + "query": "WITH prices_and_avg AS (\n SELECT\n symbol,\n price,\n avg(price) OVER (ORDER BY timestamp) AS moving_avg_price,\n timestamp\n FROM trades\n WHERE timestamp IN '[$today]'\n)\nSELECT * FROM prices_and_avg\nWHERE moving_avg_price > 100" + }, + { + "query": "-- Workdays only with timezone\nSELECT * FROM trades WHERE ts IN '[2024-01]T09:30@EST#wd;6h30m'" + }, + { + "query": "-- NYSE regular trading hours for January, holidays excluded automatically\nSELECT * FROM trades\nWHERE ts IN '[2025-01]#XNYS'" + }, + { + "query": "SELECT *\nFROM a, b\nWHERE a.id = b.id" + }, + { + "query": "-- NYSE trading hours on workdays for January\nSELECT * FROM trades\nWHERE ts IN '[2025-01]T09:30@America/New_York#workday;6h30m'" + }, + { + "query": "-- Jan 15 (1 day) + all of February (29 days) = 30 intervals\nSELECT * FROM trades WHERE ts IN '[2024-01-15, 2024-02]T09:30'" + }, + { + "query": "-- Two full months, workdays only\nSELECT * FROM trades\nWHERE ts IN '[2024-01, 2024-02]T09:30@America/New_York#workday;6h30m'" + }, + { + "query": "-- Workdays only (Monday-Friday)\nSELECT * FROM trades WHERE ts IN '[2024-01]#workday'" + }, + { + "query": "-- Weekends only\nSELECT * FROM logs WHERE ts IN '[2024-01]T02:00#weekend;4h'" + }, + { + "query": "-- Specific days\nSELECT * FROM attendance WHERE ts IN '[2024-01]#Mon,Wed,Fri'" + }, + { + "query": "-- NYSE trading hours for January workdays\nSELECT * FROM nyse_trades\nWHERE ts IN '[2024-01]T09:30@America/New_York#workday;6h30m'" + }, + { + "query": "-- Weekend maintenance (every Sat/Sun in January at 02:00)\nSELECT * FROM system_logs\nWHERE ts IN '[2024-01]T02:00#weekend;4h'" + }, + { + "query": "-- cap queries of the group's members at 2 GiB of native memory\nALTER GROUP analysts SET MEMORY LIMIT 2G" + }, + { + "query": "-- remove the limit\nALTER GROUP analysts SET MEMORY LIMIT UNLIMITED" + }, + { + "query": "ALTER GROUP analysts WITH EXTERNAL ALIAS 'CN=Analysts,OU=Users,DC=example,DC=com'" + }, + { + "query": "ALTER GROUP analysts DROP EXTERNAL ALIAS 'CN=Analysts,OU=Users,DC=example,DC=com'" + }, + { + "query": "-- cap the service account's queries at 1 GiB of native memory\nALTER SERVICE ACCOUNT client_app SET MEMORY LIMIT 1G" + }, + { + "query": "-- remove the limit\nALTER SERVICE ACCOUNT client_app SET MEMORY LIMIT UNLIMITED" + }, + { + "query": "-- cap the user's queries at 512 MiB of native memory\nALTER USER john SET MEMORY LIMIT 512M" + }, + { + "query": "-- remove the limit\nALTER USER john SET MEMORY LIMIT UNLIMITED" + }, + { + "query": "CREATE GROUP analysts WITH EXTERNAL ALIAS 'CN=Analysts,OU=Users,DC=example,DC=com'" + }, + { + "query": "CREATE TABLE trades (\n symbol SYMBOL, price DOUBLE, quantity DOUBLE,\n counterparty SYMBOL, ts TIMESTAMP\n) TIMESTAMP(ts)" + }, + { + "query": "GRANT SELECT ON trades(*) TO john" + }, + { + "query": "GRANT SELECT ON trades(* EXCLUDE (counterparty)) TO john" + }, + { + "query": "REVOKE SELECT ON trades(*) FROM john" + }, + { + "query": "REVOKE SELECT ON trades(* EXCLUDE (symbol, ts)) FROM john" + }, + { + "query": "ALTER MATERIALIZED VIEW trades_hourly\n ALTER COLUMN symbol ADD INDEX TYPE POSTING" + }, + { + "query": "SELECT name, suspended, writerTxn, sequencerTxn, errorTag\nFROM wal_tables()\nWHERE suspended" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN side ADD INDEX" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN instrument ADD INDEX TYPE POSTING" + }, + { + "query": "-- Force delta + Frame-of-Reference (benchmarking)\nALTER TABLE trades ALTER COLUMN instrument ADD INDEX TYPE POSTING DELTA" + }, + { + "query": "-- Force Elias-Fano (benchmarking)\nALTER TABLE trades ALTER COLUMN instrument ADD INDEX TYPE POSTING EF" + }, + { + "query": "ALTER TABLE trades\n ALTER COLUMN symbol ADD INDEX TYPE POSTING INCLUDE (price, quantity)" + }, + { + "query": "-- This query reads from the index sidecar, not from column files\nSELECT timestamp, price FROM trades WHERE symbol = 'AAPL'" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN side NOCACHE" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN side DROP INDEX" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN price SET PARQUET(rle_dictionary)" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN price SET PARQUET(default, zstd(3))" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN price SET PARQUET(rle_dictionary, zstd(3))" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN price SET PARQUET(default)" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN symbol SET PARQUET(default, BLOOM_FILTER)" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN symbol SET PARQUET(rle_dictionary, zstd(3), BLOOM_FILTER)" + }, + { + "query": "ALTER TABLE trades ALTER COLUMN symbol SET PARQUET(rle_dictionary)" + }, + { + "query": "ALTER TABLE fx_trades ALTER COLUMN counterparty TYPE VARCHAR" + }, + { + "query": "ALTER TABLE fx_trades ALTER COLUMN counterparty TYPE SYMBOL CAPACITY 10000 CACHE" + }, + { + "query": "ALTER TABLE fx_trades ALTER COLUMN counterparty SYMBOL CAPACITY 512" + }, + { + "query": "--DAY\nALTER TABLE trades DROP PARTITION LIST '2019-05-18'" + }, + { + "query": "--MONTH\nALTER TABLE trades DROP PARTITION LIST '2019-05'" + }, + { + "query": "--YEAR\nALTER TABLE trades DROP PARTITION LIST '2019'" + }, + { + "query": "ALTER TABLE trades DROP PARTITION LIST '2018','2019'" + }, + { + "query": "ALTER TABLE trades\nDROP PARTITION\nWHERE timestamp = to_timestamp('2019-01-01:00:00:00', 'yyyy-MM-dd:HH:mm:ss')" + }, + { + "query": "ALTER TABLE trades\nDROP PARTITION\nWHERE timestamp < to_timestamp('2018-01-01:00:00:00', 'yyyy-MM-dd:HH:mm:ss')" + }, + { + "query": "SELECT name, suspended, writerTxn, sequencerTxn\nFROM wal_tables()\nWHERE name = 'trades'" + }, + { + "query": "ALTER TABLE trades SET FORMAT PARQUET" + }, + { + "query": "ALTER TABLE table_name ENABLE STORAGE POLICY" + }, + { + "query": "ALTER TABLE table_name DISABLE STORAGE POLICY" + }, + { + "query": "ALTER TABLE table_name DROP STORAGE POLICY" + }, + { + "query": "ALTER TABLE sensor_data SET STORAGE POLICY(\n TO PARQUET 3 DAYS,\n DROP LOCAL 1 MONTH\n)" + }, + { + "query": "ALTER TABLE trades SET STORAGE POLICY(\n TO PARQUET 7 DAYS,\n TO REMOTE 14 DAYS,\n DROP LOCAL 30 DAYS,\n DROP REMOTE 7 YEARS\n)" + }, + { + "query": "ALTER TABLE sensor_data SET STORAGE POLICY(TO PARQUET 7d)" + }, + { + "query": "ALTER TABLE sensor_data DISABLE STORAGE POLICY" + }, + { + "query": "ALTER TABLE sensor_data ENABLE STORAGE POLICY" + }, + { + "query": "ALTER TABLE sensor_data DROP STORAGE POLICY" + }, + { + "query": "SHOW CREATE TABLE sensor_data" + }, + { + "query": "ALTER TABLE weather SET TTL 0h" + }, + { + "query": "ALTER TABLE trades SUSPEND WAL" + }, + { + "query": "INSERT INTO trades VALUES ('2026-08-28T10:00:00.000000Z', 'EURUSD', 1.0842)" + }, + { + "query": "SELECT count() FROM trades" + }, + { + "query": "-- Original view\nCREATE VIEW summary AS (\n SELECT ts, symbol, max(price) as max_price\n FROM trades\n SAMPLE BY 1h\n)" + }, + { + "query": "-- Alter to change aggregation\nALTER VIEW summary AS (\n SELECT ts, symbol, avg(price) as avg_price\n FROM trades\n SAMPLE BY 1h\n)" + }, + { + "query": "-- Original view\nCREATE VIEW trade_view AS (\n SELECT ts, symbol, price FROM trades\n)" + }, + { + "query": "-- Add volume column\nALTER VIEW trade_view AS (\n SELECT ts, symbol, price, quantity FROM trades\n)" + }, + { + "query": "-- Original view\nCREATE VIEW filtered AS (\n SELECT * FROM trades WHERE price > 100\n)" + }, + { + "query": "-- Change filter threshold\nALTER VIEW filtered AS (\n SELECT * FROM trades WHERE price > 200\n)" + }, + { + "query": "-- Original view with parameter\nCREATE VIEW by_price AS (\n DECLARE @min := 0\n SELECT * FROM trades WHERE price >= @min\n)" + }, + { + "query": "-- Change default value\nALTER VIEW by_price AS (\n DECLARE @min := 100\n SELECT * FROM trades WHERE price >= @min\n)" + }, + { + "query": "-- Original view without parameters\nCREATE VIEW trades_filtered AS (\n SELECT * FROM trades WHERE price > 100\n)" + }, + { + "query": "-- Add parameter\nALTER VIEW trades_filtered AS (\n DECLARE @threshold := 100\n SELECT * FROM trades WHERE price > @threshold\n)" + }, + { + "query": "-- The 'market_data' table has 'timestamp' as its designated timestamp.\n-- Even though 'timestamp' is not selected in the subquery,\n-- it is used implicitly for the ASOF JOIN.\nWITH market_subset AS (\n SELECT symbol,bids\n FROM market_data\n WHERE timestamp IN '$today'\n)\nSELECT *\nFROM market_subset ASOF JOIN core_price ON (symbol)" + }, + { + "query": "WITH trades_ordered_by_ingestion AS (\n SELECT symbol, price, ingestion_time\n FROM trades\n WHERE timestamp IN '$today'\n -- This ORDER BY clause tells QuestDB to use 'ingestion_time'\n -- as the new designated timestamp for this subquery.\n ORDER BY ingestion_time ASC\n)\n-- No extra syntax is needed here. The ASOF JOIN automatically uses\n-- the new designated timestamp from the subquery.\nSELECT *\nFROM trades_ordered_by_ingestion\nASOF JOIN quotes ON (symbol)" + }, + { + "query": "SELECT market_data.timestamp, market_data.symbol, bids, core_price.*\nFROM market_data\nASOF JOIN core_price ON (symbol) TOLERANCE 50T\nWHERE market_data.timestamp IN '$today'" + }, + { + "query": "SELECT symbol, side,\n CASE\n WHEN side = 'buy' THEN 'bullish'\n ELSE 'bearish'\n END AS sentiment\nFROM trades\nLIMIT -40" + }, + { + "query": "SELECT symbol, side,\n CASE\n WHEN side = 'buy' THEN 'bullish'\n END AS sentiment\nFROM trades\nLIMIT -40" + }, + { + "query": "COPY (SELECT * FROM trades WHERE timestamp IN '$today' AND symbol = 'BTC-USDT')\nTO 'btc_today'\nWITH FORMAT PARQUET" + }, + { + "query": "COPY trades TO 'trades_bloom'\nWITH FORMAT PARQUET\nBLOOM_FILTER_COLUMNS 'symbol,side'" + }, + { + "query": "COPY (\n SELECT\n timestamp,\n symbol,\n first(price) AS open,\n max(price) AS high,\n min(price) AS low,\n last(price) AS close,\n sum(amount) AS volume\n FROM trades\n WHERE timestamp IN '$now-7d..$now'\n SAMPLE BY 1h\n)\nTO 'ohlcv_7d'\nWITH FORMAT PARQUET" + }, + { + "query": "SELECT ts, \"table\", destination, status, rows_exported\nFROM sys.copy_export_log\nWHERE ts IN '$now-1d..$now'\nORDER BY ts DESC" + }, + { + "query": "COPY 'operationId' CANCEL" + }, + { + "query": "CREATE LIVE VIEW trades_ma\nFLUSH EVERY 1s\nSTART FROM NOW\nAS\nSELECT timestamp, symbol,\n avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING)\n AS moving_avg\nFROM trades" + }, + { + "query": "CREATE LIVE VIEW trades_ma\nFLUSH EVERY 1s\nIN MEMORY 5s\nSTART FROM NOW\nAS\nSELECT timestamp, symbol,\n avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING)\n AS moving_avg\nFROM trades" + }, + { + "query": "CREATE LIVE VIEW trades_ma\nFLUSH EVERY 1s\nPARTITION BY HOUR\nSTART FROM NOW\nAS\nSELECT timestamp, symbol,\n avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING)\n AS moving_avg\nFROM trades" + }, + { + "query": "CREATE LIVE VIEW trades_ma_from_april\nFLUSH EVERY 1s\nSTART FROM '2026-04-01T00:00:00.000000Z'\nAS\nSELECT timestamp, symbol,\n avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING)\n AS moving_avg\nFROM trades" + }, + { + "query": "CREATE LIVE VIEW trades_hourly_volume\nFLUSH EVERY 1s\nSTART FROM NOW\nAS\nSELECT timestamp, symbol,\n sum(amount) OVER w AS bucket_volume\nFROM trades\nWINDOW w AS (\n PARTITION BY symbol\n ORDER BY timestamp\n ANCHOR EXPRESSION timestamp_floor('1h', timestamp)\n)" + }, + { + "query": "CREATE LIVE VIEW IF NOT EXISTS trades_ma\nFLUSH EVERY 1s\nIN MEMORY 5s\nPARTITION BY HOUR\nSTART FROM BEGINNING\nAS\nSELECT\n timestamp,\n symbol,\n price,\n avg(price) OVER (\n PARTITION BY symbol\n ORDER BY timestamp\n ROWS 300 PRECEDING\n ) AS moving_avg\nFROM trades" + }, + { + "query": "SELECT view_name, base_table_name, view_status, lag_seqtxn\nFROM live_views()" + }, + { + "query": "GRANT CREATE LIVE VIEW TO user1" + }, + { + "query": "CREATE LIVE VIEW trades_ma\nFLUSH EVERY 1s\nSTART FROM NOW\nAS\nSELECT timestamp, symbol,\n avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING)\n AS moving_avg\nFROM trades\nOWNED BY analysts" + }, + { + "query": "CREATE MATERIALIZED VIEW IF NOT EXISTS trades_hourly_stats\nWITH BASE trades\nREFRESH EVERY 15m\n START '2025-01-01T00:00:00Z'\n TIME ZONE 'UTC'\n PERIOD (LENGTH 1h DELAY 5m)\nAS (\n SELECT\n timestamp,\n symbol,\n avg(price) AS avg_price,\n sum(amount) AS total_volume\n FROM trades\n SAMPLE BY 1h\n)\nPARTITION BY DAY TTL 30 DAYS" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL,\n price DOUBLE,\n amount DOUBLE\n) TIMESTAMP(timestamp)\nPARTITION BY DAY\nSTORAGE POLICY(TO PARQUET 3d, DROP LOCAL 1M)" + }, + { + "query": "CREATE TABLE sensors (\n ts TIMESTAMP,\n temperature DOUBLE PARQUET(rle_dictionary, zstd(3)),\n humidity FLOAT PARQUET(rle_dictionary),\n device_id VARCHAR PARQUET(default, lz4_raw, BLOOM_FILTER),\n status INT\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE t (\n a VARCHAR PARQUET(BLOOM_FILTER),\n ts TIMESTAMP\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE t (\n a INT PARQUET(delta_binary_packed, BLOOM_FILTER),\n ts TIMESTAMP\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE t (\n a INT PARQUET(delta_binary_packed, zstd(3), BLOOM_FILTER),\n ts TIMESTAMP\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL INDEX,\n price DOUBLE,\n amount DOUBLE\n) TIMESTAMP(timestamp)" + }, + { + "query": "-- Inline syntax\nCREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING,\n price DOUBLE,\n amount DOUBLE\n) TIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "-- Out-of-line syntax\nCREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL,\n price DOUBLE,\n amount DOUBLE\n), INDEX(symbol TYPE POSTING)\nTIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE trades (\n timestamp TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING INCLUDE (price, exchange),\n exchange SYMBOL,\n price DOUBLE,\n amount DOUBLE\n) TIMESTAMP(timestamp) PARTITION BY DAY" + }, + { + "query": "CREATE VIEW 日本語ビュー AS (SELECT * FROM trades)" + }, + { + "query": "CREATE VIEW Részvény_árak AS (SELECT * FROM prices)" + }, + { + "query": "DECLARE\n @x := 5\nSELECT @x" + }, + { + "query": "DECLARE\n @x := 5,\n @y := 2\nSELECT @x + @y" + }, + { + "query": "DECLARE\n @today := today(),\n @start := interval_start(@today),\n @end := interval_end(@today)\nSELECT @today = interval(@start, @end)" + }, + { + "query": "DECLARE\n @x := 5\nSELECT y FROM (\n SELECT @x AS y\n)" + }, + { + "query": "DECLARE\n @x := 5\nSELECT @x + y FROM (\n DECLARE @x := 10\n SELECT @x AS y\n)" + }, + { + "query": "DECLARE\n @subquery := (SELECT timestamp FROM trades)\nSELECT * FROM @subquery" + }, + { + "query": "DECLARE\n @timestamp := timestamp,\n @symbol := symbol,\n @subquery := (SELECT @timestamp, @symbol FROM trades)\nSELECT * FROM @subquery" + }, + { + "query": "DECLARE\n @x := 5\nWITH first AS (\n DECLARE @x := 10\n SELECT @x as a -- a = 10\n),\nsecond AS (\n DECLARE @y := 4\n SELECT\n @x + @y as b, -- b = 5 + 4 = 9\n a -- a = 10\n FROM first\n)\nSELECT a, b\nFROM second" + }, + { + "query": "SELECT DISTINCT symbol\nFROM fx_trades\nWHERE timestamp IN '$now - 1h..$now'" + }, + { + "query": "SELECT DISTINCT symbol, count()\nFROM fx_trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT DISTINCT symbol, side, count()\nFROM fx_trades\nWHERE timestamp IN '$today'\n AND price > 1" + }, + { + "query": "DROP LIVE VIEW trades_ma" + }, + { + "query": "DROP LIVE VIEW IF EXISTS trades_ma" + }, + { + "query": "GRANT DROP LIVE VIEW ON trades_ma TO user1" + }, + { + "query": "SELECT symbol, avg(price), count()\nFROM fx_trades\nWHERE timestamp IN '$today'\nGROUP BY symbol\nLIMIT 5" + }, + { + "query": "SELECT symbol, avg(price), count()\nFROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, side, avg(price), count()\nFROM fx_trades\nWHERE timestamp IN '$today'\nGROUP BY symbol, side\nLIMIT 5" + }, + { + "query": "SELECT symbol, side, avg(price), count()\nFROM fx_trades\nWHERE timestamp IN '$today'\nLIMIT 5" + }, + { + "query": "SELECT symbol, avg(price)\nFROM fx_trades\nWHERE timestamp IN '$today'\nGROUP BY symbol, side\nORDER BY symbol\nLIMIT 6" + }, + { + "query": "SELECT\n h.offset / 1_000_000_000 AS horizon_sec,\n t.symbol,\n avg((m.best_bid + m.best_ask) / 2) AS avg_mid\nFROM fx_trades AS t\nHORIZON JOIN market_data AS m ON (symbol)\nRANGE FROM 0s TO 1m STEP 5s AS h\nWHERE t.timestamp IN '$now-1h..$now'\nORDER BY t.symbol, horizon_sec" + }, + { + "query": "SELECT\n h.offset / 1_000_000_000 AS horizon_sec,\n t.symbol,\n avg((m.best_bid + m.best_ask) / 2) AS avg_mid,\n count() AS sample_size\nFROM fx_trades AS t\nHORIZON JOIN market_data AS m ON (symbol)\nRANGE FROM -5s TO 5s STEP 1s AS h\nWHERE t.timestamp IN '$now-1h..$now'\nORDER BY t.symbol, horizon_sec" + }, + { + "query": "SELECT\n h.offset / 1_000_000_000 AS horizon_sec,\n sum(((m.best_bid + m.best_ask) / 2 - t.price) * t.quantity)\n / sum(t.quantity) AS vwap_markout\nFROM fx_trades AS t\nHORIZON JOIN market_data AS m ON (symbol)\nRANGE FROM 0s TO 5m STEP 30s AS h\nWHERE t.timestamp IN '$now-1h..$now'\nORDER BY horizon_sec" + }, + { + "query": "SELECT\n h.offset / 1000000 AS horizon_ms,\n t.symbol,\n avg(m.best_bid) AS consolidated_bid,\n avg(c.bid_price) AS ecn_bid\nFROM fx_trades AS t\nHORIZON JOIN market_data AS m\n ON (t.symbol = m.symbol)\nHORIZON JOIN core_price AS c\n ON (t.symbol = c.symbol AND t.ecn = c.ecn)\n LIST (-1s, 0, 1s, 5s) AS h\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nGROUP BY horizon_ms, t.symbol\nORDER BY t.symbol, horizon_ms" + }, + { + "query": "SELECT\n h.offset / 1_000_000_000 AS horizon_sec,\n t.symbol,\n avg(b.bid) AS avg_bid,\n avg(a.ask) AS avg_ask,\n avg(a.ask - b.bid) AS avg_spread\nFROM trades AS t\nHORIZON JOIN bids AS b ON (t.symbol = b.symbol)\nHORIZON JOIN asks AS a ON (t.symbol = a.symbol)\n RANGE FROM -2s TO 2s STEP 2s AS h\nGROUP BY horizon_sec, t.symbol\nORDER BY t.symbol, horizon_sec" + }, + { + "query": "SELECT\n avg(p.price) AS avg_price,\n avg(r.rate) AS avg_rate\nFROM trades AS t\nHORIZON JOIN prices AS p ON (t.symbol = p.symbol)\nHORIZON JOIN rates AS r\n LIST (0, 1s, 5s) AS h" + }, + { + "query": "SELECT\n avg(b.bid) AS avg_bid,\n avg(a.ask) AS avg_ask,\n avg(m.mid) AS avg_mid\nFROM trades AS t\nHORIZON JOIN bids AS b ON (t.symbol = b.symbol)\nHORIZON JOIN asks AS a ON (t.symbol = a.symbol)\nHORIZON JOIN mids AS m ON (t.symbol = m.symbol)\n LIST (0) AS h" + }, + { + "query": "EXPLAIN SELECT\n h.offset / 1_000_000_000 AS horizon_sec,\n t.symbol,\n avg((m.best_bid + m.best_ask) / 2) AS avg_mid\nFROM fx_trades AS t\nHORIZON JOIN market_data AS m ON (symbol)\nRANGE FROM -1m TO 1m STEP 5s AS h\nWHERE t.timestamp IN '$now-1h..$now'\nORDER BY t.symbol, horizon_sec" + }, + { + "query": "WITH\n many_trades AS\n (SELECT * FROM trades LIMIT -1000000),\n lookup AS\n (SELECT 'BTC-USDT' AS symbol, 'Bitcoin/USDT Pair' AS description)\nSELECT *\nFROM lookup\nINNER JOIN many_trades\n ON lookup.symbol = many_trades.symbol" + }, + { + "query": "WITH\n many_trades AS\n (SELECT * FROM trades LIMIT -1000000),\n lookup AS\n (SELECT 'BTC-USDT' AS symbol, 'Bitcoin/USDT Pair' AS description)\nSELECT *\nFROM many_trades\nINNER JOIN lookup\n ON lookup.symbol = many_trades.symbol" + }, + { + "query": "WITH\n mayTrades AS (\n SELECT symbol, side, COUNT(*) as total\n FROM trades\n WHERE timestamp in '2024-05'\n ORDER BY Symbol\n LIMIT 4\n ),\n juneTrades AS (\n SELECT symbol, side, COUNT(*) as total\n FROM trades\n WHERE timestamp in '2024-06'\n ORDER BY Symbol\n LIMIT 4\n )\nSELECT mayTrades.symbol, juneTrades.symbol,\n mayTrades.side, juneTrades.side,\n mayTrades.total, juneTrades.total\nFROM mayTrades\nJOIN juneTrades\n ON mayTrades.symbol = juneTrades.symbol\n AND mayTrades.side = juneTrades.side" + }, + { + "query": "WITH\n mayTrades AS (\n SELECT symbol, side, COUNT(*) as total\n FROM trades\n WHERE timestamp in '2024-05'\n ORDER BY Symbol\n LIMIT 4\n ),\n juneTrades AS (\n SELECT symbol, side, COUNT(*) as total\n FROM trades\n WHERE timestamp in '2024-06'\n ORDER BY Symbol\n LIMIT 4\n )\nSELECT mayTrades.symbol, juneTrades.symbol,\n mayTrades.side, juneTrades.side,\n mayTrades.total, juneTrades.total\nFROM mayTrades\nJOIN juneTrades ON (symbol, side)" + }, + { + "query": "WITH\n many_trades AS\n (SELECT * FROM trades LIMIT -100),\n lookup AS\n (SELECT 'BTC-USDT' AS symbol, 'Bitcoin/USDT Pair' AS description)\nSELECT *\nFROM many_trades\nLEFT OUTER JOIN lookup\n ON lookup.symbol = many_trades.symbol" + }, + { + "query": "WITH\n many_trades AS\n (SELECT * FROM trades LIMIT -100),\n lookup AS\n (SELECT 'BTC-USDT' AS symbol, 'Bitcoin/USDT Pair' AS description)\nSELECT *\nFROM many_trades\nLEFT JOIN lookup\n ON lookup.symbol = many_trades.symbol" + }, + { + "query": "WITH\n many_trades AS\n (SELECT * FROM trades LIMIT -100),\n lookup AS\n (SELECT 'BTC-USDT' AS symbol, 'Bitcoin/USDT Pair' AS description)\nSELECT *\nFROM many_trades\nLEFT OUTER JOIN lookup\n ON lookup.symbol = many_trades.symbol\nWHERE lookup.symbol = NULL" + }, + { + "query": "WITH\n many_trades AS\n (SELECT * FROM trades LIMIT -100),\n lookup AS\n (SELECT 'BTC-USDT' AS symbol, 'Bitcoin/USDT Pair' AS description)\nSELECT *\nFROM many_trades\nRIGHT OUTER JOIN lookup\n ON lookup.symbol = many_trades.symbol" + }, + { + "query": "WITH\n may_trades AS (\n SELECT symbol, COUNT(*) AS may_total\n FROM trades\n WHERE timestamp IN '2024-05'\n ),\n june_trades AS (\n SELECT symbol, COUNT(*) AS june_total\n FROM trades\n WHERE timestamp IN '2024-06'\n )\nSELECT\n COALESCE(may_trades.symbol, june_trades.symbol) AS symbol,\n may_total,\n june_total\nFROM may_trades\nFULL OUTER JOIN june_trades\n ON may_trades.symbol = june_trades.symbol" + }, + { + "query": "WITH t AS (\n SELECT * FROM trades LIMIT -10000\n)\nSELECT * FROM t CROSS JOIN t AS t2\nWHERE t.timestamp < t2.timestamp\n AND datediff('s', t.timestamp, t2.timestamp) < 10\n AND t.symbol = t2.symbol\n AND t.side = t2.side\n AND t.price = t2.price\n AND t.amount = t2.amount" + }, + { + "query": "WITH miniTrades AS (\n SELECT timestamp, price\n FROM TRADES\n WHERE symbol = 'BTC-USD'\n LIMIT 3\n)\nSELECT tradesA.timestamp, tradesB.timestamp, tradesA.price\nFROM miniTrades tradesA\nLT JOIN miniTrades tradesB" + }, + { + "query": "CREATE TABLE orders (\n id INT,\n desk SYMBOL,\n min_qty DOUBLE,\n ts TIMESTAMP\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE fills (\n id INT,\n order_id INT,\n qty DOUBLE,\n ts TIMESTAMP\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "INSERT INTO orders VALUES\n (1, 'eq', 15.0, '2024-01-01T00:00:00.000000Z'),\n (2, 'fi', 35.0, '2024-01-01T01:00:00.000000Z'),\n (3, 'cmd', 5.0, '2024-01-01T02:00:00.000000Z')" + }, + { + "query": "INSERT INTO fills VALUES\n (1, 1, 10.0, '2024-01-01T00:10:00.000000Z'),\n (2, 1, 20.0, '2024-01-01T00:40:00.000000Z'),\n (3, 1, 30.0, '2024-01-01T01:10:00.000000Z'),\n (4, 2, 40.0, '2024-01-01T01:10:00.000000Z'),\n (5, 2, 50.0, '2024-01-01T01:40:00.000000Z')" + }, + { + "query": "SELECT o.id, o.desk, t.qty\nFROM orders o\nJOIN LATERAL (\n SELECT qty FROM fills WHERE order_id = o.id\n) t\nORDER BY o.id, t.qty" + }, + { + "query": "SELECT o.id, o.desk, t.qty\nFROM orders o\nLEFT JOIN LATERAL (\n SELECT qty FROM fills WHERE order_id = o.id\n) t\nORDER BY o.id, t.qty" + }, + { + "query": "SELECT o.id, t.cnt\nFROM orders o\nLEFT JOIN LATERAL (\n SELECT count(*) AS cnt FROM fills WHERE order_id = o.id\n) t\nORDER BY o.id" + }, + { + "query": "SELECT o.id, o.desk, t.qty\nFROM orders o\nJOIN LATERAL (\n SELECT qty\n FROM fills\n WHERE order_id = o.id\n ORDER BY qty DESC\n LIMIT 2\n) t\nORDER BY o.id, t.qty DESC" + }, + { + "query": "SELECT o.id, o.desk, t.total_qty\nFROM orders o\nJOIN LATERAL (\n SELECT sum(qty) AS total_qty\n FROM fills\n WHERE order_id = o.id\n) t\nORDER BY o.id" + }, + { + "query": "SELECT o.id, o.desk, t.total_qty\nFROM orders o\nJOIN LATERAL (\n SELECT sum(qty) AS total_qty\n FROM fills\n WHERE order_id = o.id\n AND qty > o.min_qty\n) t\nORDER BY o.id" + }, + { + "query": "SELECT o.id, t.qty, t.running_total\nFROM orders o\nJOIN LATERAL (\n SELECT qty,\n sum(qty) OVER (ORDER BY ts) AS running_total\n FROM fills\n WHERE order_id = o.id\n) t\nORDER BY o.id, t.qty" + }, + { + "query": "SELECT t.symbol, sub.ts, sub.volume\nFROM (\n SELECT * FROM fx_trades\n LATEST ON timestamp PARTITION BY symbol\n) t\nJOIN LATERAL (\n SELECT timestamp AS ts, sum(quantity) AS volume\n FROM fx_trades\n WHERE symbol = t.symbol\n AND timestamp IN '$now-6h..$now'\n SAMPLE BY 1h\n) sub\nORDER BY t.symbol, sub.ts" + }, + { + "query": "SELECT t.symbol, sub.side, sub.price, sub.quantity\nFROM (\n SELECT DISTINCT symbol FROM fx_trades\n WHERE timestamp IN '$now-1h..$now'\n) t\nJOIN LATERAL (\n SELECT side, price, quantity\n FROM fx_trades\n WHERE symbol = t.symbol\n LATEST ON timestamp PARTITION BY side\n) sub\nORDER BY t.symbol, sub.side" + }, + { + "query": "SELECT\n t.symbol,\n sub.timestamp,\n sub.side,\n sub.price,\n sub.quantity,\n sub.ecn,\n sub.bid_price,\n sub.ask_price\nFROM (\n SELECT * FROM fx_trades\n LATEST ON timestamp PARTITION BY symbol\n) t\nJOIN LATERAL (\n SELECT\n f.timestamp, f.side, f.price,\n f.quantity, f.ecn,\n c.bid_price, c.ask_price\n FROM fx_trades f\n ASOF JOIN core_price c\n ON (f.symbol = c.symbol AND f.ecn = c.ecn)\n WHERE f.symbol = t.symbol\n AND f.timestamp IN '$now-1m..$now'\n ORDER BY f.quantity DESC\n LIMIT 3\n) sub" + }, + { + "query": "SELECT o.id, t.qty, t.bucket\nFROM orders o\nJOIN LATERAL (\n SELECT qty, 'small' AS bucket FROM fills\n WHERE order_id = o.id AND qty < 30\n UNION ALL\n SELECT qty, 'large' AS bucket FROM fills\n WHERE order_id = o.id AND qty >= 30\n) t\nORDER BY o.id, t.qty" + }, + { + "query": "SELECT o.id, t.qty\nFROM orders o,\n LATERAL (SELECT qty FROM fills WHERE order_id = o.id) t\nORDER BY o.id, t.qty" + }, + { + "query": "CREATE TABLE master (\n mm_id INT,\n symbol STRING,\n ts TIMESTAMP\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE detail (\n mm_id INT,\n symbol STRING,\n qty DOUBLE,\n ts TIMESTAMP\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "SELECT m.mm_id, m.symbol, t.total\nFROM master m\nLEFT JOIN LATERAL (\n SELECT sum(qty) AS total\n FROM detail\n WHERE mm_id = m.mm_id\n AND symbol = m.symbol\n) t\nORDER BY m.mm_id" + }, + { + "query": "SELECT symbol, timestamp, price\nFROM fx_trades\nLATEST ON timestamp PARTITION BY symbol" + }, + { + "query": "(fx_trades WHERE timestamp IN '$today' LATEST ON timestamp PARTITION BY symbol)\nWHERE price > 3" + }, + { + "query": "CREATE TABLE orders (id LONG)" + }, + { + "query": "INSERT INTO orders VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10)" + }, + { + "query": "SELECT * FROM orders LIMIT 5" + }, + { + "query": "SELECT * FROM orders LIMIT -5" + }, + { + "query": "SELECT * FROM orders LIMIT 2, 5" + }, + { + "query": "SELECT * FROM orders LIMIT -5, -3" + }, + { + "query": "SELECT * FROM orders LIMIT 2, -1" + }, + { + "query": "SELECT * FROM orders LIMIT 5, 2" + }, + { + "query": "SELECT * FROM orders LIMIT -3, -5" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1m..$now'\nORDER BY symbol" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1m..$now'\nORDER BY symbol DESC" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1m..$now'\nORDER BY symbol, side DESC" + }, + { + "query": "trades PIVOT (\n avg(price)\n FOR symbol IN ('BTC-USDT', 'ETH-USDT')\n GROUP BY side\n)\nORDER BY side -- outside PIVOT parentheses\nLIMIT 10" + }, + { + "query": "SELECT symbol, avg(price)\nFROM trades\nWHERE timestamp IN '$today'\nGROUP BY symbol" + }, + { + "query": "SELECT\n avg(CASE WHEN symbol = 'BTC-USDT' THEN price END) AS \"BTC-USDT\",\n avg(CASE WHEN symbol = 'ETH-USDT' THEN price END) AS \"ETH-USDT\",\n avg(CASE WHEN symbol = 'SOL-USDT' THEN price END) AS \"SOL-USDT\",\n avg(CASE WHEN symbol = 'ADA-USDT' THEN price END) AS \"ADA-USDT\",\n avg(CASE WHEN symbol = 'AVAX-USDT' THEN price END) AS \"AVAX-USDT\"\nFROM trades\nWHERE timestamp IN '$today'" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price)\n FOR symbol IN (\n 'BTC-USDT', 'ETH-USDT', 'SOL-USDT',\n 'ADA-USDT', 'AVAX-USDT'\n )\n)" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price) AS avg_price,\n sum(price * amount) / 2 AS half_value\n FOR symbol IN ('BTC-USDT', 'ETH-USDT', 'SOL-USDT')\n)" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price)\n FOR symbol IN (\n 'BTC-USDT', 'ETH-USDT', 'SOL-USDT',\n 'ADA-USDT', 'AVAX-USDT'\n )\n side IN ('buy', 'sell')\n)" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price)\n FOR symbol IN (\n 'BTC-USDT', 'ETH-USDT', 'SOL-USDT',\n 'ADA-USDT', 'AVAX-USDT'\n )\n GROUP BY side\n) ORDER BY side" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price)\n FOR symbol IN (\n SELECT DISTINCT symbol FROM trades\n WHERE timestamp IN '$today'\n ORDER BY symbol\n )\n GROUP BY side\n)" + }, + { + "query": "WITH recent_trades AS (\n SELECT * FROM trades\n WHERE timestamp IN '$today'\n)\nSELECT * FROM recent_trades\nPIVOT (\n avg(price)\n FOR symbol IN (\n SELECT DISTINCT symbol FROM recent_trades\n ORDER BY symbol\n )\n GROUP BY side\n)" + }, + { + "query": "-- Columns: BTC-USDT, ETH-USDT\nSELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price)\n FOR symbol IN ('BTC-USDT', 'ETH-USDT')\n)" + }, + { + "query": "-- Columns: BTC-USDT_avg(price), BTC-USDT_sum(price), ...\nSELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price), sum(price)\n FOR symbol IN ('BTC-USDT', 'ETH-USDT')\n)" + }, + { + "query": "-- Columns: BTC-USDT_avg_price, BTC-USDT_total_price, ...\nSELECT * FROM trades\nWHERE timestamp IN '$today'\nPIVOT (\n avg(price) AS avg_price, sum(price) AS total_price\n FOR symbol IN ('BTC-USDT', 'ETH-USDT')\n)" + }, + { + "query": "SELECT ts, count()\nFROM trades\nSAMPLE BY h" + }, + { + "query": "SELECT ts, count()\nFROM trades\nSAMPLE BY 1h FROM '2026-01-01T00:00:00' TO '2026-01-02T00:00:00' FILL(NULL)\nALIGN TO CALENDAR TIME ZONE 'Europe/Berlin'" + }, + { + "query": "CREATE TABLE trades (\n ts TIMESTAMP,\n price DOUBLE\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (TABLES)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (TABLES, MATERIALIZED_VIEWS)" + }, + { + "query": "SHOW CREATE DATABASE EXCLUDE (MATERIALIZED_VIEWS)" + }, + { + "query": "SHOW CREATE MATERIALIZED VIEW bbo_1s" + }, + { + "query": "CREATE TABLE trades (\n\tsymbol SYMBOL CAPACITY 256 CACHE,\n\tside SYMBOL CAPACITY 256 CACHE,\n\tprice DOUBLE,\n\tamount DOUBLE,\n\ttimestamp TIMESTAMP\n) timestamp(timestamp) PARTITION BY DAY\nWITH maxUncommittedRows=500000, o3MaxLag=600000000us" + }, + { + "query": "CREATE TABLE trades (\n\tsymbol SYMBOL CAPACITY 256 CACHE INDEX TYPE POSTING INCLUDE (price, exchange, timestamp),\n\texchange SYMBOL CAPACITY 256 CACHE,\n\tprice DOUBLE,\n\tamount DOUBLE,\n\ttimestamp TIMESTAMP\n) timestamp(timestamp) PARTITION BY DAY\nWITH maxUncommittedRows=500000, o3MaxLag=600000000us" + }, + { + "query": "CREATE TABLE sensors (\n\tts TIMESTAMP,\n\ttemperature DOUBLE PARQUET(rle_dictionary, zstd(3)),\n\thumidity FLOAT PARQUET(rle_dictionary),\n\tdevice_id VARCHAR PARQUET(default, lz4_raw),\n\tstatus INT\n) timestamp(ts) PARTITION BY DAY BYPASS WAL" + }, + { + "query": "CREATE TABLE trades (\n\tsymbol SYMBOL CAPACITY 256 CACHE,\n\tside SYMBOL CAPACITY 256 CACHE,\n\tprice DOUBLE,\n\tamount DOUBLE,\n\ttimestamp TIMESTAMP\n) timestamp(timestamp) PARTITION BY DAY\nWITH maxUncommittedRows=500000, o3MaxLag=600000000us\nOWNED BY 'admin'" + }, + { + "query": "-- This query will return all parameters where the property_path is not 'cairo.root' or 'cairo.snapshot.instance.id', ordered by the first column\n(SHOW PARAMETERS) WHERE property_path NOT IN ('cairo.root', 'cairo.snapshot.instance.id') ORDER BY 1" + }, + { + "query": "SWITCH COLD STORAGE ROLE TO REFRESHER TIMEOUT 30000" + }, + { + "query": "SWITCH ROLE TO REPLICA TIMEOUT 60000" + }, + { + "query": "SELECT * FROM (SELECT symbol FROM trades WHERE symbol = 'BTC-USDT' LIMIT 1)\nUNION ALL\nSELECT * FROM (SELECT symbol FROM trades WHERE symbol = 'ETH-USDT' LIMIT 1)" + }, + { + "query": "binance_symbols UNION coinbase_symbols" + }, + { + "query": "binance_symbols\nUNION\ncoinbase_symbols WHERE base = 'NONEXISTENT'" + }, + { + "query": "binance_symbols UNION ALL coinbase_symbols" + }, + { + "query": "binance_symbols EXCEPT coinbase_symbols" + }, + { + "query": "binance_symbols EXCEPT ALL coinbase_symbols" + }, + { + "query": "binance_symbols INTERSECT coinbase_symbols" + }, + { + "query": "SELECT t.symbol, u.vol\nFROM market_data t, UNNEST(t.asks[2]) u(vol)\nWHERE t.timestamp IN '$now-1m..$now'\n AND t.symbol = 'EURUSD'" + }, + { + "query": "SELECT t.symbol, u.vol\nFROM market_data t\nCROSS JOIN UNNEST(t.asks[2]) u(vol)\nWHERE t.timestamp IN '$now-1m..$now'\n AND t.symbol = 'EURUSD'" + }, + { + "query": "SELECT u.val, u.pos\nFROM UNNEST(ARRAY[10.0, 20.0, 30.0]) WITH ORDINALITY u(val, pos)" + }, + { + "query": "SELECT u.a, u.b\nFROM UNNEST(ARRAY[1.0, 2.0, 3.0], ARRAY[10.0, 20.0]) u(a, b)" + }, + { + "query": "SELECT value\nFROM UNNEST(ARRAY[ARRAY[1.0, 2.0], ARRAY[3.0, 4.0]])" + }, + { + "query": "SELECT u.val\nFROM UNNEST(ARRAY[ARRAY[1.0, 2.0], ARRAY[3.0, 4.0]]) t(arr),\n UNNEST(t.arr) u(val)" + }, + { + "query": "SELECT u.price FROM UNNEST(ARRAY[1.5, 2.5]) u(price)" + }, + { + "query": "SELECT u.trade_id, u.price, u.size, u.side, u.time\nFROM UNNEST(\n '[{\"trade_id\":994619709,\"side\":\"sell\",\"size\":\"0.00000100\",\"price\":\"69839.36000000\",\"time\":\"2026-04-06T10:32:55.517183Z\"},\n {\"trade_id\":994619708,\"side\":\"buy\",\"size\":\"0.00000006\",\"price\":\"69839.35000000\",\"time\":\"2026-04-06T10:32:55.418434Z\"},\n {\"trade_id\":994619707,\"side\":\"buy\",\"size\":\"0.00000006\",\"price\":\"69839.35000000\",\"time\":\"2026-04-06T10:32:55.024765Z\"}]'::VARCHAR\n COLUMNS(trade_id LONG, price DOUBLE, size DOUBLE, side VARCHAR, time TIMESTAMP)\n) u" + }, + { + "query": "SELECT u.val\nFROM UNNEST('[1.5, 2.5, 3.5]'::VARCHAR COLUMNS(val DOUBLE)) u" + }, + { + "query": "SELECT u.val, u.pos\nFROM UNNEST(\n '[10, 20, 30]'::VARCHAR COLUMNS(val LONG)\n) WITH ORDINALITY u(val, pos)" + }, + { + "query": "SELECT u.ts, u.val\nFROM UNNEST(\n '[{\"ts\":\"2024-01-15T10:30:00.000000Z\",\"val\":1.5},\n {\"ts\":\"2024-06-20T14:00:00.000000Z\",\"val\":2.5}]'::VARCHAR\n COLUMNS(ts TIMESTAMP, val DOUBLE)\n) u" + }, + { + "query": "SELECT u.cost\nFROM UNNEST(\n '[{\"price\":1.5},{\"price\":2.5}]'::VARCHAR\n COLUMNS(price DOUBLE)\n) u(cost)" + }, + { + "query": "SELECT u.price\nFROM events e, UNNEST(\n json_extract(e.payload, '$.items')::VARCHAR\n COLUMNS(price DOUBLE)\n) u" + }, + { + "query": "SELECT u.a, u.b\nFROM UNNEST(\n '[{\"a\":1},{\"a\":2,\"b\":99},{\"a\":null}]'::VARCHAR\n COLUMNS(a INT, b INT)\n) u" + }, + { + "query": "SELECT t.symbol, u.vol\nFROM market_data t, UNNEST(t.asks[2]) u(vol)\nWHERE t.timestamp IN '$now-1m..$now'\n AND t.symbol = 'EURUSD'\n AND u.vol > 100.0\nORDER BY t.timestamp" + }, + { + "query": "SELECT t.symbol, sum(u.vol) AS total_ask_vol\nFROM market_data t, UNNEST(t.asks[2]) u(vol)\nWHERE t.timestamp IN '$now-1m..$now'\nGROUP BY t.symbol" + }, + { + "query": "WITH expanded AS (\n SELECT m.symbol, m.timestamp, u.vol, u.level\n FROM market_data m, UNNEST(m.asks[2]) WITH ORDINALITY u(vol, level)\n WHERE m.timestamp IN '$now-1m..$now'\n AND m.symbol = 'EURUSD'\n)\nSELECT symbol, level, avg(vol) AS avg_vol\nFROM expanded\nGROUP BY symbol, level\nORDER BY symbol, level" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now'\n AND side = 'buy'\n AND (symbol = 'BTC-USDT' OR price > 100000)\nLIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now'\n AND symbol = 'BTC-USDT'\nLIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now'\n AND symbol != 'BTC-USDT'\nLIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now'\n AND symbol ~ '^BTC'\nLIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now'\n AND symbol !~ '^BTC'\nLIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now'\n AND symbol IN ('BTC-USDT', 'ETH-USDT')\nLIMIT -20" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now'\n AND symbol NOT IN ('BTC-USDT', 'ETH-USDT')\nLIMIT -20" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now' AND amount >= 1.0\nLIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now' AND amount = 1.0\nLIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1h..$now' AND amount != 1.0\nLIMIT -3" + }, + { + "query": "SELECT * FROM instruments WHERE is_tradable" + }, + { + "query": "SELECT * FROM instruments WHERE NOT is_tradable" + }, + { + "query": "SELECT * FROM trades WHERE timestamp = '2026-04-02T12:00:00.190Z'" + }, + { + "query": "SELECT * FROM trades WHERE timestamp = '2026-04-02T12:00:00.190000Z'" + }, + { + "query": "SELECT * FROM trades WHERE timestamp IN '2026' LIMIT -3" + }, + { + "query": "SELECT * FROM trades WHERE timestamp IN '2026-04-02T12:15' LIMIT -3" + }, + { + "query": "SELECT * FROM trades WHERE timestamp IN '2026;1M' LIMIT -3" + }, + { + "query": "SELECT * FROM trades WHERE timestamp IN '2026-04;-3d' LIMIT -3" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '2025-01-01;1d;1y;2' AND symbol = 'SOL-ETH'" + }, + { + "query": "-- IN extension for time-intervals\n\nSELECT * FROM trades WHERE timestamp in '2026'" + }, + { + "query": "-- whole year\nSELECT * FROM trades WHERE timestamp in '2025-12'" + }, + { + "query": "-- whole month\nSELECT * FROM trades WHERE timestamp in '2025-12-20'" + }, + { + "query": "-- whole day\n\n-- The whole day, extending 15s into the next day\nSELECT * FROM trades WHERE timestamp in '2025-12-20;15s'" + }, + { + "query": "-- For the past 7 days, 2 seconds before and after midnight\nSELECT * from trades WHERE timestamp in '2025-09-20T23:59:58;4s;-1d;7'" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN ('2026-04-01', '2026-04-01T12:00:00.017999Z', '2026-04-02')" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp = '2026-04-01'\n OR timestamp = '2026-04-01T12:00:00.017999Z'\n OR timestamp = '2026-04-02'" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp BETWEEN '2026-04-01T00:00:23.000000Z'\n AND '2026-04-01T00:00:23.500000Z'" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp BETWEEN to_str(now(), 'yyyy-MM-dd')\nAND dateadd('y', -1, to_str(now(), 'yyyy-MM-dd'))" + }, + { + "query": "SELECT * FROM trades\nWHERE timestamp IN '$now-1y..$now'" + }, + { + "query": "SELECT *\nFROM trades\nWHERE timestamp BETWEEN '2026-04-01' AND '2026-04-03'\nLIMIT -1" + }, + { + "query": "SELECT *\nFROM trades\nWHERE timestamp BETWEEN '2026-04-01' AND '2026-04-03T00:00:00.99'\nLIMIT -1" + }, + { + "query": "SELECT\n t.symbol,\n t.price,\n t.timestamp,\n avg(c.bid_price) AS avg_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 5 seconds PRECEDING AND 5 seconds FOLLOWING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nORDER BY t.timestamp\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS avg_bid,\n count() AS num_quotes\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 5 seconds PRECEDING AND 5 seconds FOLLOWING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS avg_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol) AND c.ecn = t.ecn\n RANGE BETWEEN 2 seconds PRECEDING AND 2 seconds FOLLOWING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nORDER BY t.timestamp\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS pre_trade_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 2 seconds PRECEDING AND 1 second PRECEDING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS post_trade_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 1 second FOLLOWING AND 5 seconds FOLLOWING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n count() AS total_quotes\nFROM fx_trades t\nWINDOW JOIN core_price c\n RANGE BETWEEN 1 second PRECEDING AND 1 second FOLLOWING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n t.price,\n avg(m.best_bid) AS consolidated_bid_1s,\n avg(c.bid_price) AS ecn_bid_5s\nFROM fx_trades t\nWINDOW JOIN market_data m\n ON (t.symbol = m.symbol)\n RANGE BETWEEN 1 second PRECEDING AND 1 second FOLLOWING\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 5 seconds PRECEDING AND 5 seconds FOLLOWING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n t.lookback,\n t.lookahead,\n avg(c.bid_price) AS avg_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN t.lookback seconds PRECEDING AND t.lookahead seconds FOLLOWING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS avg_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN t.lookback seconds PRECEDING AND 5 seconds FOLLOWING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS avg_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 2 * t.lookback seconds PRECEDING AND 10 seconds FOLLOWING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS avg_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 1 second PRECEDING AND 1 second FOLLOWING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'\nLIMIT -20" + }, + { + "query": "SELECT\n t.symbol,\n t.timestamp,\n avg(c.bid_price) AS avg_bid\nFROM fx_trades t\nWINDOW JOIN core_price c\n ON (t.symbol = c.symbol)\n RANGE BETWEEN 1 second PRECEDING AND 1 second FOLLOWING\n EXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.side = 'buy'\n AND t.timestamp IN '$now-1h..$now'\nORDER BY t.timestamp\nLIMIT -20" + }, + { + "query": "EXPLAIN SELECT t.symbol, avg(c.bid_price)\nFROM fx_trades t\nWINDOW JOIN core_price c ON (t.symbol = c.symbol)\nRANGE BETWEEN 1 second PRECEDING AND 1 second FOLLOWING\nEXCLUDE PREVAILING\nWHERE t.symbol = 'EURUSD'\n AND t.timestamp IN '$now-1h..$now'" + }, + { + "query": "WITH trades_with_future_bid AS (\n SELECT\n t.symbol,\n t.price,\n first(c.bid_price) AS future_bid\n FROM fx_trades t\n WINDOW JOIN core_price c ON (t.symbol = c.symbol)\n RANGE BETWEEN 10 milliseconds FOLLOWING AND 10 milliseconds FOLLOWING\n INCLUDE PREVAILING\n WHERE t.timestamp IN '$now-1h..$now'\n)\nSELECT\n symbol,\n count(*) AS trade_count,\n avg(future_bid - price) AS avg_slippage\nFROM trades_with_future_bid\nGROUP BY symbol" + }, + { + "query": "WITH recent_eurusd AS (\n SELECT timestamp, price FROM fx_trades\n WHERE symbol = 'EURUSD'\n LIMIT -10\n)\nSELECT * FROM recent_eurusd" + }, + { + "query": "WITH recent_eurusd AS (\n SELECT timestamp, price FROM fx_trades\n WHERE symbol = 'EURUSD'\n LIMIT -10\n),\nlast_5 AS (SELECT * FROM recent_eurusd LIMIT -5)\nSELECT * FROM last_5" + }, + { + "query": "WITH eurusd_today AS (\n SELECT timestamp, price,\n avg(price) OVER () AS avg_price\n FROM fx_trades\n WHERE symbol = 'EURUSD'\n AND timestamp IN '$today'\n)\nSELECT timestamp, price, avg_price\nFROM eurusd_today\nWHERE price > avg_price" + }, + { + "query": "-- Default bitmap index — low overhead, good for most cases\nCREATE TABLE trades (\n ts TIMESTAMP,\n symbol SYMBOL INDEX,\n price DOUBLE\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "-- Posting index with covering columns — best for read-heavy, selective queries\nCREATE TABLE trades (\n ts TIMESTAMP,\n symbol SYMBOL INDEX TYPE POSTING INCLUDE (price),\n price DOUBLE,\n raw_data VARCHAR -- not in INCLUDE, read from column files\n) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "-- User can see all columns except trader_id\nGRANT SELECT ON trades(* EXCLUDE (trader_id)) TO analyst" + }, + { + "query": "ALTER USER tenant_a SET MEMORY LIMIT 1G" + }, + { + "query": "CREATE SERVICE ACCOUNT failover_bot WITH PASSWORD 'pwd'" + }, + { + "query": "GRANT HTTP TO failover_bot" + }, + { + "query": "-- POST /lifecycle/switch on port 9003\nGRANT SWITCH ROLE TO failover_bot" + }, + { + "query": "-- Now operating with trading_app's exact permissions\n-- Test what works and what doesn't...\nEXIT SERVICE ACCOUNT" + }, + { + "query": "-- Database-level: applies to all tables\nGRANT SELECT ON ALL TABLES TO user" + }, + { + "query": "-- Table-level: applies to specific tables\nGRANT SELECT ON trades, prices TO user" + }, + { + "query": "-- Column-level: applies to specific columns\nGRANT SELECT ON trades(ts, symbol, price) TO user" + }, + { + "query": "-- Database level\nREVOKE SELECT ON secret_table FROM user" + }, + { + "query": "GRANT SELECT ON trades TO user" + }, + { + "query": "-- Table level\nREVOKE SELECT ON trades(ssn) FROM user" + }, + { + "query": "GRANT SELECT ON trdaes TO user WITH VERIFICATION" + }, + { + "query": "ALTER SERVICE ACCOUNT ingest_app SET MEMORY LIMIT 1G" + }, + { + "query": "DECLARE\n @prices := asks[1],\n @volumes := asks[2],\n @best_price := @prices[1],\n @multiplier := 1.01,\n @target_price := @multiplier * @best_price,\n @relevant_volume_levels := @volumes[1:insertion_point(@prices, @target_price)]\nSELECT timestamp, array_sum(@relevant_volume_levels) total_volume\nFROM market_data WHERE symbol='EURUSD'\nLIMIT -10" + }, + { + "query": "DECLARE\n @prices := asks[1],\n @volumes := asks[2],\n @best_price := @prices[1],\n @price_delta := 0.1,\n @target_price := @best_price + @price_delta,\n @relevant_volumes := @volumes[1:insertion_point(@prices, @target_price)]\nSELECT timestamp, array_sum(@relevant_volumes) volume\nFROM market_data WHERE symbol='EURUSD'\nLIMIT -10" + }, + { + "query": "DECLARE\n @volumes := asks[2],\n @dropoff_ratio := 3.0\nSELECT * FROM (\n SELECT\n timestamp,\n array_avg(@volumes[1:3]) top,\n array_avg(@volumes[3:6]) deep\n FROM market_data\n WHERE timestamp > dateadd('m',-30,now()) )\nWHERE top > @dropoff_ratio * deep" + }, + { + "query": "DECLARE\n @top_bid_volume := bids[2, 1],\n @top_ask_volume := asks[2, 1],\n @drop_ratio := 1.5\nSELECT * FROM (\n SELECT\n timestamp,\n lag(@top_bid_volume) OVER () prev_bid_vol,\n @top_bid_volume curr_bid_vol,\n lag(@top_ask_volume) OVER () prev_ask_vol,\n @top_ask_volume curr_ask_vol\n FROM market_data WHERE timestamp > dateadd('h',-1,now()) AND symbol='EURUSD' )\nWHERE prev_bid_vol > curr_bid_vol * @drop_ratio OR prev_ask_vol > curr_ask_vol * @drop_ratio\nLIMIT 10" + }, + { + "query": "REFRESH MATERIALIZED VIEW mv STATS" + }, + { + "query": "ALTER TABLE t REBASE WAL INTO 'd1'" + }, + { + "query": "ALTER MATERIALIZED VIEW mv REBASE WAL" + }, + { + "query": "SHOW CREATE DATABASE EXCLUDE (users, groups)" + }, + { + "query": "CREATE TABLE t (a INT) TIMESTAMP(a) PARTITION BY DAY FORMAT PARQUET WAL" + }, + { + "query": "ALTER TABLE t SET FORMAT NATIVE" + }, + { + "query": "SELECT switch(x, 1, 'a', 'b') FROM t" + }, + { + "query": "GRANT SELECT ON tab(*) TO alice" + }, + { + "query": "GRANT SELECT ON tab(* EXCLUDE(a, b)) TO alice" + }, + { + "query": "GRANT SET TABLE FORMAT ON t TO alice" + }, + { + "query": "GRANT CONVERT PARTITION TO PARQUET ON tab TO alice" + }, + { + "query": "CREATE MATERIALIZED VIEW mv AS (SELECT * FROM base) EXPIRE ROWS KEEP LATEST PARTITION BY sym" + }, + { + "query": "ALTER MATERIALIZED VIEW mv DROP EXPIRE" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 5s IN MEMORY 1h START FROM BEGINNING AS (SELECT * FROM t)" + }, + { + "query": "ALTER LIVE VIEW lv RESUME WAL FROM TXN 1" + }, + { + "query": "SELECT avg(x) OVER w FROM t WINDOW w AS (ORDER BY ts ANCHOR DAILY '09:30')" + }, + { + "query": "SELECT avg(x) OVER (ORDER BY ts ANCHOR EXPRESSION timestamp_floor('1d', ts)) FROM t" + }, + { + "query": "COPY PERMISSIONS FROM src TO dst" + }, + { + "query": "CREATE TABLE t (s SYMBOL INDEX TYPE POSTING DELTA INCLUDE (a, b), a INT, b INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE t (s SYMBOL, ts TIMESTAMP), INDEX(s TYPE POSTING) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "ALTER TABLE t ALTER COLUMN sym ADD INDEX TYPE POSTING INCLUDE (p)" + }, + { + "query": "SELECT * FROM t WINDOW JOIN p ON t.sym = p.sym RANGE BETWEEN wndBound SECONDS PRECEDING AND wndBound SECONDS FOLLOWING" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY s" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY m" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY h" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY d" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY w" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY y" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY T" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY U" + }, + { + "query": "SELECT ts, avg(x) FROM t SAMPLE BY n" + }, + { + "query": "select ts, avg(x) from fromto sample by w from '2017-12-20' to '2018-01-31' fill(null) align to calendar" + }, + { + "query": "alter table t rebase wal" + }, + { + "query": "alter table base_price rebase wal" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (SCHEMA)" + }, + { + "query": "SHOW CREATE DATABASE EXCLUDE (VIEWS)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (USERS)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (GROUPS)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (SERVICE_ACCOUNTS)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (PERMISSIONS)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (ACL)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (PERMISSIONS, TABLES)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (ALL)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (VIEWS)" + }, + { + "query": "SHOW CREATE DATABASE INCLUDE (MATERIALIZED_VIEWS)" + }, + { + "query": "SHOW CREATE DATABASE EXCLUDE ALL" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY FORMAT PARQUET WAL" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL FORMAT PARQUET" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY FORMAT NATIVE WAL" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP, n LONG) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts) FORMAT PARQUET" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP, n LONG) TIMESTAMP(ts) PARTITION BY DAY WAL FORMAT PARQUET DEDUP UPSERT KEYS(ts)" + }, + { + "query": "CREATE TABLE tango (val INT, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL FORMAT PARQUET" + }, + { + "query": "CREATE TABLE tango (val INT PARQUET(BLOOM_FILTER), ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY WAL FORMAT PARQUET" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP, sym SYMBOL, n LONG) TIMESTAMP(ts) PARTITION BY DAY FORMAT PARQUET WAL" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP, s STRING, v VARCHAR, b BINARY) TIMESTAMP(ts) PARTITION BY DAY FORMAT PARQUET WAL" + }, + { + "query": "CREATE TABLE tango (ts TIMESTAMP PARQUET(plain), v LONG) TIMESTAMP(ts) PARTITION BY DAY FORMAT PARQUET WAL" + }, + { + "query": "CREATE TABLE 'tango' (ts TIMESTAMP) timestamp(ts) PARTITION BY DAY FORMAT PARQUET" + }, + { + "query": "ALTER TABLE tango SET FORMAT NATIVE" + }, + { + "query": "ALTER TABLE tango SET FORMAT PARQUET" + }, + { + "query": "ALTER TABLE no_partition SET FORMAT NATIVE" + }, + { + "query": "SWITCH ROLE TO REPLICA" + }, + { + "query": "SWITCH ROLE TO REPLICA TIMEOUT 10000" + }, + { + "query": "SWITCH ROLE TO PRIMARY TIMEOUT 10000" + }, + { + "query": "SWITCH ROLE TO REPLICA TIMEOUT 1" + }, + { + "query": "SWITCH ROLE TO REPLICA TIMEOUT 500" + }, + { + "query": "SWITCH ROLE TO REPLICA TIMEOUT 8000" + }, + { + "query": "SWITCH ROLE TO REPLICA TIMEOUT 42000" + }, + { + "query": "SWITCH ROLE TO PRIMARY TIMEOUT 1000" + }, + { + "query": "SWITCH ROLE TO PRIMARY TIMEOUT 2000" + }, + { + "query": "SWITCH ROLE TO PRIMARY TIMEOUT 42000" + }, + { + "query": "grant select on t1(*) to ddd" + }, + { + "query": "grant select on t1(* exclude(b, c)) to ddd" + }, + { + "query": "grant select on t1(* exclude(b)) to ddd with grant option" + }, + { + "query": "grant select on t1(* exclude(c)) to ddd" + }, + { + "query": "grant all on t1(*) to ddd" + }, + { + "query": "grant select, update on t1(*) to ddd" + }, + { + "query": "grant select on t1(a, b), t2(*), t3(* exclude(q)) to ddd" + }, + { + "query": "grant select on t1(* exclude(a)), t2(* exclude(z)) to ddd" + }, + { + "query": "grant select on mv1(* exclude(avg_x)) to ddd" + }, + { + "query": "grant select on mv1(*) to ddd" + }, + { + "query": "grant select on v1(*) to ddd" + }, + { + "query": "grant select on v1(* exclude(a)) to ddd" + }, + { + "query": "GRANT SELECT ON tgs1(* EXCLUDE(b)) TO ugs1" + }, + { + "query": "GRANT SELECT ON tgs1(*) TO ugs1" + }, + { + "query": "revoke select on t1(*) from ddd" + }, + { + "query": "create materialized view mv as (select * from base) EXPIRE ROWS WHEN v < 2.0" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when v < 2.0 cleanup every 30m" + }, + { + "query": "create materialized view mv as (select * from base) EXPIRE ROWS WHEN v < 2.0 cleanup every 15m" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when v < 2.0 cleanup every 90m" + }, + { + "query": "create materialized view mv as (select * from base) EXPIRE ROWS WHEN (v < 2.0) CLEANUP EVERY 30m" + }, + { + "query": "create materialized view mv as (select * from base) EXPIRE ROWS WHEN cleanup > 5" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when v < 0" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when v < 0.0" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when v > 100" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when ts < '2024-01-02T00:00:00.000000Z'" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when ts < '2024-01-02T00:00:00.000000Z' cleanup every 1h" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when ts <= '2024-01-02T00:00:00.000000Z'" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when ts < dateadd('d', -1, now())" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when ts < now()" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when ts > now()" + }, + { + "query": "create materialized view mv as (select * from base) expire rows when ts < cast(null as timestamp)" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM NOW AS SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM base" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 100ms START FROM NOW AS SELECT ts, sym, x, count(*) OVER (PARTITION BY g ORDER BY ts ROWS BETWEEN 1_000_000 PRECEDING AND CURRENT ROW) AS rn FROM base" + }, + { + "query": "CREATE LIVE VIEW IF NOT EXISTS lv FLUSH EVERY 1s START FROM NOW AS SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM base" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1_200s IN MEMORY 1_800s START FROM NOW AS SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM base" + }, + { + "query": "CREATE LIVE VIEW lv2 FLUSH EVERY 1_500ms START FROM NOW AS SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM base" + }, + { + "query": "CREATE LIVE VIEW lv0 FLUSH EVERY 1s START FROM BEGINNING AS SELECT ts, x, count(*) OVER (PARTITION BY 0 ORDER BY ts ROWS BETWEEN 1000000 PRECEDING AND CURRENT ROW) AS rn FROM base WHERE x > 0" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM '2026-04-01T00:00:15.000000Z' AS SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM base" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM '2026-04-01T00:00:15.000000123Z' AS SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM base_ns" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM NOW AS (SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM tab)" + }, + { + "query": "CREATE LIVE VIEW public.lv FLUSH EVERY 1s START FROM NOW AS SELECT ts, x, count(*) OVER (PARTITION BY x ORDER BY ts ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS rn FROM base" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM NOW AS SELECT ts, sym, sum(x) OVER w AS s FROM base WINDOW w AS (PARTITION BY sym ORDER BY ts ANCHOR EXPRESSION timestamp_floor('1d', ts))" + }, + { + "query": "CREATE LIVE VIEW lv_daily FLUSH EVERY 1s START FROM NOW AS SELECT ts, sym, x, row_number() OVER w AS rn FROM base WINDOW w AS (PARTITION BY sym ORDER BY ts ANCHOR DAILY '00:00')" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM NOW AS SELECT ts, sym, sum(x) OVER w AS s FROM base WINDOW w AS (PARTITION BY sym ORDER BY ts ANCHOR DAILY '09:30')" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM NOW AS SELECT ts, sym, sum(x) OVER w AS s FROM base WINDOW w AS (PARTITION BY sym ORDER BY ts ANCHOR DAILY '00:00' 'Europe/London')" + }, + { + "query": "CREATE LIVE VIEW lv FLUSH EVERY 1s START FROM NOW AS SELECT ts, sym, sum(x) OVER w AS s FROM base WINDOW w AS (PARTITION BY sym ORDER BY ts ANCHOR EXPRESSION timestamp_floor('4h', ts))" + }, + { + "query": "COPY PERMISSIONS FROM 'my table' TO 'other table'" + }, + { + "query": "GRANT CONVERT PARTITION TO PARQUET ON tab TO testUser" + }, + { + "query": "GRANT CONVERT PARTITION TO NATIVE ON tab TO testUser" + }, + { + "query": "REVOKE CONVERT PARTITION TO PARQUET ON tab FROM testUser" + }, + { + "query": "ALTER TABLE tab CONVERT PARTITION TO PARQUET WHERE ts > 0" + }, + { + "query": "ALTER TABLE tab CONVERT PARTITION TO NATIVE WHERE ts > 0" + }, + { + "query": "alter table test convert partition to parquet where ts > 0" + }, + { + "query": "alter table test convert partition to native where ts > 0" + }, + { + "query": "create table x (t TIMESTAMP, x SYMBOL index type posting) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, x SYMBOL index type bitmap) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, x SYMBOL index type bitmap capacity 64) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, x SYMBOL index type posting delta) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, x SYMBOL index type posting ef) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, p DOUBLE, x SYMBOL index include (p)) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, p DOUBLE, x SYMBOL index type posting include (p)) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, p DOUBLE, x SYMBOL index type posting ef include (p)) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, x SYMBOL), index(x type posting delta) timestamp(t)" + }, + { + "query": "create table x (t TIMESTAMP, x SYMBOL), index(x type posting ef) timestamp(t)" + }, + { + "query": "CREATE TABLE tab (s SYMBOL INDEX TYPE POSTING, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE tab (s SYMBOL INDEX TYPE POSTING DELTA, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE tab (s SYMBOL INDEX TYPE POSTING EF, ts TIMESTAMP) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "create table tab (s symbol index type posting, ts timestamp) timestamp(ts)" + }, + { + "query": "create table tab (s symbol index type bitmap, ts timestamp) timestamp(ts)" + }, + { + "query": "CREATE TABLE tab (ts TIMESTAMP, s SYMBOL INDEX TYPE POSTING INCLUDE (v), v DOUBLE) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "CREATE TABLE t (ts TIMESTAMP, s SYMBOL INDEX TYPE POSTING) TIMESTAMP(ts) PARTITION BY DAY BYPASS WAL" + }, + { + "query": "CREATE TABLE t (ts TIMESTAMP, s SYMBOL INDEX TYPE POSTING) TIMESTAMP(ts) PARTITION BY DAY WAL" + }, + { + "query": "CREATE TABLE tab (s SYMBOL, ts TIMESTAMP), INDEX(s TYPE POSTING) TIMESTAMP(ts) PARTITION BY DAY" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN t.price::long minutes PRECEDING AND 1 minute FOLLOWING INCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN t.price::long minutes PRECEDING AND 1 minute FOLLOWING EXCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN t.price::long PRECEDING AND 60_000_000 FOLLOWING INCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN t.price::long PRECEDING AND 60_000_000 FOLLOWING EXCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN 1 minute PRECEDING AND t.price::long seconds FOLLOWING INCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN 1 minute PRECEDING AND t.price::long seconds FOLLOWING EXCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN price::long seconds PRECEDING AND 1 minute FOLLOWING INCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_price FROM trades t WINDOW JOIN prices p RANGE BETWEEN price::long seconds PRECEDING AND 1 minute FOLLOWING EXCLUDE PREVAILING ORDER BY t.ts;" + }, + { + "query": "SELECT m.ts, m.lo_bound, m.hi_bound, sum(s.val) AS agg FROM master m WINDOW JOIN slave s RANGE BETWEEN lo_bound minutes PRECEDING AND hi_bound minutes FOLLOWING EXCLUDE PREVAILING ORDER BY m.ts" + }, + { + "query": "SELECT m.ts, m.lo_bound, m.hi_bound, sum(s.val) AS agg FROM master m WINDOW JOIN slave s RANGE BETWEEN lo_bound minutes PRECEDING AND hi_bound minutes FOLLOWING INCLUDE PREVAILING ORDER BY m.ts" + }, + { + "query": "SELECT m.ts, sum(s.val) AS agg FROM master m WINDOW JOIN slave s RANGE BETWEEN bound minutes PRECEDING AND 0 seconds FOLLOWING EXCLUDE PREVAILING ORDER BY m.ts" + }, + { + "query": "SELECT m.ts, sum(s.val) AS agg FROM (SELECT * FROM master LIMIT 4) m WINDOW JOIN slave s RANGE BETWEEN bound minutes PRECEDING AND 0 seconds FOLLOWING EXCLUDE PREVAILING ORDER BY m.ts" + }, + { + "query": "SELECT avg(b.bid) AS avg_bid, avg(a.ask) AS avg_ask FROM trades AS t HORIZON JOIN bids AS b ON (t.sym = b.sym) HORIZON JOIN asks AS a ON (t.sym = a.sym) LIST (-1s, 0s, 1s) AS h" + }, + { + "query": "SELECT avg(b.bid) AS avg_bid, avg(a.ask) AS avg_ask FROM trades AS t HORIZON JOIN bids AS b HORIZON JOIN asks AS a LIST (0s) AS h" + }, + { + "query": "SELECT avg(b.bid) AS avg_bid, avg(a.ask) AS avg_ask FROM trades AS t HORIZON JOIN bids AS b ON (t.sym = b.sym) HORIZON JOIN asks AS a ON (t.sym = a.sym) RANGE FROM -1s TO 1s STEP 1s AS h" } -] \ No newline at end of file +] diff --git a/tests/formatter/bundle.test.ts b/tests/formatter/bundle.test.ts new file mode 100644 index 0000000..05f6ae3 --- /dev/null +++ b/tests/formatter/bundle.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest" +import * as fs from "fs" +import * as path from "path" + +const distFormatter = path.join(__dirname, "..", "..", "dist", "formatter") +const bundlePath = path.join(distFormatter, "index.js") + +const built = fs.existsSync(bundlePath) + +describe("formatter bundle", () => { + it.skipIf(!built)("carries the parser but not the AST layer", () => { + const bundle = fs.readFileSync(bundlePath, "utf-8") + // The capitalize option needs the grammar to tell syntax from names. + expect(bundle).toContain("performSelfAnalysis") + // Nothing needs the CST-to-AST visitor or its serializer. + expect(bundle).not.toContain("toSql") + }) +}) diff --git a/tests/formatter/capitalize.test.ts b/tests/formatter/capitalize.test.ts new file mode 100644 index 0000000..d356705 --- /dev/null +++ b/tests/formatter/capitalize.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest" +import * as fs from "fs" +import * as path from "path" +import { format } from "../../src/formatter/index" +import { parse, parseToAst, toSql } from "../../src/index" + +const docsQueries: string[] = ( + JSON.parse( + fs.readFileSync( + path.join(__dirname, "..", "fixtures", "docs-queries.json"), + "utf-8", + ), + ) as Array<{ query: string }> +).map((entry) => entry.query) + +const wide = { maxWidth: 80, capitalize: true } as const + +describe("format with capitalize", () => { + it.each([ + [ + "raises types and options but not column names", + "create table tab (s symbol index type bitmap, timestamp timestamp) timestamp(timestamp) partition by day wal with maxUncommittedRows = 10", + [ + "CREATE TABLE tab (", + " s SYMBOL INDEX TYPE BITMAP,", + " timestamp TIMESTAMP", + ") TIMESTAMP(timestamp) PARTITION BY DAY WAL", + "WITH maxUncommittedRows = 10", + ].join("\n"), + ], + [ + "leaves columns alone where a keyword names one", + "select symbol, avg(price) from trades where ts in today() latest on ts partition by symbol", + [ + "SELECT symbol, avg(price)", + "FROM trades", + "WHERE ts IN today()", + "LATEST ON ts PARTITION BY symbol", + ].join("\n"), + ], + [ + "raises the modifiers of a SAMPLE BY", + "select ts, avg(px) from trades sample by 1h align to calendar", + [ + "SELECT ts, avg(px)", + "FROM trades", + "SAMPLE BY 1h ALIGN TO CALENDAR", + ].join("\n"), + ], + [ + "raises statements the formatter has no layout for", + "rename table trades_new to trades", + "RENAME TABLE trades_new TO trades", + ], + [ + "raises a permission list", + "grant select on all tables to dashboard", + "GRANT SELECT ON ALL TABLES TO dashboard", + ], + [ + "leaves a quoted name alone", + 'select "select", status from t', + ['SELECT "select", status', "FROM t"].join("\n"), + ], + [ + "leaves the case of SQL the parser cannot read", + "select * from t where", + ["select *", "from t", "where"].join("\n"), + ], + ])("%s", (_name, input, expected) => { + expect(format(input, wide)).toBe(expected) + }) + + it("lays out exactly like format without it", () => { + // Given + const sql = "select a, b from t where x = 1 order by a" + + // When / Then + expect(format(sql, wide).toLowerCase()).toBe( + format(sql, { maxWidth: 80 }).toLowerCase(), + ) + }) +}) + +type CstNode = { name?: string; children?: Record } +type CstToken = { image?: string; tokenType?: unknown } + +/** + * The words the grammar read as a table, view or column name. Written here + * rather than imported so the check does not share code with what it checks. + */ +const namesIn = (sql: string): string[] => { + const names: string[] = [] + const walk = (node: unknown, insideName: boolean) => { + if (node === null || typeof node !== "object") return + const token = node as CstToken + if (token.image !== undefined && token.tokenType !== undefined) { + if (insideName) names.push(token.image) + return + } + const rule = node as CstNode + const naming = insideName || rule.name === "identifier" + for (const children of Object.values(rule.children ?? {})) { + for (const child of children) walk(child, naming) + } + } + walk(parse(sql).cst, false) + return names +} + +/** + * The parser decides which words are syntax, so a mistake would rewrite a + * table or column name. Names must come back exactly as written, and nothing + * but case may change anywhere else. + */ +describe("docs corpus, capitalized", () => { + const parseable = docsQueries.filter( + (query) => parseToAst(query).errors.length === 0, + ) + + it.each(parseable.map((query, index) => [index, query] as const))( + "#%i keeps every name and the meaning of the statement", + (_index, query) => { + // When + const output = format(query, { capitalize: true }) + const reparsed = parseToAst(output) + + // Then + expect(reparsed.errors).toEqual([]) + expect(namesIn(output)).toEqual(namesIn(query)) + expect(toSql(reparsed.ast).toLowerCase()).toBe( + toSql(parseToAst(query).ast).toLowerCase(), + ) + expect(format(output, { capitalize: true })).toBe(output) + }, + ) +}) diff --git a/tests/formatter/corpus.test.ts b/tests/formatter/corpus.test.ts new file mode 100644 index 0000000..0f8918b --- /dev/null +++ b/tests/formatter/corpus.test.ts @@ -0,0 +1,22 @@ +import { describe, it } from "vitest" +import * as fs from "fs" +import * as path from "path" +import { assertPreserved } from "./oracles" + +const docsQueries: string[] = ( + JSON.parse( + fs.readFileSync( + path.join(__dirname, "..", "fixtures", "docs-queries.json"), + "utf-8", + ), + ) as Array<{ query: string }> +).map((entry) => entry.query) + +describe("docs corpus", () => { + it.each(docsQueries.map((query, index) => [index, query] as const))( + "#%i preserves tokens, adjacency, meaning, and is idempotent", + (_index, query) => { + assertPreserved(query) + }, + ) +}) diff --git a/tests/formatter/fixtures.ts b/tests/formatter/fixtures.ts new file mode 100644 index 0000000..3d94a5f --- /dev/null +++ b/tests/formatter/fixtures.ts @@ -0,0 +1,743 @@ +import { FormatOptions } from "../../src/formatter/index" + +export type Fixture = { + name: string + input: string + expected: string + options?: FormatOptions +} + +const columns = (count: number, prefix = "col_") => + Array.from({ length: count }, (_, i) => `${prefix}${i}`) + +export const fixtures: Fixture[] = [ + { + name: "LATEST ON keeps PARTITION BY on its line", + input: + "SELECT * FROM fx_trades WHERE symbol = 'EURUSD' LATEST ON timestamp PARTITION BY venue", + expected: [ + "SELECT *", + "FROM fx_trades", + "WHERE symbol = 'EURUSD'", + "LATEST ON timestamp PARTITION BY venue", + ].join("\n"), + }, + { + name: "ALTER TABLE action starts a line", + input: "ALTER TABLE market_data ADD COLUMN venue SYMBOL CAPACITY 256 CACHE", + expected: [ + "ALTER TABLE market_data", + "ADD COLUMN venue SYMBOL CAPACITY 256 CACHE", + ].join("\n"), + }, + { + name: "LATEST BY is a clause of its own", + input: "SELECT * FROM trades LATEST BY symbol", + expected: ["SELECT *", "FROM trades", "LATEST BY symbol"].join("\n"), + }, + { + name: "ALTER TABLE WAL and storage policy actions start a line", + input: "ALTER TABLE t SUSPEND WAL", + expected: ["ALTER TABLE t", "SUSPEND WAL"].join("\n"), + }, + { + name: "ALTER TABLE DROP STORAGE POLICY starts a line", + input: "ALTER TABLE t DROP STORAGE POLICY", + expected: ["ALTER TABLE t", "DROP STORAGE POLICY"].join("\n"), + }, + { + name: "ALTER MATERIALIZED VIEW DROP EXPIRE starts a line", + input: "ALTER MATERIALIZED VIEW price_1h DROP EXPIRE", + expected: ["ALTER MATERIALIZED VIEW price_1h", "DROP EXPIRE"].join("\n"), + }, + { + name: "EXPIRE ROWS starts a line", + input: + "CREATE MATERIALIZED VIEW price_1h AS (SELECT ts, avg(px) FROM trades SAMPLE BY 1h) PARTITION BY DAY EXPIRE ROWS KEEP LATEST PARTITION BY sym", + expected: [ + "CREATE MATERIALIZED VIEW price_1h AS (", + " SELECT ts, avg(px)", + " FROM trades", + " SAMPLE BY 1h", + ") PARTITION BY DAY", + "EXPIRE ROWS KEEP LATEST PARTITION BY sym", + ].join("\n"), + }, + { + name: "UPDATE joins break like SELECT joins", + input: "UPDATE t SET a = 1 FROM u JOIN v ON u.x = v.x WHERE t.id = u.id", + expected: [ + "UPDATE t", + "SET a = 1", + "FROM u", + "JOIN v ON u.x = v.x", + "WHERE t.id = u.id", + ].join("\n"), + }, + { + name: "short select list stays inline", + input: + "SELECT symbol, approx_percentile(price, 0.5, 2) AS median, count() FROM trades WHERE timestamp IN today() GROUP BY symbol ORDER BY median DESC", + expected: [ + "SELECT symbol, approx_percentile(price, 0.5, 2) AS median, count()", + "FROM trades", + "WHERE timestamp IN today()", + "GROUP BY symbol", + "ORDER BY median DESC", + ].join("\n"), + }, + { + name: "CREATE TABLE options", + input: + "CREATE TABLE trades (ts TIMESTAMP, price DOUBLE) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts) TTL 30 DAYS", + expected: [ + "CREATE TABLE trades (", + " ts TIMESTAMP,", + " price DOUBLE", + ") TIMESTAMP(ts) PARTITION BY DAY WAL", + "DEDUP UPSERT KEYS(ts)", + "TTL 30 DAYS", + ].join("\n"), + }, + { + name: "short CREATE TABLE stays on one line with its options", + input: + "CREATE TABLE t (ts TIMESTAMP, price DOUBLE) TIMESTAMP(ts) PARTITION BY DAY", + expected: + "CREATE TABLE t (ts TIMESTAMP, price DOUBLE) TIMESTAMP(ts) PARTITION BY DAY", + }, + { + name: "CREATE TABLE column comments and options after the closing parenthesis", + input: + "CREATE TABLE trades_new (\n event_time TIMESTAMP, -- new designated timestamp\n ingest_time TIMESTAMP,\n symbol SYMBOL,\n price DOUBLE\n) TIMESTAMP(event_time) PARTITION BY DAY", + expected: [ + "CREATE TABLE trades_new (", + " event_time TIMESTAMP, -- new designated timestamp", + " ingest_time TIMESTAMP,", + " symbol SYMBOL,", + " price DOUBLE", + ") TIMESTAMP(event_time) PARTITION BY DAY", + ].join("\n"), + }, + { + name: "STORAGE POLICY starts a line", + input: + "CREATE MATERIALIZED VIEW test WITH BASE trades AS (SELECT ts, k, avg(v) FROM trades SAMPLE BY 30s), INDEX (k CAPACITY 1024) PARTITION BY DAY STORAGE POLICY(TO PARQUET 10d, TO REMOTE 1M, DROP LOCAL 3M) IN VOLUME vol1", + expected: [ + "CREATE MATERIALIZED VIEW test", + "WITH", + " BASE trades AS (", + " SELECT ts, k, avg(v)", + " FROM trades", + " SAMPLE BY 30s", + " ),", + " INDEX (k CAPACITY 1024) PARTITION BY DAY", + "STORAGE POLICY(TO PARQUET 10d, TO REMOTE 1M, DROP LOCAL 3M)", + "IN VOLUME vol1", + ].join("\n"), + }, + { + name: "CREATE TABLE AS keeps options after the query", + input: + "CREATE TABLE t AS (SELECT * FROM trades) TIMESTAMP(ts) PARTITION BY DAY WAL", + expected: [ + "CREATE TABLE t AS (", + " SELECT *", + " FROM trades", + ") TIMESTAMP(ts) PARTITION BY DAY WAL", + ].join("\n"), + }, + { + name: "long CREATE TABLE column list expands", + input: `CREATE TABLE trades (${columns(8, "column_number_") + .map((c) => `${c} DOUBLE`) + .join(", ")}) TIMESTAMP(ts) PARTITION BY DAY WAL`, + expected: [ + "CREATE TABLE trades (", + ...columns(8, "column_number_").map( + (c, i) => ` ${c} DOUBLE${i < 7 ? "," : ""}`, + ), + ") TIMESTAMP(ts) PARTITION BY DAY WAL", + ].join("\n"), + }, + { + name: "INSERT ... SELECT", + input: + "INSERT INTO trades_archive SELECT * FROM trades WHERE timestamp < dateadd('d', -30, now())", + expected: [ + "INSERT INTO trades_archive", + " SELECT *", + " FROM trades", + " WHERE timestamp < dateadd('d', -30, now())", + ].join("\n"), + }, + { + name: "INSERT ... VALUES rows stay inline when short", + input: "INSERT INTO t (a, b) VALUES (1, 2), (3, 4)", + expected: ["INSERT INTO t (a, b)", "VALUES (1, 2), (3, 4)"].join("\n"), + }, + { + name: "long VALUES row expands", + input: `INSERT INTO t VALUES (${columns(10, "'value_number_") + .map((c) => `${c}'`) + .join(", ")})`, + expected: [ + "INSERT INTO t", + "VALUES (", + ...columns(10, "'value_number_").map( + (c, i) => ` ${c}'${i < 9 ? "," : ""}`, + ), + ")", + ].join("\n"), + }, + { + name: "PARTITION LIST keeps LIST in the clause head when the list breaks", + input: + "ALTER TABLE tab ATTACH PARTITION LIST '2022', '2023', '2024', '2025', '2026', '2027'", + options: { maxWidth: 50 }, + expected: [ + "ALTER TABLE tab", + "ATTACH PARTITION LIST", + " '2022',", + " '2023',", + " '2024',", + " '2025',", + " '2026',", + " '2027'", + ].join("\n"), + }, + { + name: "DROP PARTITION WHERE stays inline", + input: "ALTER TABLE tab DROP PARTITION WHERE timestamp < '2024-01-01'", + expected: [ + "ALTER TABLE tab", + "DROP PARTITION WHERE timestamp < '2024-01-01'", + ].join("\n"), + }, + { + name: "UPDATE", + input: "UPDATE t SET a = 1, b = 2 WHERE id = 3", + expected: ["UPDATE t", "SET a = 1, b = 2", "WHERE id = 3"].join("\n"), + }, + { + name: "long select list expands one item per line", + input: `SELECT ${columns(12).join(", ")} FROM trades`, + expected: [ + "SELECT", + ...columns(12).map((c, i) => ` ${c}${i < 11 ? "," : ""}`), + "FROM trades", + ].join("\n"), + }, + { + name: "short predicates stay inline", + input: "SELECT a FROM t WHERE x = 1 AND y = 2", + expected: ["SELECT a", "FROM t", "WHERE x = 1 AND y = 2"].join("\n"), + }, + { + name: "long predicates break before AND, BETWEEN keeps its AND", + input: + "SELECT a FROM t WHERE ts BETWEEN '2024-01-01' AND '2024-02-01' AND x = 1 OR y = 2", + options: { maxWidth: 30 }, + expected: [ + "SELECT a", + "FROM t", + "WHERE", + " ts BETWEEN '2024-01-01' AND '2024-02-01'", + " AND x = 1", + " OR y = 2", + ].join("\n"), + }, + { + name: "join ON predicates", + input: + "SELECT * FROM a INNER JOIN b ON a.id = b.id AND a.ts = b.ts WHERE a.x = 1", + expected: [ + "SELECT *", + "FROM a", + "INNER JOIN b ON a.id = b.id AND a.ts = b.ts", + "WHERE a.x = 1", + ].join("\n"), + }, + { + name: "long WINDOW JOIN breaks into sub-clauses", + input: + "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_sum FROM trades t WINDOW JOIN prices p ON (t.sym = p.sym) RANGE BETWEEN 1 minute PRECEDING AND 1 minute FOLLOWING EXCLUDE PREVAILING ORDER BY t.ts", + expected: [ + "SELECT t.sym, t.price, t.ts, sum(p.price) AS window_sum", + "FROM trades t", + "WINDOW JOIN prices p", + " ON (t.sym = p.sym)", + " RANGE BETWEEN 1 minute PRECEDING AND 1 minute FOLLOWING", + " EXCLUDE PREVAILING", + "ORDER BY t.ts", + ].join("\n"), + }, + { + name: "join predicates keep AND inline inside the ON sub-clause", + input: + "SELECT t.sym FROM trades t WINDOW JOIN prices p ON (t.sym = p.sym) AND p.price < 300 RANGE BETWEEN 2 minutes PRECEDING AND 2 minutes FOLLOWING EXCLUDE PREVAILING", + expected: [ + "SELECT t.sym", + "FROM trades t", + "WINDOW JOIN prices p", + " ON (t.sym = p.sym) AND p.price < 300", + " RANGE BETWEEN 2 minutes PRECEDING AND 2 minutes FOLLOWING", + " EXCLUDE PREVAILING", + ].join("\n"), + }, + { + name: "short join keeps its sub-clauses inline", + input: "SELECT * FROM trades ASOF JOIN quotes ON (symbol) TOLERANCE 5s", + expected: [ + "SELECT *", + "FROM trades", + "ASOF JOIN quotes ON (symbol) TOLERANCE 5s", + ].join("\n"), + }, + { + name: "ASOF JOIN", + input: + "SELECT * FROM trades ASOF JOIN quotes ON (symbol) WHERE ts > now() - 2d", + expected: [ + "SELECT *", + "FROM trades", + "ASOF JOIN quotes ON (symbol)", + "WHERE ts > now() - 2d", + ].join("\n"), + }, + { + name: "SAMPLE BY keeps bounds, FILL, ALIGN TO, and WITH OFFSET", + input: + "SELECT ts, avg(price) FROM trades SAMPLE BY 1h FROM '2024-01-01' TO '2024-02-01' FILL(PREV) ALIGN TO CALENDAR WITH OFFSET '00:30'", + expected: [ + "SELECT ts, avg(price)", + "FROM trades", + "SAMPLE BY 1h FROM '2024-01-01' TO '2024-02-01' FILL(PREV) ALIGN TO CALENDAR WITH OFFSET '00:30'", + ].join("\n"), + }, + { + name: "subquery block", + input: "SELECT * FROM (SELECT a FROM t) x", + expected: ["SELECT *", "FROM (", " SELECT a", " FROM t", ") x"].join( + "\n", + ), + }, + { + name: "CTE", + input: "WITH c AS (SELECT a FROM t) SELECT * FROM c", + expected: [ + "WITH c AS (", + " SELECT a", + " FROM t", + ")", + "SELECT *", + "FROM c", + ].join("\n"), + }, + { + name: "IN subquery block", + input: "SELECT a FROM t WHERE x IN (SELECT y FROM u)", + expected: [ + "SELECT a", + "FROM t", + "WHERE x IN (", + " SELECT y", + " FROM u", + ")", + ].join("\n"), + }, + { + name: "short CASE stays inline", + input: "SELECT CASE WHEN a THEN 1 ELSE 0 END AS f, b FROM t", + expected: ["SELECT CASE WHEN a THEN 1 ELSE 0 END AS f, b", "FROM t"].join( + "\n", + ), + }, + { + name: "long CASE breaks per branch", + input: "SELECT CASE WHEN a THEN 1 ELSE 0 END AS f, b FROM t", + options: { maxWidth: 30 }, + expected: [ + "SELECT", + " CASE", + " WHEN a THEN 1", + " ELSE 0", + " END AS f,", + " b", + "FROM t", + ].join("\n"), + }, + { + name: "window PARTITION BY and ORDER BY stay inside the parentheses", + input: + "SELECT ts, avg(price) OVER (PARTITION BY symbol ORDER BY ts) FROM trades", + expected: [ + "SELECT ts, avg(price) OVER (PARTITION BY symbol ORDER BY ts)", + "FROM trades", + ].join("\n"), + }, + { + name: "long window breaks into sub-clauses inside OVER", + input: + "SELECT timestamp, symbol, price, avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING) AS moving_avg FROM trades", + expected: [ + "SELECT", + " timestamp,", + " symbol,", + " price,", + " avg(price) OVER (", + " PARTITION BY symbol", + " ORDER BY timestamp", + " ROWS 300 PRECEDING", + " ) AS moving_avg", + "FROM trades", + ].join("\n"), + }, + { + name: "WINDOW clause with a short named window", + input: + "SELECT timestamp, symbol, sum(amount) OVER w AS cumulative_volume FROM trades WINDOW w AS (PARTITION BY symbol ORDER BY timestamp ANCHOR DAILY '00:00')", + expected: [ + "SELECT timestamp, symbol, sum(amount) OVER w AS cumulative_volume", + "FROM trades", + "WINDOW w AS (PARTITION BY symbol ORDER BY timestamp ANCHOR DAILY '00:00')", + ].join("\n"), + }, + { + name: "WINDOW clause with long named windows breaks each definition", + input: + "SELECT sum(amount) OVER w1, avg(price) OVER w2 FROM trades WINDOW w1 AS (PARTITION BY symbol ORDER BY timestamp ANCHOR DAILY '00:00' 'Europe/London'), w2 AS (PARTITION BY symbol ORDER BY timestamp ROWS BETWEEN 100 PRECEDING AND CURRENT ROW)", + expected: [ + "SELECT sum(amount) OVER w1, avg(price) OVER w2", + "FROM trades", + "WINDOW", + " w1 AS (", + " PARTITION BY symbol", + " ORDER BY timestamp", + " ANCHOR DAILY '00:00' 'Europe/London'", + " ),", + " w2 AS (", + " PARTITION BY symbol", + " ORDER BY timestamp", + " ROWS BETWEEN 100 PRECEDING AND CURRENT ROW", + " )", + ].join("\n"), + }, + { + name: "short PIVOT stays inline", + input: + "SELECT * FROM trades PIVOT (avg(price) FOR symbol IN ('BTC-USDT', 'ETH-USDT'))", + expected: [ + "SELECT *", + "FROM trades", + "PIVOT (avg(price) FOR symbol IN ('BTC-USDT', 'ETH-USDT'))", + ].join("\n"), + }, + { + name: "long PIVOT breaks aggregates, FOR, and GROUP BY", + input: + "SELECT * FROM markouts PIVOT (count() AS fills, avg(quantity) AS avg_size, sum(quantity) AS volume, avg(((best_bid + best_ask) / 2 - price) / price * 10000) AS markout_bps FOR offset IN (0 AS at_fill, 5000000000 AS t_5s, 60000000000 AS t_1m) GROUP BY symbol, ecn) ORDER BY t_5s_markout_bps", + expected: [ + "SELECT *", + "FROM markouts", + "PIVOT (", + " count() AS fills,", + " avg(quantity) AS avg_size,", + " sum(quantity) AS volume,", + " avg(((best_bid + best_ask) / 2 - price) / price * 10000) AS markout_bps", + " FOR offset IN (0 AS at_fill, 5000000000 AS t_5s, 60000000000 AS t_1m)", + " GROUP BY symbol, ecn", + ")", + "ORDER BY t_5s_markout_bps", + ].join("\n"), + }, + { + name: "window PARTITION BY list stays on its sub-clause line", + input: + "SELECT symbol, avg(price) OVER (PARTITION BY symbol, venue ORDER BY timestamp ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) AS moving FROM trades", + expected: [ + "SELECT", + " symbol,", + " avg(price) OVER (", + " PARTITION BY symbol, venue", + " ORDER BY timestamp", + " ROWS BETWEEN 100 PRECEDING AND CURRENT ROW", + " ) AS moving", + "FROM trades", + ].join("\n"), + }, + { + name: "UNION ALL", + input: "SELECT a FROM t UNION ALL SELECT b FROM u", + expected: ["SELECT a", "FROM t", "UNION ALL", "SELECT b", "FROM u"].join( + "\n", + ), + }, + { + name: "EXPLAIN stays inline with the first clause", + input: "EXPLAIN SELECT a FROM t WHERE x = 1", + expected: ["EXPLAIN SELECT a", "FROM t", "WHERE x = 1"].join("\n"), + }, + { + name: "EXPLAIN alone", + input: "EXPLAIN", + expected: "EXPLAIN", + }, + { + name: "implicit SELECT", + input: "trades WHERE symbol = 'BTC-USD'", + expected: ["trades", "WHERE symbol = 'BTC-USD'"].join("\n"), + }, + { + name: "DECLARE with one variable stays inline", + input: "DECLARE OVERRIDABLE @x := 1 SELECT @x FROM t", + expected: ["DECLARE OVERRIDABLE @x := 1", "SELECT @x", "FROM t"].join("\n"), + }, + { + name: "DECLARE with several variables breaks one per line", + input: "DECLARE @x := 5, OVERRIDABLE @y := 6 SELECT @x + @y FROM t", + expected: [ + "DECLARE", + " @x := 5,", + " OVERRIDABLE @y := 6", + "SELECT @x + @y", + "FROM t", + ].join("\n"), + }, + { + name: "materialized view", + input: + "CREATE MATERIALIZED VIEW mv WITH BASE trades REFRESH INCREMENTAL AS (SELECT ts, avg(price) FROM trades SAMPLE BY 1h) PARTITION BY DAY", + expected: [ + "CREATE MATERIALIZED VIEW mv", + "WITH BASE trades", + "REFRESH INCREMENTAL AS (", + " SELECT ts, avg(price)", + " FROM trades", + " SAMPLE BY 1h", + ") PARTITION BY DAY", + ].join("\n"), + }, + { + name: "AS followed by a query keyword ends the line and formats the query", + input: + "CREATE MATERIALIZED VIEW trades_daily REFRESH PERIOD (LENGTH 1d TIME ZONE 'Europe/London' DELAY 2h) AS SELECT timestamp, symbol, avg(price) AS avg_price FROM trades SAMPLE BY 1d", + expected: [ + "CREATE MATERIALIZED VIEW trades_daily", + "REFRESH PERIOD (LENGTH 1d TIME ZONE 'Europe/London' DELAY 2h) AS", + " SELECT timestamp, symbol, avg(price) AS avg_price", + " FROM trades", + " SAMPLE BY 1d", + ].join("\n"), + }, + { + name: "CREATE TABLE AS SELECT without parentheses", + input: "CREATE TABLE t AS SELECT * FROM trades WHERE x = 1", + expected: [ + "CREATE TABLE t AS", + " SELECT *", + " FROM trades", + " WHERE x = 1", + ].join("\n"), + }, + { + name: "INSERT with a CTE indents the whole query", + input: "INSERT INTO t WITH c AS (SELECT 1 AS x) SELECT x FROM c", + expected: [ + "INSERT INTO t", + " WITH c AS (", + " SELECT 1 AS x", + " )", + " SELECT x", + " FROM c", + ].join("\n"), + }, + { + name: "implicit select in FROM opens a block", + input: + "SELECT symbol, side FROM (trades_latest_1d LATEST ON timestamp PARTITION BY symbol, side) ORDER BY timestamp DESC", + expected: [ + "SELECT symbol, side", + "FROM (", + " trades_latest_1d", + " LATEST ON timestamp PARTITION BY symbol, side", + ")", + "ORDER BY timestamp DESC", + ].join("\n"), + }, + { + name: "implicit select in a CTE opens a block", + input: "WITH c AS (trades WHERE x = 1) SELECT * FROM c", + expected: [ + "WITH c AS (", + " trades", + " WHERE x = 1", + ")", + "SELECT *", + "FROM c", + ].join("\n"), + }, + { + name: "a plain parenthesized table after JOIN stays inline", + input: "SELECT * FROM orders ASOF JOIN (md) ON (symbol)", + expected: ["SELECT *", "FROM orders", "ASOF JOIN (md) ON (symbol)"].join( + "\n", + ), + }, + { + name: "FROM inside function arguments is not a clause", + input: "SELECT extract(hour FROM ts), substring(s FROM 1) FROM t", + expected: [ + "SELECT extract(hour FROM ts), substring(s FROM 1)", + "FROM t", + ].join("\n"), + }, + { + name: "keyword-shaped identifiers are not clauses", + input: "SELECT status, type FROM t", + expected: ["SELECT status, type", "FROM t"].join("\n"), + }, + { + name: "operator spacing", + input: + "SELECT a->b, a- >b, count( * ), a::long, arr[1:3], x = -2, 1 - 2, -1 FROM t", + expected: [ + "SELECT a -> b, a - > b, count(*), a::long, arr[1:3], x = -2, 1 - 2, -1", + "FROM t", + ].join("\n"), + }, + { + name: "unknown characters keep their gaps", + input: "SELECT a $$$ b, a$$$b FROM t", + expected: ["SELECT a $$$ b, a$$$b", "FROM t"].join("\n"), + }, + { + name: "unknown character keeps a newline gap", + input: "SELECT a\n$$$ b FROM t", + expected: ["SELECT a", "$$$ b", "FROM t"].join("\n"), + }, + { + name: "function parentheses keep source spacing", + input: "SELECT count (*), now() FROM t WHERE x IN(1, 2)", + expected: ["SELECT count (*), now()", "FROM t", "WHERE x IN(1, 2)"].join( + "\n", + ), + }, + { + name: "unterminated string preserves the rest", + input: "SELECT a FROM t WHERE x = 'oops; SELECT b;", + expected: ["SELECT a", "FROM t", "WHERE x = 'oops; SELECT b;"].join("\n"), + }, + { + name: "unterminated block comment preserves the rest", + input: "SELECT a FROM t /* open", + expected: ["SELECT a", "FROM t /* open"].join("\n"), + }, + { + name: "unclosed parenthesis preserves the rest", + input: "SELECT (a; SELECT b;", + expected: "SELECT (a; SELECT b;", + }, + { + name: "mismatched delimiters preserve from the outermost opener", + input: "SELECT f([)] FROM t", + expected: "SELECT f([)] FROM t", + }, + { + name: "stray closer is an ordinary token", + input: "SELECT a) FROM t", + expected: ["SELECT a)", "FROM t"].join("\n"), + }, + { + name: "line comment after a comma stays on the item line", + input: "SELECT a, -- first\n b FROM t", + expected: ["SELECT", " a, -- first", " b", "FROM t"].join("\n"), + }, + { + name: "trailing line comment", + input: "SELECT a FROM t -- trailing\n", + expected: ["SELECT a", "FROM t -- trailing"].join("\n"), + }, + { + name: "inline block comment", + input: "SELECT a /* c */, b FROM t", + expected: ["SELECT a /* c */, b", "FROM t"].join("\n"), + }, + { + name: "block comment on its own line", + input: "/* header */\nSELECT a FROM t", + expected: ["/* header */", "SELECT a", "FROM t"].join("\n"), + }, + { + name: "block comment followed by a newline ends its line", + input: "SELECT /*+ hint */\n a, b FROM t", + expected: ["SELECT", " /*+ hint */", " a,", " b", "FROM t"].join("\n"), + }, + { + name: "comment inside a phrase leaves the phrase inline", + input: "SELECT * FROM t LATEST /* x */ ON ts", + expected: ["SELECT *", "FROM t LATEST /* x */ ON ts"].join("\n"), + }, + { + name: "long IN list expands", + input: `SELECT * FROM t WHERE symbol IN (${columns(12, "'SYM") + .map((c) => `${c}'`) + .join(", ")})`, + expected: [ + "SELECT *", + "FROM t", + "WHERE symbol IN (", + ...columns(12, "'SYM").map((c, i) => ` ${c}'${i < 11 ? "," : ""}`), + ")", + ].join("\n"), + }, + { + name: "long function call stays inline", + input: `SELECT f(${columns(12, "argument_").join(", ")}) FROM t`, + expected: [ + `SELECT f(${columns(12, "argument_").join(", ")})`, + "FROM t", + ].join("\n"), + }, + { + name: "multiple statements", + input: "SELECT a FROM t; SELECT b FROM u", + expected: ["SELECT a", "FROM t;", "", "SELECT b", "FROM u"].join("\n"), + }, + { + name: "trailing comma before a clause starter survives", + input: "CREATE TABLE t (a INT) WITH maxUncommittedRows=1,\n IN VOLUME v", + expected: [ + "CREATE TABLE t (a INT)", + "WITH maxUncommittedRows = 1,", + "IN VOLUME v", + ].join("\n"), + }, + { + name: "other statements normalize spacing without clause breaks", + input: + "CREATE USER \nadministrator \nWITH PASSWORD \nadminpwd;\nGRANT \nALL\n TO administrator \nWITH GRANT OPTION;", + expected: [ + "CREATE USER administrator WITH PASSWORD adminpwd;", + "", + "GRANT ALL TO administrator WITH GRANT OPTION;", + ].join("\n"), + }, + { + name: "other statements keep comments and unknown input", + input: "DROP TABLE t -- gone\n$$$ x", + expected: ["DROP TABLE t -- gone", "$$$ x"].join("\n"), + }, + { + name: "lower case is preserved and tabs indent", + input: `select ${columns(12).join(",")} from t where x=1 limit 10`, + options: { indent: "\t" }, + expected: [ + "select", + ...columns(12).map((c, i) => `\t${c}${i < 11 ? "," : ""}`), + "from t", + "where x = 1", + "limit 10", + ].join("\n"), + }, +] diff --git a/tests/formatter/format.test.ts b/tests/formatter/format.test.ts new file mode 100644 index 0000000..357c431 --- /dev/null +++ b/tests/formatter/format.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest" +import { format } from "../../src/formatter/index" +import { fixtures } from "./fixtures" +import { assertPreserved } from "./oracles" + +// Fixtures pin the layout at width 80 unless a case sets its own width. +const FIXTURE_WIDTH = 80 + +describe("format fixtures", () => { + it.each(fixtures.map((fixture) => [fixture.name, fixture] as const))( + "%s", + (_name, fixture) => { + // Given + const options = { maxWidth: FIXTURE_WIDTH, ...fixture.options } + + // When + const output = format(fixture.input, options) + + // Then + expect(output).toBe(fixture.expected) + assertPreserved(fixture.input, options) + }, + ) +}) + +describe("format options", () => { + it("rejects an indent that is not whitespace", () => { + expect(() => format("SELECT 1", { indent: "--" })).toThrow(TypeError) + }) + + it.each([0, -1, Number.NaN, Number.POSITIVE_INFINITY])( + "rejects maxWidth %s", + (maxWidth) => { + expect(() => format("SELECT 1", { maxWidth })).toThrow(TypeError) + }, + ) + + it("returns an empty string for whitespace input", () => { + expect(format(" \n ")).toBe("") + }) +}) diff --git a/tests/formatter/grammar-drift.test.ts b/tests/formatter/grammar-drift.test.ts new file mode 100644 index 0000000..401ae15 --- /dev/null +++ b/tests/formatter/grammar-drift.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it } from "vitest" +import { StatementKind } from "../../src/formatter/context" +import { clausePhrases } from "../../src/formatter/phrases" +import { partsOf } from "./grammarClauses" + +/** + * The formatter's phrase table is hand-written, while QuestDB syntax arrives + * through the parser. These tests read the grammar and fail when the two drift + * apart, so adding a clause to the parser forces a layout decision here. + */ + +type Kind = Exclude + +/** Grammar rules that make up each statement kind the formatter lays out. */ +/** Dispatch rules that carry the leading words of each statement. */ +const HEAD_RULES = ["createStatement", "alterStatement"] + +const STATEMENT_RULES: Record = { + select: [ + "selectStatement", + "selectBody", + "simpleSelect", + "fromClause", + "joinClause", + ], + insert: ["insertStatement"], + update: ["updateStatement"], + createTable: ["createTableBody"], + createMaterializedView: ["createMaterializedViewBody"], + createLiveView: ["createLiveViewBody"], + alterTable: ["alterTableAction"], + alterMaterializedView: ["alterMaterializedViewAction"], +} + +/** + * Sub-rules whose leading keywords are values or belong to a nested query, + * not to a clause of the statement being laid out. + */ +const NON_CLAUSE_RULES = new Set([ + "booleanLiteral", + "columnDefinition", + "columnRef", + "dataType", + "expression", + "fromClause", + "fromSource", + "identifier", + "implicitSelectBody", + "indexDefinition", + "partitionPeriod", + "pivotBody", + "qualifiedName", + "selectList", + "selectStatement", + "setClause", + "stringOrIdentifier", + "stringOrQualifiedName", + "tableName", + "tableNameOrString", + "tableRef", + "timeUnit", +]) + +/** + * Keyword sequences the grammar can start a part with that the formatter keeps + * on the current line on purpose. Every entry is a layout decision, not a gap. + */ +const INLINE_BY_DESIGN: Record = { + select: ["Distinct"], + insert: ["Into", "Atomic", "Batch"], + update: [], + createTable: [ + "Table", + "Atomic", + "Batch", + "If Not Exists", + "As", + "Timestamp", + "Partition By", + "Wal", + "Bypass Wal", + "Format", + "Owned By", + ], + createMaterializedView: [ + "Materialized View", + "If Not Exists", + "As", + "Timestamp", + "Partition By", + "Wal", + "Bypass Wal", + "Format", + "Owned By", + // REFRESH strategies stay on the REFRESH line. + "Immediate", + "Manual", + "Every", + "Period", + "Incremental", + "Deferred", + "Start", + "Limit", + "Time Zone", + ], + createLiveView: [ + "Live View", + "If Not Exists", + "Flush Every", + "In Memory", + "Partition By", + "Start From", + "As", + "Owned By", + // The query after AS, with or without parentheses. + "Select", + "Declare", + "With", + ], + alterTable: [], + alterMaterializedView: [], +} + +const words = (phrase: string) => phrase.split(" ") + +/** True when one phrase is a leading run of the other, in either direction. */ +const related = (a: string, b: string) => { + const [short, long] = words(a).length <= words(b).length ? [a, b] : [b, a] + return words(short).every((word, i) => word === words(long)[i]) +} + +const tablePhrases = (kind: Kind) => + clausePhrases[kind].map((phrase) => phrase.names.join(" ")) + +const grammarParts = (kind: Kind) => + STATEMENT_RULES[kind] + .flatMap(partsOf) + .filter((part) => !NON_CLAUSE_RULES.has(part.source)) + .flatMap((part) => part.phrases.map((phrase) => ({ phrase, part }))) + +const kinds = Object.keys(STATEMENT_RULES) as Kind[] + +describe("phrase table follows the grammar", () => { + it.each(kinds)( + "%s: every clause the grammar can start is laid out or inline by design", + (kind) => { + // Given + const known = [...tablePhrases(kind), ...INLINE_BY_DESIGN[kind]] + + // When + const unhandled = grammarParts(kind) + .filter(({ phrase }) => !known.some((entry) => related(entry, phrase))) + .map(({ phrase, part }) => `${phrase} (${part.source})`) + + // Then + expect( + unhandled, + "add a phrase to phrases.ts, or list it as inline by design", + ).toEqual([]) + }, + ) + + it("every phrase in the table still exists in the grammar", () => { + // Given + const grammar = [ + ...kinds.flatMap((kind) => + grammarParts(kind).map(({ phrase }) => phrase), + ), + ...HEAD_RULES.flatMap(partsOf).flatMap((part) => part.phrases), + ] + + // When + const dead = [...new Set(kinds.flatMap(tablePhrases))].filter( + (phrase) => !grammar.some((entry) => related(entry, phrase)), + ) + + // Then + expect(dead, "the grammar no longer starts a part with this").toEqual([]) + }) +}) diff --git a/tests/formatter/grammarClauses.ts b/tests/formatter/grammarClauses.ts new file mode 100644 index 0000000..ea9af1d --- /dev/null +++ b/tests/formatter/grammarClauses.ts @@ -0,0 +1,139 @@ +import { + Alternation, + IProduction, + NonTerminal, + Option, + Repetition, + RepetitionMandatory, + RepetitionMandatoryWithSeparator, + RepetitionWithSeparator, + Rule, + Terminal, +} from "chevrotain" +import { parser } from "../../src/parser/parser" +import { keywordTokenArray } from "../../src/parser/tokens" + +const keywordNames = new Set(keywordTokenArray.map((token) => token.name)) +const MAX_WORDS = 3 +const MAX_DEPTH = 4 + +const productions: Record = parser.getGAstProductions() + +/** Chevrotain types the children of a group loosely; read them structurally. */ +const childrenOf = (item: IProduction): IProduction[] => + (item as { definition?: IProduction[] }).definition ?? [] + +const isGroup = (item: IProduction) => + item instanceof Option || + item instanceof Repetition || + item instanceof RepetitionMandatory || + item instanceof RepetitionWithSeparator || + item instanceof RepetitionMandatoryWithSeparator + +const isOptionalGroup = (item: IProduction) => + item instanceof Option || + item instanceof Repetition || + item instanceof RepetitionWithSeparator + +const isKeyword = (item: IProduction): item is Terminal => + item instanceof Terminal && keywordNames.has(item.terminalType.name) + +const ruleBody = (item: NonTerminal): IProduction[] => + item.referencedRule?.definition ?? + productions[item.nonTerminalName].definition + +/** Keyword sequences that can begin `definition`, e.g. [["Latest", "On"]]. */ +const firstKeywords = ( + definition: IProduction[], + prefix: string[], + depth: number, +): string[][] => { + const done = prefix.length > 0 ? [prefix] : [] + if (prefix.length >= MAX_WORDS || depth > MAX_DEPTH) return done + const [head, ...rest] = definition + if (head === undefined) return done + + if (head instanceof Terminal) { + if (!isKeyword(head)) return done + return firstKeywords(rest, [...prefix, head.terminalType.name], depth) + } + if (head instanceof NonTerminal) { + return prefix.length > 0 + ? done + : firstKeywords(ruleBody(head), prefix, depth + 1) + } + if (head instanceof Alternation) { + return childrenOf(head).flatMap((alternative) => + firstKeywords([...childrenOf(alternative), ...rest], prefix, depth), + ) + } + if (isGroup(head)) { + return [ + ...firstKeywords([...childrenOf(head), ...rest], prefix, depth), + ...(isOptionalGroup(head) ? firstKeywords(rest, prefix, depth) : []), + ] + } + return done +} + +export type GrammarPart = { source: string; phrases: string[] } + +/** + * The keyword sequences that can begin each part of a grammar rule. Optional + * and repeated groups are flattened, since they only structure the rule. A + * choice contributes the leading keywords of every alternative, a sub-rule the + * keywords it can begin with, and consecutive inline keywords form one phrase. + */ +export const partsOf = (ruleName: string): GrammarPart[] => { + const rule = productions[ruleName] + if (rule === undefined) throw new Error(`unknown grammar rule: ${ruleName}`) + const parts: GrammarPart[] = [] + + const push = (source: string, sequences: string[][]) => { + const phrases = [ + ...new Set(sequences.map((words) => words.join(" ")).filter(Boolean)), + ] + if (phrases.length > 0) parts.push({ source, phrases }) + } + + const walk = (items: IProduction[], source: string) => { + let run: string[] = [] + // Keywords after a choice belong to it: `OR(IN | ...) VOLUME` is IN VOLUME. + let absorbedByChoice = false + + const flushRun = () => { + if (run.length > 0) push(`${source}/keywords`, [run]) + run = [] + } + + items.forEach((item, index) => { + if (item instanceof Terminal) { + if (!isKeyword(item)) { + flushRun() + absorbedByChoice = false + } else if (!absorbedByChoice) { + run.push(item.terminalType.name) + } + return + } + flushRun() + absorbedByChoice = false + if (item instanceof NonTerminal) { + push(item.nonTerminalName, firstKeywords([item], [], 0)) + } else if (item instanceof Alternation) { + const continuation = items.slice(index + 1) + const alternatives = childrenOf(item).map((alternative) => + firstKeywords([...childrenOf(alternative), ...continuation], [], 0), + ) + alternatives.forEach((phrases, i) => push(`${source}/alt${i}`, phrases)) + absorbedByChoice = alternatives.some((phrases) => phrases.length > 0) + } else if (isGroup(item)) { + walk(childrenOf(item), `${source}/${index}`) + } + }) + flushRun() + } + + walk(rule.definition, ruleName) + return parts +} diff --git a/tests/formatter/lexer.test.ts b/tests/formatter/lexer.test.ts new file mode 100644 index 0000000..286bcb5 --- /dev/null +++ b/tests/formatter/lexer.test.ts @@ -0,0 +1,220 @@ +import { describe, expect, it } from "vitest" +import * as fs from "fs" +import * as path from "path" +import { Lexer } from "chevrotain" +import { allTokens } from "../../src/parser/lexer" +import { + delimiterTokenNames, + operatorTokenNames, + wordTokenNames, +} from "../../src/formatter/classification" +import { + buildFormatterTokens, + reconstruct, + scan, + StreamToken, +} from "../../src/formatter/lexer" + +const docsQueries: string[] = ( + JSON.parse( + fs.readFileSync( + path.join(__dirname, "..", "fixtures", "docs-queries.json"), + "utf-8", + ), + ) as Array<{ query: string }> +).map((entry) => entry.query) + +const signature = (tokens: StreamToken[]) => + tokens + .filter((token) => token.kind !== "whitespace") + .map((token) => `${token.kind}:${token.image}`) + +describe("formatter lexer definition", () => { + it("classifies every parser token", () => { + // Given + const triviaNames = new Set(["WhiteSpace", "LineComment", "BlockComment"]) + const unclassified = allTokens + .filter((token) => !triviaNames.has(token.name)) + .filter((token) => token.PATTERN !== Lexer.NA) + .map((token) => token.name) + .filter( + (name) => + !wordTokenNames.has(name) && + !operatorTokenNames.has(name) && + !delimiterTokenNames.has(name), + ) + + // Then + expect(unclassified).toEqual([]) + }) + + it("builds without definition errors and keeps first-character optimization", () => { + // When / Then + expect( + () => + new Lexer(buildFormatterTokens(), { + positionTracking: "onlyOffset", + ensureOptimizations: true, + }), + ).not.toThrow() + }) + + it("does not mutate the parser token list", () => { + // Given + const parserNames = allTokens.map((token) => token.name) + + // When + buildFormatterTokens() + + // Then + expect(allTokens.map((token) => token.name)).toEqual(parserNames) + expect(parserNames).not.toContain("UnterminatedString") + }) +}) + +describe("scan", () => { + it("reconstructs every docs query exactly", () => { + for (const query of docsQueries) { + expect(reconstruct(scan(query))).toBe(query) + } + }) + + it("keeps whitespace and comments as trivia tokens", () => { + // Given + const sql = "SELECT a -- one\n /* two */ FROM t" + + // When + const tokens = scan(sql) + + // Then + expect(tokens.map((token) => token.kind)).toEqual([ + "word", + "whitespace", + "word", + "whitespace", + "lineComment", + "whitespace", + "blockComment", + "whitespace", + "word", + "whitespace", + "word", + ]) + expect(reconstruct(tokens)).toBe(sql) + }) + + it("classifies operators and delimiters", () => { + // Given + const sql = "a::long <<= (b, c[1:2])" + + // When + const tokens = signature(scan(sql)) + + // Then + expect(tokens).toEqual([ + "word:a", + "operator:::", + "word:long", + "operator:<<=", + "delimiter:(", + "word:b", + "delimiter:,", + "word:c", + "delimiter:[", + "word:1", + "operator::", + "word:2", + "delimiter:]", + "delimiter:)", + ]) + }) + + it.each([ + ["'a''", ["tolerant:'a''"]], + ["'a''b", ["tolerant:'a''b"]], + ["'a' 'b", ["word:'a'", "tolerant:'b"]], + ['"a""', ['tolerant:"a""']], + ["x /* open", ["word:x", "tolerant:/* open"]], + ["'x /* y' z", ["word:'x /* y'", "word:z"]], + ["'x -- y' z", ["word:'x -- y'", "word:z"]], + ["-- it's\nz", ["lineComment:-- it's", "word:z"]], + ["/* it's */ z", ["blockComment:/* it's */", "word:z"]], + ])("lexes %s tolerantly", (sql, expected) => { + // When + const tokens = scan(sql) + + // Then + expect(signature(tokens)).toEqual(expected) + expect(reconstruct(tokens)).toBe(sql) + }) + + it.each([ + ["'unterminated at start", 0], + ["SELECT 'in the middle FROM t", 7], + ["SELECT a FROM t WHERE x = '", 26], + ])( + "consumes the rest of the input from an unterminated string in %s", + (sql, offset) => { + // When + const tokens = scan(sql) + const tolerant = tokens.find((token) => token.kind === "tolerant") + + // Then + expect(tolerant).toMatchObject({ + startOffset: offset, + endOffset: sql.length, + }) + expect(tokens[tokens.length - 1]).toBe(tolerant) + expect(reconstruct(tokens)).toBe(sql) + }, + ) + + it("turns unknown characters into one opaque token", () => { + // Given + const sql = "SELECT a $$$ b" + + // When + const tokens = scan(sql) + + // Then + expect(signature(tokens)).toEqual([ + "word:SELECT", + "word:a", + "opaque:$$$", + "word:b", + ]) + expect(reconstruct(tokens)).toBe(sql) + }) + + it("keeps an opaque token adjacent to a tolerant token", () => { + // Given + const sql = "SELECT $'abc" + + // When + const tokens = scan(sql) + + // Then + expect(signature(tokens)).toEqual([ + "word:SELECT", + "opaque:$", + "tolerant:'abc", + ]) + expect(reconstruct(tokens)).toBe(sql) + }) + + it("keeps offsets contiguous", () => { + // Given + const sql = "SELECT a,\n\tb $ FROM t /* c" + + // When + const tokens = scan(sql) + + // Then + tokens.forEach((token, index) => { + expect(token.startOffset).toBe( + index === 0 ? 0 : tokens[index - 1].endOffset, + ) + }) + expect(tokens[tokens.length - 1].endOffset).toBe(sql.length) + }) +}) diff --git a/tests/formatter/malformed.test.ts b/tests/formatter/malformed.test.ts new file mode 100644 index 0000000..8598a90 --- /dev/null +++ b/tests/formatter/malformed.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "vitest" +import * as fs from "fs" +import * as path from "path" +import { format } from "../../src/formatter/index" +import { scan, StreamToken } from "../../src/formatter/lexer" +import { + assertIdempotent, + assertSameOperatorAdjacency, + assertSameStream, +} from "./oracles" + +const docsQueries: string[] = ( + JSON.parse( + fs.readFileSync( + path.join(__dirname, "..", "fixtures", "docs-queries.json"), + "utf-8", + ), + ) as Array<{ query: string }> +).map((entry) => entry.query) + +const FULL_TRUNCATION_QUERIES = 100 +const RANDOM_TRUNCATIONS_PER_QUERY = 10 + +const createRandom = (seed: number) => { + let state = seed >>> 0 + return (bound: number) => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0 + return state % bound + } +} + +const significant = (sql: string): StreamToken[] => + scan(sql).filter((token) => token.kind !== "whitespace") + +const insertAt = (sql: string, offset: number, text: string) => + sql.slice(0, offset) + text + sql.slice(offset) + +const removeAt = (sql: string, offset: number, length: number) => + sql.slice(0, offset) + sql.slice(offset + length) + +type Mutation = (sql: string, random: (bound: number) => number) => string[] + +const truncations: Mutation = (sql, random) => { + const offsets = + docsQueries.indexOf(sql) < FULL_TRUNCATION_QUERIES + ? Array.from({ length: sql.length }, (_, i) => i) + : Array.from({ length: RANDOM_TRUNCATIONS_PER_QUERY }, () => + random(sql.length), + ) + return offsets.map((offset) => sql.slice(0, offset)) +} + +const removeCloser: Mutation = (sql, random) => { + const closers = significant(sql).filter( + (token) => token.tokenName === "RParen" || token.tokenName === "RBracket", + ) + if (closers.length === 0) return [] + const closer = closers[random(closers.length)] + return [removeAt(sql, closer.startOffset, 1)] +} + +const insertOpener: Mutation = (sql, random) => { + const tokens = significant(sql) + if (tokens.length === 0) return [] + const token = tokens[random(tokens.length)] + return [insertAt(sql, token.startOffset, "(")] +} + +const mismatchCloser: Mutation = (sql, random) => { + const closers = significant(sql).filter( + (token) => token.tokenName === "RParen", + ) + if (closers.length === 0) return [] + const closer = closers[random(closers.length)] + return [ + insertAt(removeAt(sql, closer.startOffset, 1), closer.startOffset, "]"), + ] +} + +const semicolonInsideParens: Mutation = (sql, random) => { + const openers = significant(sql).filter( + (token) => token.tokenName === "LParen", + ) + if (openers.length === 0) return [] + const opener = openers[random(openers.length)] + return [insertAt(sql, opener.endOffset, ";")] +} + +const foreignTokens: Mutation = (sql, random) => { + const tokens = significant(sql) + if (tokens.length === 0) return [] + return ["$$$", "->", "<=>", "größe", "$$$'", "/*", "--"].map((text) => + insertAt(sql, tokens[random(tokens.length)].endOffset, ` ${text} `), + ) +} + +const commentInsidePhrase: Mutation = (sql) => { + const tokens = significant(sql) + return tokens.flatMap((token, index) => { + const next = tokens[index + 1] + if (next === undefined) return [] + const phrase = `${token.tokenName} ${next.tokenName}` + const isPhrase = [ + "Latest On", + "Sample By", + "Group By", + "Order By", + "Partition By", + "Asof Join", + "Add Column", + "Union All", + ].includes(phrase) + return isPhrase ? [insertAt(sql, token.endOffset, " /* c */")] : [] + }) +} + +const mutations: Record = { + truncations, + removeCloser, + insertOpener, + mismatchCloser, + semicolonInsideParens, + foreignTokens, + commentInsidePhrase, +} + +const failuresFor = (mutate: Mutation, seed: number) => { + const random = createRandom(seed) + const failures: string[] = [] + for (const query of docsQueries) { + for (const input of mutate(query, random)) { + try { + const output = format(input) + assertSameStream(input, output) + assertSameOperatorAdjacency(input, output) + assertIdempotent(input) + } catch (error) { + failures.push( + `${JSON.stringify(input)}\n${error instanceof Error ? error.message : String(error)}`, + ) + } + } + } + return failures +} + +describe("malformed input", () => { + it.each( + Object.entries(mutations).map( + ([name, mutate], index) => [name, mutate, index] as const, + ), + )("%s never throws and preserves the stream", (_name, mutate, index) => { + const failures = failuresFor(mutate, 1000 + index) + expect(failures.slice(0, 5)).toEqual([]) + expect(failures).toHaveLength(0) + }) +}) diff --git a/tests/formatter/oracles.ts b/tests/formatter/oracles.ts new file mode 100644 index 0000000..c45a5df --- /dev/null +++ b/tests/formatter/oracles.ts @@ -0,0 +1,57 @@ +import { expect } from "vitest" +import { parseToAst, toSql } from "../../src/index" +import { format, FormatOptions } from "../../src/formatter/index" +import { scan, StreamToken } from "../../src/formatter/lexer" + +const significant = (sql: string): StreamToken[] => + scan(sql).filter((token) => token.kind !== "whitespace") + +export const streamSignature = (sql: string): string[] => + significant(sql).map((token) => `${token.kind}:${token.image}`) + +export const assertSameStream = (input: string, output: string) => { + expect(streamSignature(output)).toEqual(streamSignature(input)) +} + +const isOperator = (token: StreamToken) => token.kind === "operator" + +const isUncertain = (token: StreamToken) => + token.kind === "opaque" || token.kind === "tolerant" + +const adjacencyPairs = (sql: string) => + significant(sql).flatMap((token, index, tokens) => { + const next = tokens[index + 1] + if (next === undefined) return [] + const guarded = + (isOperator(token) && isOperator(next)) || + isUncertain(token) || + isUncertain(next) + if (!guarded) return [] + return [ + `${token.image}${next.startOffset > token.endOffset ? " " : ""}${next.image}`, + ] + }) + +export const assertSameOperatorAdjacency = (input: string, output: string) => { + expect(adjacencyPairs(output)).toEqual(adjacencyPairs(input)) +} + +export const assertIdempotent = (sql: string, options?: FormatOptions) => { + const once = format(sql, options) + expect(format(once, options)).toBe(once) +} + +const parsesCleanly = (sql: string) => parseToAst(sql).errors.length === 0 + +export const assertSameAst = (input: string, output: string) => { + expect(parsesCleanly(output)).toBe(true) + expect(toSql(parseToAst(output).ast)).toBe(toSql(parseToAst(input).ast)) +} + +export const assertPreserved = (input: string, options?: FormatOptions) => { + const output = format(input, options) + assertSameStream(input, output) + assertSameOperatorAdjacency(input, output) + assertIdempotent(input, options) + if (parsesCleanly(input)) assertSameAst(input, output) +} diff --git a/tests/formatter/performance.test.ts b/tests/formatter/performance.test.ts new file mode 100644 index 0000000..85da65b --- /dev/null +++ b/tests/formatter/performance.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest" +import * as fs from "fs" +import * as path from "path" +import { format } from "../../src/formatter/index" + +/** + * A wall-clock budget does not survive a shared CI runner, where this file + * runs beside suites that saturate the machine. These tests check the shape + * of the cost instead: work must grow with the input, not with its square. A + * quadratic regression fails here, a slow machine does not. + * + * Measured on 2026-09-18: ten times the script costs 7.5 to 9.4 times the + * work, and four times the nesting costs 2.4 times the work. + */ +const GROWTH_LIMIT = 25 +const DEPTH_LIMIT = 12 +/** Only catches a hang; a loaded runner is an order of magnitude under this. */ +const HANG_LIMIT_MS = 5000 + +const docsQueries: string[] = ( + JSON.parse( + fs.readFileSync( + path.join(__dirname, "..", "fixtures", "docs-queries.json"), + "utf-8", + ), + ) as Array<{ query: string }> +).map((entry) => entry.query) + +const buildScript = (lines: number) => { + let script = "" + let index = 0 + while (script.split("\n").length < lines) { + script += docsQueries[index++ % docsQueries.length] + ";\n" + } + return script +} + +const buildNested = (depth: number) => { + let inner = "t" + for (let level = 0; level < depth; level++) { + inner = `(SELECT * FROM ${inner} WHERE x = ${level})` + } + return `SELECT * FROM ${inner}` +} + +const medianDuration = (sql: string) => { + format(sql) + const durations: number[] = [] + for (let run = 0; run < 9; run++) { + const start = performance.now() + format(sql) + durations.push(performance.now() - start) + } + return durations.sort((a, b) => a - b)[4] +} + +describe("formatter performance", () => { + it("costs grow with the size of a script, not with its square", () => { + // Given + const small = medianDuration(buildScript(500)) + const large = medianDuration(buildScript(5000)) + + // Then + expect(large / small).toBeLessThan(GROWTH_LIMIT) + expect(large).toBeLessThan(HANG_LIMIT_MS) + }) + + it("costs grow with nesting depth, not with its square", () => { + // Given + const shallow = medianDuration(buildNested(50)) + const deep = medianDuration(buildNested(200)) + + // Then + expect(deep / shallow).toBeLessThan(DEPTH_LIMIT) + expect(deep).toBeLessThan(HANG_LIMIT_MS) + }) +}) diff --git a/tests/formatter/phrases.test.ts b/tests/formatter/phrases.test.ts new file mode 100644 index 0000000..70a8bac --- /dev/null +++ b/tests/formatter/phrases.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest" +import { scan, StreamToken } from "../../src/formatter/lexer" +import { clausePhrases, matchPhrase } from "../../src/formatter/phrases" + +const indexOfImage = (tokens: StreamToken[], image: string) => + tokens.findIndex((token) => token.image.toUpperCase() === image) + +const matchAt = ( + sql: string, + image: string, + kind: keyof typeof clausePhrases = "select", +) => { + const tokens = scan(sql) + const match = matchPhrase( + tokens, + indexOfImage(tokens, image), + clausePhrases[kind], + ) + return match === null ? null : match.phrase.names.join(" ") +} + +describe("matchPhrase", () => { + it("matches a two-word phrase across one space", () => { + expect(matchAt("SELECT * FROM t LATEST ON ts", "LATEST")).toBe("Latest On") + }) + + it("matches across several spaces and a newline", () => { + expect(matchAt("SELECT * FROM t LATEST \n ON ts", "LATEST")).toBe( + "Latest On", + ) + }) + + it("matches regardless of keyword case", () => { + expect(matchAt("select * from t latest on ts", "LATEST")).toBe("Latest On") + }) + + it("prefers the longest phrase", () => { + expect(matchAt("SELECT 1 UNION ALL SELECT 2", "UNION")).toBe("Union All") + expect(matchAt("SELECT 1 UNION SELECT 2", "UNION")).toBe("Union") + expect(matchAt("SELECT * FROM a LEFT OUTER JOIN b ON x", "LEFT")).toBe( + "Left Outer Join", + ) + }) + + it("stops at a comment between phrase words", () => { + expect(matchAt("SELECT * FROM t LATEST /* x */ ON ts", "LATEST")).toBeNull() + expect(matchAt("SELECT * FROM t LATEST -- x\n ON ts", "LATEST")).toBeNull() + }) + + it("stops at an opaque token between phrase words", () => { + expect(matchAt("SELECT * FROM t LATEST $ ON ts", "LATEST")).toBeNull() + }) + + it("does not match a keyword-shaped identifier as a clause", () => { + // Given + const tokens = scan("SELECT status, type FROM t") + + // When / Then + expect( + matchPhrase(tokens, indexOfImage(tokens, "STATUS"), clausePhrases.select), + ).toBeNull() + expect( + matchPhrase(tokens, indexOfImage(tokens, "TYPE"), clausePhrases.select), + ).toBeNull() + }) + + it("keeps table options inline and matches DEDUP only for CREATE kinds", () => { + // Given + const sql = + "CREATE TABLE t (a INT) TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts)" + + // When / Then + expect(matchAt(sql, "TIMESTAMP", "createTable")).toBeNull() + expect(matchAt(sql, "PARTITION", "createTable")).toBeNull() + expect(matchAt(sql, "WAL", "createTable")).toBeNull() + expect(matchAt(sql, "DEDUP", "createTable")).toBe("Dedup") + expect(matchAt(sql, "DEDUP", "select")).toBeNull() + }) + + it("matches alter actions after the table name", () => { + expect( + matchAt("ALTER TABLE t ADD COLUMN v SYMBOL", "ADD", "alterTable"), + ).toBe("Add Column") + expect( + matchAt("ALTER TABLE t ALTER COLUMN v ADD INDEX", "ALTER", "alterTable"), + ).toBe("Alter Table") + }) + + it("has no phrases for other statements", () => { + expect(matchAt("DROP TABLE t", "DROP", "other")).toBeNull() + }) + + it("returns the index after the last matched token", () => { + // Given + const tokens = scan("SELECT * FROM t SAMPLE BY 1h") + + // When + const match = matchPhrase( + tokens, + indexOfImage(tokens, "SAMPLE"), + clausePhrases.select, + ) + + // Then + expect(tokens[match!.endIndex].kind).toBe("whitespace") + expect(tokens[match!.endIndex + 1].image).toBe("1h") + }) +}) diff --git a/tests/formatter/statements.test.ts b/tests/formatter/statements.test.ts new file mode 100644 index 0000000..e04e891 --- /dev/null +++ b/tests/formatter/statements.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest" +import { reconstruct, scan } from "../../src/formatter/lexer" +import { splitStatements } from "../../src/formatter/statements" +import { detectStatementKind } from "../../src/formatter/context" + +const split = (sql: string) => splitStatements(scan(sql)) + +const texts = (sql: string) => split(sql).map((s) => reconstruct(s.tokens)) + +const verbatimText = (sql: string) => { + const [statement] = split(sql) + return statement.verbatimFrom === null + ? null + : reconstruct(statement.tokens.slice(statement.verbatimFrom)) +} + +describe("splitStatements", () => { + it("splits at semicolons outside delimiters and strings", () => { + // Given + const sql = "SELECT (a; b) FROM t; SELECT ';' FROM u; SELECT c" + + // When / Then + expect(texts(sql)).toEqual([ + "SELECT (a; b) FROM t;", + " SELECT ';' FROM u;", + " SELECT c", + ]) + }) + + it("drops trailing whitespace after the last semicolon", () => { + // When / Then + expect(texts("SELECT a;\n\n")).toEqual(["SELECT a;"]) + }) + + it("keeps a trailing comment as its own statement", () => { + // When / Then + expect(texts("SELECT a; -- done")).toEqual(["SELECT a;", " -- done"]) + }) + + it("marks no verbatim region for balanced input", () => { + // When / Then + expect(split("SELECT (a) FROM t")[0].verbatimFrom).toBeNull() + }) + + it("preserves from an opener that never closes, including later semicolons", () => { + // Given + const sql = "SELECT (a; SELECT b;" + + // When + const statements = split(sql) + + // Then + expect(statements).toHaveLength(1) + expect(verbatimText(sql)).toBe("(a; SELECT b;") + }) + + it("preserves from the outermost open delimiter on a mismatch", () => { + // Given + const sql = "SELECT f([)] FROM t; SELECT b" + + // When + const statements = split(sql) + + // Then + expect(statements).toHaveLength(1) + expect(verbatimText(sql)).toBe("([)] FROM t; SELECT b") + }) + + it("preserves from the outermost unclosed opener with several open", () => { + // When / Then + expect(verbatimText("SELECT (a, (b FROM t")).toBe("(a, (b FROM t") + }) + + it("treats a stray closer as an ordinary token", () => { + // Given + const sql = "SELECT a) FROM t; SELECT b" + + // When + const statements = split(sql) + + // Then + expect(statements).toHaveLength(2) + expect(statements[0].verbatimFrom).toBeNull() + }) + + it("preserves from a tolerant token and swallows later statements", () => { + // Given + const sql = "SELECT a FROM t WHERE x = 'oops; SELECT b;" + + // When + const statements = split(sql) + + // Then + expect(statements).toHaveLength(1) + expect(verbatimText(sql)).toBe("'oops; SELECT b;") + }) + + it("reconstructs the input from all statements", () => { + // Given + const sql = "SELECT a; /* c */ SELECT (b; SELECT 'x" + + // When / Then + expect(texts(sql).join("")).toBe(sql) + }) +}) + +describe("detectStatementKind", () => { + it.each([ + ["SELECT a FROM t", "select"], + ["select a from t", "select"], + ["trades WHERE x = 1", "select"], + ['"my table" WHERE x = 1', "select"], + ["WITH c AS (SELECT 1) SELECT * FROM c", "select"], + ["DECLARE @x := 1 SELECT @x", "select"], + ["(SELECT a FROM t) UNION (SELECT b FROM u)", "select"], + ["INSERT INTO t VALUES (1)", "insert"], + ["UPDATE t SET a = 1", "update"], + ["CREATE TABLE t (a INT)", "createTable"], + ["CREATE MATERIALIZED VIEW mv AS SELECT 1", "createMaterializedView"], + ["CREATE LIVE VIEW lv AS SELECT 1", "createLiveView"], + ["ALTER TABLE t ADD COLUMN a INT", "alterTable"], + ["ALTER MATERIALIZED VIEW mv SET TTL 1d", "alterMaterializedView"], + [" -- c\n SELECT 1", "select"], + ["EXPLAIN SELECT 1", "select"], + ["EXPLAIN INSERT INTO t VALUES (1)", "insert"], + ["DROP TABLE t", "other"], + ["CREATE USER u", "other"], + ["", "other"], + ])("detects %s as %s", (sql, kind) => { + expect(detectStatementKind(scan(sql))).toBe(kind) + }) +}) diff --git a/tests/parser.test.ts b/tests/parser.test.ts index 1fb158a..7fe2247 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -8622,3 +8622,160 @@ orders PIVOT (sum(amount) FOR status IN ('open'))` } }) }) + +describe("Docs gaps (2026-09): memory limits, outer joins, unnest joins, fill prev column, parquet bloom filters, live view owner", () => { + const roundtrip = (sql: string) => { + const result = parseToAst(sql) + expect(result.errors).toHaveLength(0) + const regenerated = toSql(result.ast[0]) + const reparsed = parseToAst(regenerated) + expect(reparsed.errors).toHaveLength(0) + expect(toSql(reparsed.ast[0])).toBe(regenerated) + return { ast: result.ast[0], regenerated } + } + + describe("SET MEMORY LIMIT", () => { + it.each([ + ["ALTER USER john SET MEMORY LIMIT 512M", "512M"], + ["ALTER USER tenant_a SET MEMORY LIMIT 1G", "1G"], + ["ALTER USER john SET MEMORY LIMIT 0", "0"], + ["ALTER USER john SET MEMORY LIMIT UNLIMITED", "UNLIMITED"], + ])("parses %s", (sql, limit) => { + const { ast } = roundtrip(sql) + expect(ast.type).toBe("alterUser") + if (ast.type === "alterUser") { + expect(ast.action).toEqual({ actionType: "setMemoryLimit", limit }) + } + }) + + it("parses ALTER SERVICE ACCOUNT ... SET MEMORY LIMIT", () => { + const { ast, regenerated } = roundtrip( + "ALTER SERVICE ACCOUNT client_app SET MEMORY LIMIT 1G", + ) + expect(ast.type).toBe("alterServiceAccount") + expect(regenerated).toBe( + "ALTER SERVICE ACCOUNT client_app SET MEMORY LIMIT 1G", + ) + }) + + it("parses ALTER GROUP ... SET MEMORY LIMIT and keeps alias actions", () => { + const { ast } = roundtrip( + "ALTER GROUP analysts SET MEMORY LIMIT UNLIMITED", + ) + if (ast.type === "alterGroup") { + expect(ast.action).toBe("setMemoryLimit") + expect(ast.memoryLimit).toBe("UNLIMITED") + } + const alias = roundtrip("ALTER GROUP analysts WITH EXTERNAL ALIAS 'ext'") + if (alias.ast.type === "alterGroup") { + expect(alias.ast.action).toBe("setAlias") + expect(alias.ast.externalAlias).toBe("ext") + } + }) + }) + + describe("RIGHT and FULL OUTER JOIN", () => { + it.each([ + ["SELECT * FROM a RIGHT JOIN b ON a.x = b.x", "right", false], + ["SELECT * FROM a RIGHT OUTER JOIN b ON a.x = b.x", "right", true], + ["SELECT * FROM a FULL JOIN b ON a.x = b.x", "full", false], + ["SELECT * FROM a FULL OUTER JOIN b ON a.x = b.x", "full", true], + ])("parses %s", (sql, joinType, outer) => { + const { ast, regenerated } = roundtrip(sql) + if (ast.type === "select") { + const join = ast.from?.[0].joins?.[0] + expect(join?.joinType).toBe(joinType) + expect(!!join?.outer).toBe(outer) + } + expect(regenerated).toBe(sql) + }) + + it("parses the docs FULL OUTER JOIN with CTEs", () => { + roundtrip( + "WITH may_trades AS (SELECT symbol, COUNT(*) AS may_total FROM trades WHERE timestamp IN '2024-05'), june_trades AS (SELECT symbol, COUNT(*) AS june_total FROM trades WHERE timestamp IN '2024-06') SELECT COALESCE(may_trades.symbol, june_trades.symbol) AS symbol, may_total, june_total FROM may_trades FULL OUTER JOIN june_trades ON may_trades.symbol = june_trades.symbol", + ) + }) + }) + + describe("UNNEST after JOIN", () => { + it("parses CROSS JOIN UNNEST with a column alias list", () => { + const { ast } = roundtrip( + "SELECT t.symbol, u.vol FROM market_data t CROSS JOIN UNNEST(t.asks[2]) u(vol) WHERE t.symbol = 'EURUSD'", + ) + if (ast.type === "select") { + const join = ast.from?.[0].joins?.[0] + expect(join?.joinType).toBe("cross") + expect(join?.table.alias).toBe("u") + expect(join?.table.columnAliases).toEqual(["vol"]) + expect((join?.table.table as { type: string }).type).toBe("unnest") + } + }) + + it("still parses plain JOIN with a table", () => { + const { ast } = roundtrip("SELECT * FROM a JOIN b ON a.x = b.x") + if (ast.type === "select") { + expect(ast.from?.[0].joins?.[0].table.table).toEqual({ + type: "qualifiedName", + parts: ["b"], + }) + } + }) + }) + + describe("FILL with a column reference", () => { + it("parses FILL(PREV(other_column), PREV)", () => { + const { ast, regenerated } = roundtrip( + "SELECT timestamp, symbol, avg(bid_price) AS bid_price, avg(ask_price) AS ask_price FROM core_price WHERE symbol = 'EURUSD' SAMPLE BY 100T FILL(PREV(ask_price), PREV)", + ) + if (ast.type === "select") { + expect(ast.sampleBy?.fill).toEqual(["PREV(ask_price)", "PREV"]) + } + expect(regenerated).toContain("FILL(PREV(ask_price), PREV)") + }) + }) + + describe("CONVERT PARTITION ... WITH options", () => { + it("parses bloom filter options after WHERE", () => { + const sql = + "ALTER TABLE trades CONVERT PARTITION TO PARQUET WHERE timestamp < '2025-08-31' WITH (bloom_filter_columns = 'symbol,side', bloom_filter_fpp = 0.01)" + const { ast, regenerated } = roundtrip(sql) + if ( + ast.type === "alterTable" && + ast.action.actionType === "convertPartition" + ) { + expect(ast.action.withParams?.map((p) => p.name)).toEqual([ + "bloom_filter_columns", + "bloom_filter_fpp", + ]) + expect(ast.action.where).toBeDefined() + } + expect(regenerated).toBe(sql) + }) + + it("parses WITH options without WHERE", () => { + roundtrip( + "ALTER TABLE trades CONVERT PARTITION TO PARQUET WITH (bloom_filter_columns = 'symbol')", + ) + }) + }) + + describe("CREATE LIVE VIEW ... OWNED BY", () => { + it("parses OWNED BY after a bare SELECT query", () => { + const { ast, regenerated } = roundtrip( + "CREATE LIVE VIEW trades_ma FLUSH EVERY 1s START FROM NOW AS SELECT timestamp, symbol, avg(price) OVER (PARTITION BY symbol ORDER BY timestamp ROWS 300 PRECEDING) AS moving_avg FROM trades OWNED BY analysts", + ) + if (ast.type === "createLiveView") { + expect(ast.ownedBy).toBe("analysts") + expect(ast.query.from?.[0].alias).toBeUndefined() + } + expect(regenerated.endsWith("OWNED BY analysts")).toBe(true) + }) + + it("parses OWNED BY after a parenthesized query", () => { + const { ast } = roundtrip( + "CREATE LIVE VIEW v FLUSH EVERY 1s AS (SELECT ts FROM trades) OWNED BY 'ops team'", + ) + if (ast.type === "createLiveView") expect(ast.ownedBy).toBe("ops team") + }) + }) +}) diff --git a/tsup.config.ts b/tsup.config.ts index a7a9256..19ee47f 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup" export default defineConfig({ - entry: ["src/index.ts", "src/grammar/index.ts"], + entry: ["src/index.ts", "src/grammar/index.ts", "src/formatter/index.ts"], format: ["esm", "cjs"], outDir: "dist", splitting: false,