add sql formatter and fix parser gaps

README.md

@@ -69,6 +69,33 @@ const statements = parseStatements(`
 console.log(statements.length); // 3
 ```
 
+### Format SQL
+
+```typescript
+import { format } from "@questdb/sql-parser/formatter";
+
+format("SELECT * FROM trades WHERE symbol = 'BTC-USD' LATEST ON ts PARTITION BY symbol");
+// SELECT *
+// FROM trades
+// WHERE symbol = 'BTC-USD'
+// LATEST ON ts PARTITION BY symbol
+```
+
+The formatter is token-based and never throws on SQL content. It preserves every token, comment, and unknown character, changes whitespace only, and leaves unterminated or unbalanced input verbatim from the point of the problem.
+
+Options: `indent` (default two spaces), `maxWidth` (default 50), and `capitalize` (default `false`).
+
+```typescript
+format(
+  "create table tab (s symbol index type bitmap, timestamp timestamp) timestamp(timestamp) partition by day wal",
+  { capitalize: true },
+);
+// CREATE TABLE tab (s SYMBOL INDEX TYPE BITMAP, timestamp TIMESTAMP)
+// TIMESTAMP(timestamp) PARTITION BY DAY WAL
+```
+
+With `capitalize`, the formatter parses the statement and raises only the words the grammar read as syntax, so a keyword that names a table, view or column keeps its case, as `timestamp` does above. Most QuestDB keywords are non-reserved and double as names, which is why this needs the grammar. SQL the parser cannot read comes back with its case untouched, and the layout is the same either way.
+
 ### Autocomplete
 
 ```typescript
@@ -183,6 +210,7 @@ The parser uses Chevrotain's [CST pattern](https://chevrotain.io/docs/guide/conc
 | `parseOne(sql)`        | Parse a single statement, throws on errors or multiple statements |
 | `parseStatements(sql)` | Parse multiple statements, throws on errors                       |
 | `toSql(ast)`           | Convert `Statement[]` back to a SQL string                        |
+| `format(sql, options)` | Format SQL text; tolerant of invalid input, whitespace-only edits |
 
 ### Low-Level API
 

package.json

@@ -27,6 +27,16 @@
         "types": "./dist/grammar/index.d.ts",
         "default": "./dist/grammar/index.cjs"
       }
+    },
+    "./formatter": {
+      "import": {
+        "types": "./dist/formatter/index.d.ts",
+        "default": "./dist/formatter/index.js"
+      },
+      "require": {
+        "types": "./dist/formatter/index.d.ts",
+        "default": "./dist/formatter/index.cjs"
+      }
     }
   },
   "packageManager": "yarn@4.9.1",
@@ -46,6 +56,7 @@
     "questdb",
     "sql",
     "parser",
+    "formatter",
     "chevrotain",
     "ast"
   ],

src/formatter/classification.ts

@@ -0,0 +1,81 @@
+import { IdentifierKeyword, keywordTokenArray } from "../parser/tokens"
+
+export type SignificantKind = "word" | "operator" | "delimiter"
+
+export const operatorTokenNames: ReadonlySet<string> = new Set([
+  "Star",
+  "Plus",
+  "Minus",
+  "Divide",
+  "Modulo",
+  "Equals",
+  "NotEquals",
+  "LessThan",
+  "GreaterThan",
+  "LessThanOrEqual",
+  "GreaterThanOrEqual",
+  "IPv4ContainedBy",
+  "IPv4ContainedByOrEqual",
+  "IPv4Contains",
+  "IPv4ContainsOrEqual",
+  "Concat",
+  "BitAnd",
+  "BitXor",
+  "BitOr",
+  "RegexMatch",
+  "RegexNotMatch",
+  "RegexNotEquals",
+  "DoubleColon",
+  "ColonEquals",
+  "Colon",
+  "AtSign",
+])
+
+export const delimiterTokenNames: ReadonlySet<string> = new Set([
+  "LParen",
+  "RParen",
+  "LBracket",
+  "RBracket",
+  "Comma",
+  "Semicolon",
+  "Dot",
+])
+
+const literalTokenNames = [
+  "GeohashBinaryLiteral",
+  "GeohashLiteral",
+  "LongLiteral",
+  "DecimalLiteral",
+  "DurationLiteral",
+  "NumberLiteral",
+  "StringLiteral",
+  "QuotedIdentifier",
+  "Identifier",
+  "VariableReference",
+]
+
+export const wordTokenNames: ReadonlySet<string> = new Set([
+  ...keywordTokenArray.map((token) => token.name),
+  ...literalTokenNames,
+])
+
+const nonReservedKeywordNames = keywordTokenArray
+  .filter((token) => token.CATEGORIES?.includes(IdentifierKeyword))
+  .map((token) => token.name)
+
+export const operandTokenNames: ReadonlySet<string> = new Set([
+  ...literalTokenNames,
+  ...nonReservedKeywordNames,
+  "Null",
+  "True",
+  "False",
+  "NaN",
+  "RParen",
+  "RBracket",
+])
+
+export const classify = (tokenName: string): SignificantKind => {
+  if (operatorTokenNames.has(tokenName)) return "operator"
+  if (delimiterTokenNames.has(tokenName)) return "delimiter"
+  return "word"
+}

src/formatter/context.ts

@@ -0,0 +1,50 @@
+import { StreamToken } from "./lexer"
+
+export type StatementKind =
+  | "select"
+  | "insert"
+  | "update"
+  | "createTable"
+  | "createMaterializedView"
+  | "createLiveView"
+  | "alterTable"
+  | "alterMaterializedView"
+  | "other"
+
+const selectStarters: ReadonlySet<string> = new Set([
+  "Select",
+  "With",
+  "Declare",
+  "Identifier",
+  "QuotedIdentifier",
+  "LParen",
+])
+
+export const isSignificant = (token: StreamToken) =>
+  token.kind === "word" ||
+  token.kind === "operator" ||
+  token.kind === "delimiter"
+
+export const isExplainPrefix = (token: StreamToken) =>
+  token.tokenName === "Explain"
+
+export const detectStatementKind = (tokens: StreamToken[]): StatementKind => {
+  const names = tokens.filter(isSignificant).map((token) => token.tokenName)
+  if (names[0] === "Explain") names.shift()
+  const [first, second] = names
+
+  if (first === undefined || first === null) return "other"
+  if (selectStarters.has(first)) return "select"
+  if (first === "Insert") return "insert"
+  if (first === "Update") return "update"
+  if (first === "Create") {
+    if (second === "Table") return "createTable"
+    if (second === "Materialized") return "createMaterializedView"
+    if (second === "Live") return "createLiveView"
+  }
+  if (first === "Alter") {
+    if (second === "Table") return "alterTable"
+    if (second === "Materialized") return "alterMaterializedView"
+  }
+  return "other"
+}

src/formatter/doc.ts

@@ -0,0 +1,168 @@
+export type Doc =
+  | { type: "text"; text: string }
+  | { type: "gap"; text: string }
+  | { type: "line"; flat: string }
+  | { type: "hardline" }
+  | { type: "concat"; parts: Doc[] }
+  | { type: "group"; doc: Doc }
+  | { type: "indent"; doc: Doc }
+  | { type: "verbatim"; text: string }
+
+export const text = (value: string): Doc => ({ type: "text", text: value })
+export const gap = (value: string): Doc => ({ type: "gap", text: value })
+export const line = (flat: string): Doc => ({ type: "line", flat })
+export const hardline: Doc = { type: "hardline" }
+export const concat = (parts: Doc[]): Doc => ({ type: "concat", parts })
+export const group = (doc: Doc): Doc => ({ type: "group", doc })
+export const indent = (doc: Doc): Doc => ({ type: "indent", doc })
+export const verbatim = (value: string): Doc => ({
+  type: "verbatim",
+  text: value,
+})
+
+export const asLine = (leading: Doc): Doc =>
+  leading.type === "gap" ? line(leading.text) : leading
+
+export type PrintOptions = {
+  indent: string
+  maxWidth: number
+}
+
+type Mode = "flat" | "break"
+
+type Command = { level: number; mode: Mode; doc: Doc }
+
+const firstLineLength = (value: string) => {
+  const newline = value.indexOf("\n")
+  return newline === -1 ? value.length : newline
+}
+
+const fits = (next: Command, rest: Command[], width: number): boolean => {
+  let remaining = width
+  let restIndex = rest.length - 1
+  const stack: Command[] = [next]
+
+  while (remaining >= 0) {
+    const command = stack.pop()
+    if (command === undefined) {
+      if (restIndex < 0) return true
+      stack.push(rest[restIndex--])
+      continue
+    }
+    const { level, mode, doc } = command
+    switch (doc.type) {
+      case "text":
+      case "gap":
+        remaining -= doc.text.length
+        break
+      case "line":
+        if (mode === "break") return true
+        remaining -= doc.flat.length
+        break
+      case "hardline":
+        return mode === "break"
+      case "verbatim":
+        remaining -= firstLineLength(doc.text)
+        if (doc.text.includes("\n")) return remaining >= 0
+        break
+      case "concat":
+        for (let i = doc.parts.length - 1; i >= 0; i--) {
+          stack.push({ level, mode, doc: doc.parts[i] })
+        }
+        break
+      case "group":
+      case "indent":
+        stack.push({ level, mode, doc: doc.doc })
+        break
+    }
+  }
+  return false
+}
+
+export const printDoc = (doc: Doc, options: PrintOptions): string => {
+  const out: string[] = []
+  let column = 0
+  let pendingGap = ""
+  let lineHasContent = false
+  let indentIndex = -1
+  const commands: Command[] = [{ level: 0, mode: "break", doc }]
+
+  const emit = (value: string) => {
+    if (pendingGap !== "") {
+      out.push(pendingGap)
+      column += pendingGap.length
+      pendingGap = ""
+    }
+    out.push(value)
+    lineHasContent = true
+  }
+
+  const newline = (level: number) => {
+    const indentation = options.indent.repeat(level)
+    pendingGap = ""
+    if (!lineHasContent) {
+      if (indentIndex >= 0) out[indentIndex] = indentation
+      column = indentation.length
+      return
+    }
+    out.push("\n")
+    indentIndex = out.push(indentation) - 1
+    column = indentation.length
+    lineHasContent = false
+  }
+
+  while (commands.length > 0) {
+    const command = commands.pop()!
+    const { level, mode, doc: current } = command
+    switch (current.type) {
+      case "text":
+        emit(current.text)
+        column += current.text.length
+        break
+      case "gap":
+        if (lineHasContent) pendingGap = current.text
+        break
+      case "line":
+        if (mode === "flat") pendingGap = current.flat
+        else newline(level)
+        break
+      case "hardline":
+        newline(level)
+        break
+      case "verbatim": {
+        emit(current.text)
+        const lastNewline = current.text.lastIndexOf("\n")
+        column =
+          lastNewline === -1
+            ? column + current.text.length
+            : current.text.length - lastNewline - 1
+        break
+      }
+      case "concat":
+        for (let i = current.parts.length - 1; i >= 0; i--) {
+          commands.push({ level, mode, doc: current.parts[i] })
+        }
+        break
+      case "indent":
+        commands.push({ level: level + 1, mode, doc: current.doc })
+        break
+      case "group": {
+        if (mode === "flat") {
+          commands.push({ level, mode: "flat", doc: current.doc })
+          break
+        }
+        const flat: Command = { level, mode: "flat", doc: current.doc }
+        const width = options.maxWidth - column - pendingGap.length
+        commands.push(
+          fits(flat, commands, width)
+            ? flat
+            : { level, mode: "break", doc: current.doc },
+        )
+        break
+      }
+    }
+  }
+
+  if (!lineHasContent && indentIndex >= 0) out.length = indentIndex - 1
+  return out.join("")
+}