From 0e82ffd21a710fbb2b3c33488ea10de7087e9672 Mon Sep 17 00:00:00 2001 From: Graham Ollis Date: Thu, 24 Sep 2026 22:10:10 -0600 Subject: [PATCH 1/5] starter --- lib/SQL/AST/Simple.pm | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/SQL/AST/Simple.pm b/lib/SQL/AST/Simple.pm index c119def..a0c6c43 100644 --- a/lib/SQL/AST/Simple.pm +++ b/lib/SQL/AST/Simple.pm @@ -1,7 +1,11 @@ -use warnings; -use v5.42; +package SQL::AST::Simple; -package SQL::AST::Simple { +use 5.042; +use warnings; +use FFI::Platypus 2.00; - # ABSTRACT: Round trip parser and AST for SQL -} +my $ffi = FFI::Platypus->new( + api => 2, + lang => 'Rust', +); +$ffi->bundle; From cc2f8ec0e744bf003666d84fe567d5d9708adef8 Mon Sep 17 00:00:00 2001 From: Graham Ollis Date: Thu, 24 Sep 2026 22:10:37 -0600 Subject: [PATCH 2/5] update ci --- .github/workflows/linux.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ea8cb20..206633c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,18 +17,9 @@ jobs: fail-fast: false matrix: cip_tag: - - "5.41" - - "5.40" - - "5.38" - - "5.36" - - "5.34" - - "5.32" - - "5.30" - - "5.28" - - "5.26" - - "5.24" - - "5.22" - - "5.20" + - "5.45" + - "5.44" + - "5.42" env: CIP_TAG: ${{ matrix.cip_tag }} From 0f153fac3b3af8ca53351c6a647a51ad4ec7e07d Mon Sep 17 00:00:00 2001 From: Graham Ollis Date: Thu, 24 Sep 2026 22:42:53 -0600 Subject: [PATCH 3/5] Implement parse and unparse via bundled sqlparser crate Bundle a small Rust cdylib around the sqlparser crate (serde feature) exposing parse, unparse and free with a C ABI, built at install time with FFI::Build::File::Cargo. The Perl side exports parse(), which returns the AST as plain Perl data decoded from JSON, and unparse(), which takes the same structure back and returns SQL text. Co-Authored-By: Claude Fable 5.1 --- .gitignore | 3 + README.md | 138 +++++++++++++++++++++++++++++ author.yml | 7 +- dist.ini | 5 +- ffi/Cargo.toml | 15 ++++ ffi/src/lib.rs | 94 ++++++++++++++++++++ lib/SQL/AST/Simple.pm | 201 ++++++++++++++++++++++++++++++++++++++++++ t/00_diag.t | 94 ++++++++++++++++++++ t/sql_ast_simple.t | 62 ++++++++++++- 9 files changed, 615 insertions(+), 4 deletions(-) create mode 100644 README.md create mode 100644 ffi/Cargo.toml create mode 100644 ffi/src/lib.rs create mode 100644 t/00_diag.t diff --git a/.gitignore b/.gitignore index ff44331..763eb7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ SQL-AST-Simple-* /.build/ *.swp +/ffi/target/ +/ffi/_build/ +/ffi/Cargo.lock diff --git a/README.md b/README.md new file mode 100644 index 0000000..a38dc24 --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# SQL::AST::Simple ![static](https://github.com/uperl/SQL-AST-Simple/workflows/static/badge.svg) ![linux](https://github.com/uperl/SQL-AST-Simple/workflows/linux/badge.svg) + +Parse SQL into a plain Perl data structure and back again + +# SYNOPSIS + +```perl +use SQL::AST::Simple qw( parse unparse ); + +my $ast = parse('SELECT a, b FROM t WHERE a > 1', dialect => 'postgresql'); + +# $ast is an array reference of statements, each a tree of plain +# hashes, arrays and scalars. Poke at it however you like: +$ast->[0]{Query}{body}{Select}{from}[0]{relation}{Table}{name}[0]{Identifier}{value} = 'u'; + +say unparse($ast); # SELECT a, b FROM u WHERE a > 1 +``` + +# DESCRIPTION + +This module bundles the Rust +[sqlparser](https://github.com/apache/datafusion-sqlparser-rs) crate and +exposes exactly two operations: turning SQL text into the parser's abstract +syntax tree as an ordinary Perl data structure, and turning such a data +structure back into SQL text. There is no object layer; the tree is what +the crate's serde serialization produces, decoded from JSON. That keeps +the module small and makes every node the crate knows about available +without any wrapping, at the cost of a somewhat verbose structure. + +Nothing is exported by default. + +# FUNCTIONS + +## parse + +```perl +my $ast = parse($sql); +my $ast = parse($sql, dialect => $name); +``` + +Parses `$sql`, which may contain several semicolon separated statements, +and returns an array reference with one element per statement. Throws an +exception with the parser's message, including line and column, if the +text cannot be parsed. + +Options: + +- dialect + + Which SQL dialect to parse with. Defaults to `generic`, which is the + most permissive. Recognized names (case insensitive) are `generic`, + `ansi`, `postgresql` (or `postgres`), `mysql`, `sqlite`, `mssql`, + `oracle`, `snowflake`, `bigquery`, `redshift`, `clickhouse`, + `duckdb`, `databricks`, `hive`, `spark` (or `sparksql`) and + `teradata`. + +## unparse + +```perl +my $sql = unparse($ast); +my $sql = unparse($ast, pretty => 1); +``` + +Takes an array reference of statements as returned by ["parse"](#parse), or a +single statement hash reference, and returns the SQL text. Multiple +statements are joined with `"; "`. Throws an exception if the structure +does not deserialize into a valid AST. + +Options: + +- pretty + + If true, statements are formatted with indentation and newlines rather + than on a single line, and are joined with `";\n"`. + +# THE DATA STRUCTURE + +The tree mirrors the Rust types of the `sqlparser` crate one to one, as +serialized by serde. A few rules of thumb cover most of it: + +- Rust enums are "externally tagged": a hash with a single key naming the +variant, whose value is the payload. A `SELECT` statement is +`{ Query => {...} }`, a column reference in an expression is +`{ Identifier => {...} }`, a literal is `{ Value => {...} }`. +Variants without payload are plain strings. +- Rust structs are hashes keyed by field name; `Option` fields that are +absent are `undef`; `Vec` fields are array references. +- Booleans come back as JSON boolean objects. When you set a boolean field +yourself use `\1` or `\0` (or the `true`/`false` constants from your +JSON module). A plain Perl `1` would be encoded as a number and rejected +by ["unparse"](#unparse). +- Numeric literals are kept as strings, exactly as they appeared in the +source, so that precision is never lost. Any field that holds a string +must be given a Perl string; if you have computed a number, stringify it +first. +- Most nodes carry a `span` hash recording where they appeared in the +source. ["unparse"](#unparse) ignores the contents but requires the field to be +present, so the easiest way to build a new node is to parse a small +snippet and lift the piece you need out of the result, rather than +constructing hashes by hand. + +The easiest way to learn the shape for a given construct is to parse an +example and dump it. The exact shape depends on the bundled crate +version, which is pinned in the distribution's `ffi/Cargo.toml`; a +release that bumps it may change the structure and will say so in the +change log. + +# CAVEATS + +The parser is syntactic only and deliberately permissive. It will accept +some SQL that a given database would reject, and occasionally reject +vendor syntax it does not yet know. Round tripping is not byte for byte: +comments are dropped, keywords are upper cased, and whitespace is +normalized. + +Building this distribution requires a Rust toolchain (`cargo`) at +install time. + +# SEE ALSO + +- [https://github.com/apache/datafusion-sqlparser-rs](https://github.com/apache/datafusion-sqlparser-rs) + + The parser this module wraps. + +- [FFI::Platypus::Lang::Rust](https://metacpan.org/pod/FFI::Platypus::Lang::Rust) + + How the Rust code is bundled and called. + +# AUTHOR + +Graham Ollis + +# COPYRIGHT AND LICENSE + +This software is copyright (c) 2026 by Graham Ollis. + +This is free software; you can redistribute it and/or modify it under +the same terms as the Perl 5 programming language system itself. diff --git a/author.yml b/author.yml index c64a272..a2a318a 100644 --- a/author.yml +++ b/author.yml @@ -5,7 +5,12 @@ pod_spelling_system: # (regardless of what spell check thinks) # or stuff that I like to spell incorrectly # intentionally - stopwords: [] + stopwords: + - AST + - serde + - sqlparser + - unparse + - postgresql pod_coverage: skip: 0 diff --git a/dist.ini b/dist.ini index 8550fb9..6dba53b 100644 --- a/dist.ini +++ b/dist.ini @@ -14,11 +14,14 @@ default_branch = main test2_v0 = 1 workflow = static workflow = linux -version_plugin = PkgVersion::Block irc = irc://irc.perl.org/#native [Author::Plicease::Core] +[FFI::Build] +lang = Rust +build = Cargo + [Author::Plicease::Upload] cpan = 1 diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml new file mode 100644 index 0000000..ffd8e2c --- /dev/null +++ b/ffi/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sql_ast_simple" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# The JSON shape of the AST is this crate's serde output, so the minor +# version is effectively part of the Perl API. Keep it pinned to a +# single minor release and bump deliberately. +sqlparser = { version = "0.63", features = ["serde"] } +serde_json = "1.0" diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs new file mode 100644 index 0000000..44d323a --- /dev/null +++ b/ffi/src/lib.rs @@ -0,0 +1,94 @@ +//! C ABI shim around the `sqlparser` crate for SQL::AST::Simple. +//! +//! Two operations are exported: parse SQL text into the serde JSON form of +//! the `sqlparser` AST, and turn that JSON back into SQL text. Every string +//! returned to the caller is a NUL terminated, heap allocated C string that +//! must be released with `sql_ast_simple_free`. + +use sqlparser::ast::Statement; +use sqlparser::dialect::dialect_from_str; +use sqlparser::parser::Parser; +use std::ffi::{c_char, CStr, CString}; +use std::ptr; + +/// Borrow a C string as `&str`, rejecting NULL and invalid UTF-8. +unsafe fn borrow<'a>(p: *const c_char) -> Result<&'a str, String> { + if p.is_null() { + return Err("NULL string argument".to_string()); + } + CStr::from_ptr(p) + .to_str() + .map_err(|e| format!("argument is not valid UTF-8: {e}")) +} + +/// Convert a result into the C return convention: on success return the +/// string and clear `*err_out`; on failure return NULL and store the +/// message in `*err_out`. +fn finish(err_out: *mut *mut c_char, result: Result) -> *mut c_char { + let result = result.and_then(|s| { + CString::new(s).map_err(|_| "result contains an embedded NUL byte".to_string()) + }); + match result { + Ok(cstr) => { + if !err_out.is_null() { + unsafe { *err_out = ptr::null_mut() }; + } + cstr.into_raw() + } + Err(msg) => { + if !err_out.is_null() { + let msg = CString::new(msg.replace('\0', "")).expect("NUL bytes were stripped"); + unsafe { *err_out = msg.into_raw() }; + } + ptr::null_mut() + } + } +} + +/// Parse `sql` using the named dialect and return the AST as a JSON array +/// of statements. +#[no_mangle] +pub extern "C" fn sql_ast_simple_parse( + dialect: *const c_char, + sql: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + let result = (|| { + let dialect_name = unsafe { borrow(dialect) }?; + let sql = unsafe { borrow(sql) }?; + let dialect = + dialect_from_str(dialect_name).ok_or_else(|| format!("unknown dialect: {dialect_name}"))?; + let ast = Parser::parse_sql(&*dialect, sql).map_err(|e| e.to_string())?; + serde_json::to_string(&ast).map_err(|e| e.to_string()) + })(); + finish(err_out, result) +} + +/// Turn a JSON array of statements (as produced by `sql_ast_simple_parse`, +/// possibly modified) back into SQL text. Statements are joined with +/// `"; "`, or `";\n"` when `pretty` is set. +#[no_mangle] +pub extern "C" fn sql_ast_simple_unparse( + json: *const c_char, + pretty: bool, + err_out: *mut *mut c_char, +) -> *mut c_char { + let result = (|| { + let json = unsafe { borrow(json) }?; + let ast: Vec = serde_json::from_str(json).map_err(|e| e.to_string())?; + let parts: Vec = ast + .iter() + .map(|s| if pretty { format!("{s:#}") } else { s.to_string() }) + .collect(); + Ok(parts.join(if pretty { ";\n" } else { "; " })) + })(); + finish(err_out, result) +} + +/// Release a string returned by any function in this library. +#[no_mangle] +pub extern "C" fn sql_ast_simple_free(p: *mut c_char) { + if !p.is_null() { + unsafe { drop(CString::from_raw(p)) }; + } +} diff --git a/lib/SQL/AST/Simple.pm b/lib/SQL/AST/Simple.pm index a0c6c43..ccd1253 100644 --- a/lib/SQL/AST/Simple.pm +++ b/lib/SQL/AST/Simple.pm @@ -3,9 +3,210 @@ package SQL::AST::Simple; use 5.042; use warnings; use FFI::Platypus 2.00; +use JSON::MaybeXS (); +use Carp (); +use Exporter qw( import ); + +# ABSTRACT: Parse SQL into a plain Perl data structure and back again +# VERSION + +=head1 SYNOPSIS + + use SQL::AST::Simple qw( parse unparse ); + + my $ast = parse('SELECT a, b FROM t WHERE a > 1', dialect => 'postgresql'); + + # $ast is an array reference of statements, each a tree of plain + # hashes, arrays and scalars. Poke at it however you like: + $ast->[0]{Query}{body}{Select}{from}[0]{relation}{Table}{name}[0]{Identifier}{value} = 'u'; + + say unparse($ast); # SELECT a, b FROM u WHERE a > 1 + +=head1 DESCRIPTION + +This module bundles the Rust +L crate and +exposes exactly two operations: turning SQL text into the parser's abstract +syntax tree as an ordinary Perl data structure, and turning such a data +structure back into SQL text. There is no object layer; the tree is what +the crate's serde serialization produces, decoded from JSON. That keeps +the module small and makes every node the crate knows about available +without any wrapping, at the cost of a somewhat verbose structure. + +Nothing is exported by default. + +=head1 FUNCTIONS + +=head2 parse + + my $ast = parse($sql); + my $ast = parse($sql, dialect => $name); + +Parses C<$sql>, which may contain several semicolon separated statements, +and returns an array reference with one element per statement. Throws an +exception with the parser's message, including line and column, if the +text cannot be parsed. + +Options: + +=over 4 + +=item dialect + +Which SQL dialect to parse with. Defaults to C, which is the +most permissive. Recognized names (case insensitive) are C, +C, C (or C), C, C, C, +C, C, C, C, C, +C, C, C, C (or C) and +C. + +=back + +=head2 unparse + + my $sql = unparse($ast); + my $sql = unparse($ast, pretty => 1); + +Takes an array reference of statements as returned by L, or a +single statement hash reference, and returns the SQL text. Multiple +statements are joined with C<"; ">. Throws an exception if the structure +does not deserialize into a valid AST. + +Options: + +=over 4 + +=item pretty + +If true, statements are formatted with indentation and newlines rather +than on a single line, and are joined with C<";\n">. + +=back + +=head1 THE DATA STRUCTURE + +The tree mirrors the Rust types of the C crate one to one, as +serialized by serde. A few rules of thumb cover most of it: + +=over 4 + +=item * + +Rust enums are "externally tagged": a hash with a single key naming the +variant, whose value is the payload. A C