From c9110785abd004933931768d4e065b5a4be5f386 Mon Sep 17 00:00:00 2001 From: Graham Ollis Date: Fri, 25 Sep 2026 06:53:53 -0600 Subject: [PATCH 1/3] Document the module as bindings for sqlparser, not a bundle The crate is fetched by cargo at build time rather than shipped in the distribution. Drop the FFI::Platypus::Lang::Rust reference from SEE ALSO as an implementation detail and link sqlparser via crates.io. Co-Authored-By: Claude Fable 5.1 --- README.md | 18 +++++++----------- lib/SQL/AST/Simple.pm | 18 +++++++----------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a38dc24..d8d0197 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ say unparse($ast); # SELECT a, b FROM u WHERE a > 1 # DESCRIPTION -This module bundles the Rust -[sqlparser](https://github.com/apache/datafusion-sqlparser-rs) crate and +This module provides Perl bindings for the Rust +[sqlparser](https://crates.io/crates/sqlparser) crate. It exposes exactly two operations: turning SQL text into the parser's abstract syntax tree as an ordinary Perl data structure, and turning such a data structure back into SQL text. There is no object layer; the tree is what @@ -100,10 +100,10 @@ snippet and lift the piece you need out of the result, rather than constructing hashes by hand. The easiest way to learn the shape for a given construct is to parse an -example and dump it. The exact shape depends on the bundled crate -version, which is pinned in the distribution's `ffi/Cargo.toml`; a -release that bumps it may change the structure and will say so in the -change log. +example and dump it. The exact shape depends on the version of the +crate the bindings are built against, which is pinned in the +distribution's `ffi/Cargo.toml`; a release that bumps it may change the +structure and will say so in the change log. # CAVEATS @@ -118,14 +118,10 @@ install time. # SEE ALSO -- [https://github.com/apache/datafusion-sqlparser-rs](https://github.com/apache/datafusion-sqlparser-rs) +- [https://crates.io/crates/sqlparser](https://crates.io/crates/sqlparser) The parser this module wraps. -- [FFI::Platypus::Lang::Rust](https://metacpan.org/pod/FFI::Platypus::Lang::Rust) - - How the Rust code is bundled and called. - # AUTHOR Graham Ollis diff --git a/lib/SQL/AST/Simple.pm b/lib/SQL/AST/Simple.pm index ccd1253..22dc361 100644 --- a/lib/SQL/AST/Simple.pm +++ b/lib/SQL/AST/Simple.pm @@ -24,8 +24,8 @@ use Exporter qw( import ); =head1 DESCRIPTION -This module bundles the Rust -L crate and +This module provides Perl bindings for the Rust +L crate. It exposes exactly two operations: turning SQL text into the parser's abstract syntax tree as an ordinary Perl data structure, and turning such a data structure back into SQL text. There is no object layer; the tree is what @@ -128,10 +128,10 @@ constructing hashes by hand. =back The easiest way to learn the shape for a given construct is to parse an -example and dump it. The exact shape depends on the bundled crate -version, which is pinned in the distribution's C; a -release that bumps it may change the structure and will say so in the -change log. +example and dump it. The exact shape depends on the version of the +crate the bindings are built against, which is pinned in the +distribution's C; a release that bumps it may change the +structure and will say so in the change log. =head1 CAVEATS @@ -148,14 +148,10 @@ install time. =over 4 -=item L +=item L The parser this module wraps. -=item L - -How the Rust code is bundled and called. - =back =cut From 5929989e5072400b50ad0253b1f755429119f577 Mon Sep 17 00:00:00 2001 From: Graham Ollis Date: Fri, 25 Sep 2026 07:05:08 -0600 Subject: [PATCH 2/3] Use Ref::Util and import croak Replace the ref eq comparisons in unparse with is_plain_hashref and is_plain_arrayref, and import croak from Carp instead of calling it fully qualified. Co-Authored-By: Claude Fable 5.1 --- lib/SQL/AST/Simple.pm | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/SQL/AST/Simple.pm b/lib/SQL/AST/Simple.pm index 22dc361..0c535b4 100644 --- a/lib/SQL/AST/Simple.pm +++ b/lib/SQL/AST/Simple.pm @@ -4,7 +4,8 @@ use 5.042; use warnings; use FFI::Platypus 2.00; use JSON::MaybeXS (); -use Carp (); +use Carp qw( croak ); +use Ref::Util qw( is_plain_arrayref is_plain_hashref ); use Exporter qw( import ); # ABSTRACT: Parse SQL into a plain Perl data structure and back again @@ -184,24 +185,24 @@ sub _call ($xsub, @args) { unless(defined $ptr) { my $msg = defined $err ? _take($err) : 'unknown error'; utf8::decode($msg); - Carp::croak($msg); + croak($msg); } return _take($ptr); } sub parse ($sql, %opt) { my $dialect = delete $opt{dialect} // 'generic'; - Carp::croak("unknown options: @{[ sort keys %opt ]}") if %opt; - Carp::croak("sql must be defined") unless defined $sql; + croak("unknown options: @{[ sort keys %opt ]}") if %opt; + croak("sql must be defined") unless defined $sql; utf8::encode($sql); return $json->decode(_call(\&_parse, $dialect, $sql)); } sub unparse ($ast, %opt) { my $pretty = delete $opt{pretty} // 0; - Carp::croak("unknown options: @{[ sort keys %opt ]}") if %opt; - $ast = [$ast] if ref $ast eq 'HASH'; - Carp::croak("ast must be an array or hash reference") unless ref $ast eq 'ARRAY'; + croak("unknown options: @{[ sort keys %opt ]}") if %opt; + $ast = [$ast] if is_plain_hashref $ast; + croak("ast must be an array or hash reference") unless is_plain_arrayref $ast; my $sql = _call(\&_unparse, $json->encode($ast), !!$pretty); utf8::decode($sql); return $sql; From f2754c64b87981ebdaaadc473eec0e06b4bce5a1 Mon Sep 17 00:00:00 2001 From: Graham Ollis Date: Fri, 25 Sep 2026 07:17:59 -0600 Subject: [PATCH 3/3] Add parse_expr and unparse_expr for lone expressions Expose the crate's expression parser so fragments such as a WHERE condition can be parsed and rendered without wrapping them in a statement. parse_expr requires the whole input to be consumed; unparse_expr takes a single expression hash and has no pretty option. Co-Authored-By: Claude Fable 5.1 --- README.md | 45 +++++++++++++++++++++++---- ffi/src/lib.rs | 54 +++++++++++++++++++++++++++++--- lib/SQL/AST/Simple.pm | 71 +++++++++++++++++++++++++++++++++++-------- t/00_diag.t | 1 + t/sql_ast_simple.t | 24 ++++++++++++++- 5 files changed, 171 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d8d0197..b3ec073 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Parse SQL into a plain Perl data structure and back again # SYNOPSIS ```perl -use SQL::AST::Simple qw( parse unparse ); +use SQL::AST::Simple qw( parse unparse parse_expr unparse_expr ); my $ast = parse('SELECT a, b FROM t WHERE a > 1', dialect => 'postgresql'); @@ -14,15 +14,20 @@ my $ast = parse('SELECT a, b FROM t WHERE a > 1', dialect => 'postgresql'); $ast->[0]{Query}{body}{Select}{from}[0]{relation}{Table}{name}[0]{Identifier}{value} = 'u'; say unparse($ast); # SELECT a, b FROM u WHERE a > 1 + +# Expressions can be handled on their own, without a statement around them: +$ast->[0]{Query}{body}{Select}{selection} = parse_expr('a > 1 AND b = 2'); +say unparse_expr($ast->[0]{Query}{body}{Select}{selection}); # a > 1 AND b = 2 ``` # DESCRIPTION This module provides Perl bindings for the Rust [sqlparser](https://crates.io/crates/sqlparser) crate. It -exposes exactly two operations: turning SQL text into the parser's abstract -syntax tree as an ordinary Perl data structure, and turning such a data -structure back into SQL text. There is no object layer; the tree is what +exposes two operations: turning SQL text into the parser's abstract syntax +tree as an ordinary Perl data structure, and turning such a data structure +back into SQL text. Each comes in a form for whole statements and a form +for a lone expression. There is no object layer; the tree is what the crate's serde serialization produces, decoded from JSON. That keeps the module small and makes every node the crate knows about available without any wrapping, at the cost of a somewhat verbose structure. @@ -54,6 +59,23 @@ Options: `duckdb`, `databricks`, `hive`, `spark` (or `sparksql`) and `teradata`. +## parse\_expr + +```perl +my $expr = parse_expr($sql); +my $expr = parse_expr($sql, dialect => $name); +``` + +Parses `$sql` as a single expression, such as the condition of a `WHERE` +clause, and returns it as a hash reference. The whole of `$sql` must be +consumed by the expression; a leading `WHERE` keyword or anything left +over after the expression is an error. Takes the same `dialect` option +as ["parse"](#parse). + +The result is exactly what appears inside a statement wherever the crate +expects an expression, so it can be spliced into a tree from ["parse"](#parse), +for instance as the `selection` of a `SELECT`. + ## unparse ```perl @@ -73,6 +95,17 @@ Options: If true, statements are formatted with indentation and newlines rather than on a single line, and are joined with `";\n"`. +## unparse\_expr + +```perl +my $sql = unparse_expr($expr); +``` + +Takes an expression hash reference, as returned by ["parse\_expr"](#parse_expr) or +lifted out of a statement, and returns the SQL text. Throws an exception +if the structure does not deserialize into a valid expression. There is +no `pretty` option; expressions are always rendered on one line. + # THE DATA STRUCTURE The tree mirrors the Rust types of the `sqlparser` crate one to one, as @@ -96,8 +129,8 @@ first. - Most nodes carry a `span` hash recording where they appeared in the source. ["unparse"](#unparse) ignores the contents but requires the field to be present, so the easiest way to build a new node is to parse a small -snippet and lift the piece you need out of the result, rather than -constructing hashes by hand. +snippet (with ["parse\_expr"](#parse_expr) for an expression) and lift the piece you +need out of the result, rather than constructing hashes by hand. The easiest way to learn the shape for a given construct is to parse an example and dump it. The exact shape depends on the version of the diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs index 44d323a..27cb22a 100644 --- a/ffi/src/lib.rs +++ b/ffi/src/lib.rs @@ -5,9 +5,10 @@ //! returned to the caller is a NUL terminated, heap allocated C string that //! must be released with `sql_ast_simple_free`. -use sqlparser::ast::Statement; -use sqlparser::dialect::dialect_from_str; +use sqlparser::ast::{Expr, Statement}; +use sqlparser::dialect::{dialect_from_str, Dialect}; use sqlparser::parser::Parser; +use sqlparser::tokenizer::Token; use std::ffi::{c_char, CStr, CString}; use std::ptr; @@ -45,6 +46,12 @@ fn finish(err_out: *mut *mut c_char, result: Result) -> *mut c_c } } +/// Look up a dialect by name. +fn dialect(name: *const c_char) -> Result, String> { + let name = unsafe { borrow(name) }?; + dialect_from_str(name).ok_or_else(|| format!("unknown dialect: {name}")) +} + /// Parse `sql` using the named dialect and return the AST as a JSON array /// of statements. #[no_mangle] @@ -54,16 +61,38 @@ pub extern "C" fn sql_ast_simple_parse( err_out: *mut *mut c_char, ) -> *mut c_char { let result = (|| { - let dialect_name = unsafe { borrow(dialect) }?; + let dialect = self::dialect(dialect)?; let sql = unsafe { borrow(sql) }?; - let dialect = - dialect_from_str(dialect_name).ok_or_else(|| format!("unknown dialect: {dialect_name}"))?; let ast = Parser::parse_sql(&*dialect, sql).map_err(|e| e.to_string())?; serde_json::to_string(&ast).map_err(|e| e.to_string()) })(); finish(err_out, result) } +/// Parse `sql` as a single expression, such as the body of a `WHERE` +/// clause, and return it as JSON. Anything left over after the expression +/// is an error. +#[no_mangle] +pub extern "C" fn sql_ast_simple_parse_expr( + dialect: *const c_char, + sql: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + let result = (|| { + let dialect = self::dialect(dialect)?; + let sql = unsafe { borrow(sql) }?; + let mut parser = Parser::new(&*dialect) + .try_with_sql(sql) + .map_err(|e| e.to_string())?; + let expr = parser.parse_expr().map_err(|e| e.to_string())?; + parser + .expect_token(&Token::EOF) + .map_err(|e| e.to_string())?; + serde_json::to_string(&expr).map_err(|e| e.to_string()) + })(); + finish(err_out, result) +} + /// Turn a JSON array of statements (as produced by `sql_ast_simple_parse`, /// possibly modified) back into SQL text. Statements are joined with /// `"; "`, or `";\n"` when `pretty` is set. @@ -85,6 +114,21 @@ pub extern "C" fn sql_ast_simple_unparse( finish(err_out, result) } +/// Turn a JSON expression (as produced by `sql_ast_simple_parse_expr`, +/// possibly modified) back into SQL text. +#[no_mangle] +pub extern "C" fn sql_ast_simple_unparse_expr( + json: *const c_char, + err_out: *mut *mut c_char, +) -> *mut c_char { + let result = (|| { + let json = unsafe { borrow(json) }?; + let expr: Expr = serde_json::from_str(json).map_err(|e| e.to_string())?; + Ok(expr.to_string()) + })(); + finish(err_out, result) +} + /// Release a string returned by any function in this library. #[no_mangle] pub extern "C" fn sql_ast_simple_free(p: *mut c_char) { diff --git a/lib/SQL/AST/Simple.pm b/lib/SQL/AST/Simple.pm index 0c535b4..3c1b600 100644 --- a/lib/SQL/AST/Simple.pm +++ b/lib/SQL/AST/Simple.pm @@ -13,7 +13,7 @@ use Exporter qw( import ); =head1 SYNOPSIS - use SQL::AST::Simple qw( parse unparse ); + use SQL::AST::Simple qw( parse unparse parse_expr unparse_expr ); my $ast = parse('SELECT a, b FROM t WHERE a > 1', dialect => 'postgresql'); @@ -23,13 +23,18 @@ use Exporter qw( import ); say unparse($ast); # SELECT a, b FROM u WHERE a > 1 + # Expressions can be handled on their own, without a statement around them: + $ast->[0]{Query}{body}{Select}{selection} = parse_expr('a > 1 AND b = 2'); + say unparse_expr($ast->[0]{Query}{body}{Select}{selection}); # a > 1 AND b = 2 + =head1 DESCRIPTION This module provides Perl bindings for the Rust L crate. It -exposes exactly two operations: turning SQL text into the parser's abstract -syntax tree as an ordinary Perl data structure, and turning such a data -structure back into SQL text. There is no object layer; the tree is what +exposes two operations: turning SQL text into the parser's abstract syntax +tree as an ordinary Perl data structure, and turning such a data structure +back into SQL text. Each comes in a form for whole statements and a form +for a lone expression. There is no object layer; the tree is what the crate's serde serialization produces, decoded from JSON. That keeps the module small and makes every node the crate knows about available without any wrapping, at the cost of a somewhat verbose structure. @@ -63,6 +68,21 @@ C. =back +=head2 parse_expr + + my $expr = parse_expr($sql); + my $expr = parse_expr($sql, dialect => $name); + +Parses C<$sql> as a single expression, such as the condition of a C +clause, and returns it as a hash reference. The whole of C<$sql> must be +consumed by the expression; a leading C keyword or anything left +over after the expression is an error. Takes the same C option +as L. + +The result is exactly what appears inside a statement wherever the crate +expects an expression, so it can be spliced into a tree from L, +for instance as the C of a C