diff --git a/bench/bench.py b/bench/bench.py index ea15d4c..b2c742e 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -82,7 +82,9 @@ def load_lints(checkout): mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) lints = dict(mod.load_sql_files(checkout / "lints")) - lints["all"] = "\nunion all\n".join(lints.values()) + # What splinter.sql runs, minus the header (sent separately): each checkout's own + # compile_sql, so changes to how lints are combined are measured too. + lints["all"] = mod.compile_sql(dict(lints)).removeprefix(mod.HEADER) return mod.HEADER, lints diff --git a/bin/compile.py b/bin/compile.py index e7beba9..33bc27e 100755 --- a/bin/compile.py +++ b/bin/compile.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from typing import Dict @@ -59,8 +60,38 @@ def load_sql_files(directory_path) -> Dict[str, str]: return queries +# A CTE that several lints compute identically, between these markers. Each lint +# keeps its own copy so it still runs on its own (tests, splinter.json); splinter.sql +# lifts one materialized copy above the union so the work is done once. +SHARED_CTE_RE = re.compile( + r"[ \t]*-- shared-cte-start: (\w+)\n(.*?)\n[ \t]*-- shared-cte-end\n", re.S +) + + +def hoist_shared_ctes(sql_map: Dict[str, str]) -> tuple[Dict[str, str], Dict[str, str]]: + shared: Dict[str, str] = {} + hoisted: Dict[str, str] = {} + for stem, query in sql_map.items(): + for name, body in SHARED_CTE_RE.findall(query): + body = body.strip().rstrip(",") + if shared.setdefault(name, body) != body: + raise Exception( + f"{stem}: shared CTE {name} differs from another lint's copy" + ) + hoisted[stem] = SHARED_CTE_RE.sub("", query) + return shared, hoisted + + def compile_sql(sql_map: Dict[str, str]) -> str: - return HEADER + "\n" + "\nunion all\n".join(sql_map.values()) + shared, sql_map = hoist_shared_ctes(sql_map) + with_clause = "" + if shared: + ctes = [ + body.replace(f"{name} as (", f"{name} as materialized (", 1) + for name, body in shared.items() + ] + with_clause = "with\n" + ",\n".join(ctes) + "\n" + return HEADER + "\n" + with_clause + "\nunion all\n".join(sql_map.values()) def main() -> None: diff --git a/lints/0003_auth_rls_initplan.sql b/lints/0003_auth_rls_initplan.sql index 4f49a54..4136f27 100644 --- a/lints/0003_auth_rls_initplan.sql +++ b/lints/0003_auth_rls_initplan.sql @@ -1,6 +1,21 @@ create view lint."0003_auth_rls_initplan" as -with policies as materialized ( +with + -- shared-cte-start: splinter_policies + -- pg_policy with qual and with_check as pg_policies reports them + splinter_policies as ( + select + polname, + polrelid, + polcmd, + polpermissive, + polroles, + pg_catalog.pg_get_expr(polqual, polrelid) as qual, + pg_catalog.pg_get_expr(polwithcheck, polrelid) as with_check + from pg_catalog.pg_policy + ), + -- shared-cte-end + policies as materialized ( select nsp.nspname as schema_name, pc.relname as table_name, @@ -15,11 +30,10 @@ with policies as materialized ( when 'd' then 'DELETE' when '*' then 'ALL' end as command, - -- what pg_policies reports as qual and with_check - pg_catalog.pg_get_expr(pa.polqual, pa.polrelid) as qual, - pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid) as with_check + pa.qual, + pa.with_check from - pg_catalog.pg_policy pa + splinter_policies pa join pg_catalog.pg_class pc on pa.polrelid = pc.oid join pg_catalog.pg_namespace nsp diff --git a/lints/0015_rls_references_user_metadata.sql b/lints/0015_rls_references_user_metadata.sql index a810884..0040ebd 100644 --- a/lints/0015_rls_references_user_metadata.sql +++ b/lints/0015_rls_references_user_metadata.sql @@ -1,15 +1,29 @@ create view lint."0015_rls_references_user_metadata" as -with policies as ( +with + -- shared-cte-start: splinter_policies + -- pg_policy with qual and with_check as pg_policies reports them + splinter_policies as ( + select + polname, + polrelid, + polcmd, + polpermissive, + polroles, + pg_catalog.pg_get_expr(polqual, polrelid) as qual, + pg_catalog.pg_get_expr(polwithcheck, polrelid) as with_check + from pg_catalog.pg_policy + ), + -- shared-cte-end + policies as ( select nsp.nspname as schema_name, pc.relname as table_name, polname as policy_name, - -- what pg_policies reports as qual and with_check - pg_catalog.pg_get_expr(pa.polqual, pa.polrelid) as qual, - pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid) as with_check + pa.qual, + pa.with_check from - pg_catalog.pg_policy pa + splinter_policies pa join pg_catalog.pg_class pc on pa.polrelid = pc.oid join pg_catalog.pg_namespace nsp diff --git a/lints/0024_rls_policy_always_true.sql b/lints/0024_rls_policy_always_true.sql index 2e5a389..f72ef07 100644 --- a/lints/0024_rls_policy_always_true.sql +++ b/lints/0024_rls_policy_always_true.sql @@ -2,7 +2,22 @@ create view lint."0024_rls_policy_always_true" as -- Detects RLS policies that are overly permissive (e.g., USING (true), USING (1=1)) -- These policies effectively disable row-level security while giving a false sense of security -with policies as ( +with + -- shared-cte-start: splinter_policies + -- pg_policy with qual and with_check as pg_policies reports them + splinter_policies as ( + select + polname, + polrelid, + polcmd, + polpermissive, + polroles, + pg_catalog.pg_get_expr(polqual, polrelid) as qual, + pg_catalog.pg_get_expr(polwithcheck, polrelid) as with_check + from pg_catalog.pg_policy + ), + -- shared-cte-end + policies as ( select nsp.nspname as schema_name, pc.relname as table_name, @@ -18,14 +33,13 @@ with policies as ( when 'd' then 'DELETE' when '*' then 'ALL' end as command, - -- what pg_policies reports as qual and with_check - pg_catalog.pg_get_expr(pa.polqual, pa.polrelid) as qual, - pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid) as with_check, + pa.qual, + pa.with_check, -- Normalize expressions by removing whitespace and lowercasing - replace(replace(replace(lower(coalesce(pg_catalog.pg_get_expr(pa.polqual, pa.polrelid), '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_qual, - replace(replace(replace(lower(coalesce(pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid), '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_with_check + replace(replace(replace(lower(coalesce(pa.qual, '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_qual, + replace(replace(replace(lower(coalesce(pa.with_check, '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_with_check from - pg_catalog.pg_policy pa + splinter_policies pa join pg_catalog.pg_class pc on pa.polrelid = pc.oid join pg_catalog.pg_namespace nsp diff --git a/splinter.sql b/splinter.sql index 9a1708a..c4d5924 100644 --- a/splinter.sql +++ b/splinter.sql @@ -22,6 +22,19 @@ begin perform pg_catalog.set_config('splinter.public_buckets', buckets, true); end $$; +with +-- pg_policy with qual and with_check as pg_policies reports them + splinter_policies as materialized ( + select + polname, + polrelid, + polcmd, + polpermissive, + polroles, + pg_catalog.pg_get_expr(polqual, polrelid) as qual, + pg_catalog.pg_get_expr(polwithcheck, polrelid) as with_check + from pg_catalog.pg_policy + ) ( with foreign_keys as ( select @@ -189,7 +202,8 @@ group by c.oid) union all ( -with policies as materialized ( +with + policies as materialized ( select nsp.nspname as schema_name, pc.relname as table_name, @@ -204,11 +218,10 @@ with policies as materialized ( when 'd' then 'DELETE' when '*' then 'ALL' end as command, - -- what pg_policies reports as qual and with_check - pg_catalog.pg_get_expr(pa.polqual, pa.polrelid) as qual, - pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid) as with_check + pa.qual, + pa.with_check from - pg_catalog.pg_policy pa + splinter_policies pa join pg_catalog.pg_class pc on pa.polrelid = pc.oid join pg_catalog.pg_namespace nsp @@ -778,16 +791,16 @@ where and pe.extnamespace::regnamespace::text = 'public') union all ( -with policies as ( +with + policies as ( select nsp.nspname as schema_name, pc.relname as table_name, polname as policy_name, - -- what pg_policies reports as qual and with_check - pg_catalog.pg_get_expr(pa.polqual, pa.polrelid) as qual, - pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid) as with_check + pa.qual, + pa.with_check from - pg_catalog.pg_policy pa + splinter_policies pa join pg_catalog.pg_class pc on pa.polrelid = pc.oid join pg_catalog.pg_namespace nsp @@ -1286,7 +1299,8 @@ union all ( -- Detects RLS policies that are overly permissive (e.g., USING (true), USING (1=1)) -- These policies effectively disable row-level security while giving a false sense of security -with policies as ( +with + policies as ( select nsp.nspname as schema_name, pc.relname as table_name, @@ -1302,14 +1316,13 @@ with policies as ( when 'd' then 'DELETE' when '*' then 'ALL' end as command, - -- what pg_policies reports as qual and with_check - pg_catalog.pg_get_expr(pa.polqual, pa.polrelid) as qual, - pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid) as with_check, + pa.qual, + pa.with_check, -- Normalize expressions by removing whitespace and lowercasing - replace(replace(replace(lower(coalesce(pg_catalog.pg_get_expr(pa.polqual, pa.polrelid), '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_qual, - replace(replace(replace(lower(coalesce(pg_catalog.pg_get_expr(pa.polwithcheck, pa.polrelid), '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_with_check + replace(replace(replace(lower(coalesce(pa.qual, '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_qual, + replace(replace(replace(lower(coalesce(pa.with_check, '')), ' ', ''), E'\n', ''), E'\t', '') as normalized_with_check from - pg_catalog.pg_policy pa + splinter_policies pa join pg_catalog.pg_class pc on pa.polrelid = pc.oid join pg_catalog.pg_namespace nsp