From c654b3e06aea75288337a2292231490cd11c28dd Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 14:50:44 -0400 Subject: [PATCH 01/23] install alembic_utils_extended step 1 of quickstart in the repo Signed-off-by: Adrian Edwards --- pyproject.toml | 1 + uv.lock | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9d354c0a0..072fb5ffc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ dependencies = [ "xgboost==3.0.2", "xlrd==2.0.1", "xlsxwriter==1.3.7", + "alembic-utils-extended>=1.5.0", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index b58078ea0..9ebf2b45b 100644 --- a/uv.lock +++ b/uv.lock @@ -59,6 +59,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/01/d55bd80997df2ec1ff2fd40cd3eeadec93c4b3c5492df3c6852b29f9e393/alembic_utils-0.8.8-py3-none-any.whl", hash = "sha256:2c2545dc545833c5deb63bce2c3cde01c1807bf99da5efab2497bc8d817cb86e", size = 31044, upload-time = "2025-04-10T18:58:12.247Z" }, ] +[[package]] +name = "alembic-utils-extended" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alembic" }, + { name = "flupy" }, + { name = "parse" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/b0/5187a03dcea41dca1f684a815309689680a987db56e0d777d9a608d0cfc3/alembic_utils_extended-1.5.0.tar.gz", hash = "sha256:636963c932458d52c4f489830140c677755038ef4dc2ad49f86789b98a1224b8", size = 42298, upload-time = "2026-08-13T00:12:13.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/3f/70b7b79b4738b4c85c3a5af1fd9357009d169b331872d92b16b4d37d0392/alembic_utils_extended-1.5.0-py3-none-any.whl", hash = "sha256:9a53a9767291239fd29126d99d5c2c7ffe7c4a8120bc7be69dc89f5cdd978995", size = 53262, upload-time = "2026-08-13T00:12:11.806Z" }, +] + [[package]] name = "amqp" version = "5.3.1" @@ -397,6 +413,7 @@ source = { editable = "." } dependencies = [ { name = "alembic" }, { name = "alembic-utils" }, + { name = "alembic-utils-extended" }, { name = "beaker" }, { name = "boto3" }, { name = "bs4" }, @@ -502,6 +519,7 @@ test = [ requires-dist = [ { name = "alembic", specifier = ">=1.17.1" }, { name = "alembic-utils", specifier = "==0.8.8" }, + { name = "alembic-utils-extended", specifier = ">=1.5.0" }, { name = "beaker", specifier = "==1.11.0" }, { name = "boto3", specifier = "==1.17.57" }, { name = "bs4", specifier = "==0.0.1" }, From a6b2a8c10fe47ba038845c6621f7d609af52222f Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 14:51:49 -0400 Subject: [PATCH 02/23] add logger to alembic.ini per step 2 of the quickstart Signed-off-by: Adrian Edwards --- alembic.ini | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/alembic.ini b/alembic.ini index 7d0469d9d..7b3787c4d 100644 --- a/alembic.ini +++ b/alembic.ini @@ -68,7 +68,7 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # Logging configuration [loggers] -keys = root,sqlalchemy,alembic +keys = root,sqlalchemy,alembic,alembic_utils_extended [handlers] keys = console @@ -100,3 +100,9 @@ formatter = generic [formatter_generic] format = %(levelname)-5.5s [%(name)s] %(message)s datefmt = %H:%M:%S + + +[logger_alembic_utils_extended] +level = INFO +handlers = +qualname = alembic_utils_extended \ No newline at end of file From 6de4d52143ff48fecb04358305844f1a34cf6e1d Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 14:58:41 -0400 Subject: [PATCH 03/23] remove alembic utils Signed-off-by: Adrian Edwards --- pyproject.toml | 1 - uv.lock | 18 ------------------ 2 files changed, 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 072fb5ffc..14277283a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ classifiers = [ ] dependencies = [ "alembic>=1.17.1", - "alembic-utils==0.8.8", "Beaker==1.11.0", "boto3==1.17.57", "bs4==0.0.1", diff --git a/uv.lock b/uv.lock index 9ebf2b45b..6038ebf47 100644 --- a/uv.lock +++ b/uv.lock @@ -43,22 +43,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/32/7df1d81ec2e50fb661944a35183d87e62d3f6c6d9f8aff64a4f245226d55/alembic-1.17.1-py3-none-any.whl", hash = "sha256:cbc2386e60f89608bb63f30d2d6cc66c7aaed1fe105bd862828600e5ad167023", size = 247848, upload-time = "2025-10-29T00:23:18.79Z" }, ] -[[package]] -name = "alembic-utils" -version = "0.8.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "alembic" }, - { name = "flupy" }, - { name = "parse" }, - { name = "sqlalchemy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ec/7a/eae622a97ba1721fd7e420c60060a74852b800ac1fecbaa2e67a35941d6d/alembic_utils-0.8.8.tar.gz", hash = "sha256:99de5d13194f26536bc0322f0c1660020a305015700d8447ccfc20e7d1494e5b", size = 21638, upload-time = "2025-04-10T18:58:13.212Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/01/d55bd80997df2ec1ff2fd40cd3eeadec93c4b3c5492df3c6852b29f9e393/alembic_utils-0.8.8-py3-none-any.whl", hash = "sha256:2c2545dc545833c5deb63bce2c3cde01c1807bf99da5efab2497bc8d817cb86e", size = 31044, upload-time = "2025-04-10T18:58:12.247Z" }, -] - [[package]] name = "alembic-utils-extended" version = "1.5.0" @@ -412,7 +396,6 @@ name = "collectoss" source = { editable = "." } dependencies = [ { name = "alembic" }, - { name = "alembic-utils" }, { name = "alembic-utils-extended" }, { name = "beaker" }, { name = "boto3" }, @@ -518,7 +501,6 @@ test = [ [package.metadata] requires-dist = [ { name = "alembic", specifier = ">=1.17.1" }, - { name = "alembic-utils", specifier = "==0.8.8" }, { name = "alembic-utils-extended", specifier = ">=1.5.0" }, { name = "beaker", specifier = "==1.11.0" }, { name = "boto3", specifier = "==1.17.57" }, From f6466a97c2edc8de424806c9267cd55f67a25d68 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 15:02:57 -0400 Subject: [PATCH 04/23] swap imports Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 2 +- collectoss/application/schema/alembic/env.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 5d2923636..62785bd68 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -22,7 +22,7 @@ from dataclasses import dataclass -from alembic_utils.pg_materialized_view import PGMaterializedView +from alembic_utils_extended.pg_materialized_view import PGMaterializedView @dataclass(frozen=True) diff --git a/collectoss/application/schema/alembic/env.py b/collectoss/application/schema/alembic/env.py index 77f2bd2e4..90049330a 100644 --- a/collectoss/application/schema/alembic/env.py +++ b/collectoss/application/schema/alembic/env.py @@ -41,8 +41,8 @@ # # WARNING: If MATERIALIZED_VIEWS is ever emptied, autogenerate will propose # dropping all registered views. Keep the list complete. -from alembic_utils.pg_materialized_view import PGMaterializedView -from alembic_utils.replaceable_entity import register_entities +from alembic_utils_extended.pg_materialized_view import PGMaterializedView +from alembic_utils_extended.replaceable_entity import register_entities from collectoss.application.db.materialized_views import MATERIALIZED_VIEWS _materialized_view_entities = [view.to_pg_view() for view in MATERIALIZED_VIEWS] register_entities(_materialized_view_entities, entity_types=[PGMaterializedView]) From b5f7b3c3b57f02670e477ebdeb9fb1c93c7e204a Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 15:04:18 -0400 Subject: [PATCH 05/23] update docs references to alembic_utils_extended Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 62785bd68..0c7f084b6 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -6,18 +6,14 @@ immutable snapshots), but this registry is the canonical reference for what views exist and how to refresh them. -alembic_utils autogenerate is wired up (phase 2 complete). Running +alembic_utils_extended autogenerate is wired up (phase 2 complete). Running `alembic revision --autogenerate` will detect changes to view SQL definitions. -alembic_utils uses a round-trip comparison: it creates the Python-defined view +alembic_utils_extended uses a round-trip comparison: it creates the Python-defined view in a savepoint, reads back what PostgreSQL stored, and diffs that against the live database. Manual SQL normalization is not required. WARNING: ALL views must be registered here or autogenerate will propose dropping unregistered ones. Keep this list complete. - -alembic_utils does NOT manage indexes on materialized views. See the developer -note in collectoss/application/schema/alembic/env.py for the full post-replace -procedure (index recreation + non-concurrent refresh). """ from dataclasses import dataclass From c262081439e0aaadbe3840cdc854cf86e83bc749 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 15:13:32 -0400 Subject: [PATCH 06/23] remove unused import from alembic_utils in an old migration Signed-off-by: Adrian Edwards --- .../schema/alembic/versions/44_remove_redundant_pr_constraint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/collectoss/application/schema/alembic/versions/44_remove_redundant_pr_constraint.py b/collectoss/application/schema/alembic/versions/44_remove_redundant_pr_constraint.py index 770a1701b..06bfce979 100644 --- a/collectoss/application/schema/alembic/versions/44_remove_redundant_pr_constraint.py +++ b/collectoss/application/schema/alembic/versions/44_remove_redundant_pr_constraint.py @@ -7,7 +7,6 @@ """ from alembic import op import sqlalchemy as sa -from alembic_utils.pg_materialized_view import PGMaterializedView from sqlalchemy import text as sql_text # revision identifiers, used by Alembic. From 582e6269ecc46d46393877ea06da7a7d6e9d07f4 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 15:29:48 -0400 Subject: [PATCH 07/23] Convert MaterializedView dataclass into a thin subclass wrapper Generated-by: Sonnet 4.6 medium Signed-off-by: Adrian Edwards --- .../application/db/materialized_views.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 0c7f084b6..f0f032033 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -16,17 +16,33 @@ dropping unregistered ones. Keep this list complete. """ -from dataclasses import dataclass - from alembic_utils_extended.pg_materialized_view import PGMaterializedView -@dataclass(frozen=True) -class MaterializedView: - name: str - schema: str - sql: str - unique_index_columns: tuple[str, ...] = () +class MaterializedView(PGMaterializedView): + """Thin subclass of ``PGMaterializedView`` with CollectOSS-specific helpers. + + The constructor uses CollectOSS-style kwargs (``name``, ``sql``, + ``unique_index_columns``) and delegates to ``PGMaterializedView.__init__``, + so every instance *is* a ``PGMaterializedView`` and can be passed directly + to ``alembic_utils_extended.register_entities`` without any conversion. + """ + + def __init__( + self, + name: str, + schema: str, + sql: str, + unique_index_columns: tuple[str, ...] = (), + with_data: bool = False, + ) -> None: + super().__init__( + schema=schema, + signature=name, + definition=sql, + with_data=with_data, + ) + self.unique_index_columns = unique_index_columns @property def fqn(self) -> str: @@ -45,8 +61,7 @@ def to_pg_view(self, with_data: bool = False) -> PGMaterializedView: ) def __repr__(self) -> str: - # Default dataclass repr would dump the full SQL string (some views - # are 100+ lines), making logs and debug output unreadable. + # Avoid dumping the full SQL string (some views are 100+ lines) in logs. return ( f"MaterializedView(name={self.name!r}, schema={self.schema!r}, " f"unique_index_columns={self.unique_index_columns!r})" From 5c245dfd1c5f3063c3402b93fe0f80e8d745c574 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 15:30:57 -0400 Subject: [PATCH 08/23] remove to_pg_view since we now are a view Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 8 -------- collectoss/application/schema/alembic/env.py | 3 +-- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index f0f032033..638b03e70 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -52,14 +52,6 @@ def refresh_sql(self, concurrently: bool = True) -> str: mode = "CONCURRENTLY " if concurrently else "" return f"REFRESH MATERIALIZED VIEW {mode}{self.fqn} WITH DATA;" - def to_pg_view(self, with_data: bool = False) -> PGMaterializedView: - return PGMaterializedView( - schema=self.schema, - signature=self.name, - definition=self.sql, - with_data=with_data, - ) - def __repr__(self) -> str: # Avoid dumping the full SQL string (some views are 100+ lines) in logs. return ( diff --git a/collectoss/application/schema/alembic/env.py b/collectoss/application/schema/alembic/env.py index 90049330a..a648cf69a 100644 --- a/collectoss/application/schema/alembic/env.py +++ b/collectoss/application/schema/alembic/env.py @@ -44,8 +44,7 @@ from alembic_utils_extended.pg_materialized_view import PGMaterializedView from alembic_utils_extended.replaceable_entity import register_entities from collectoss.application.db.materialized_views import MATERIALIZED_VIEWS -_materialized_view_entities = [view.to_pg_view() for view in MATERIALIZED_VIEWS] -register_entities(_materialized_view_entities, entity_types=[PGMaterializedView]) +register_entities(MATERIALIZED_VIEWS, entity_types=[PGMaterializedView]) # other values from the config, defined by the needs of env.py, # can be acquired: From bf9c8551c32626a277a89912d9fd3455d35953b4 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 15:33:50 -0400 Subject: [PATCH 09/23] make refresh_sql aware of the with_data setting, but allow an optional override Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 638b03e70..8a0a2677d 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -48,9 +48,17 @@ def __init__( def fqn(self) -> str: return f"{self.schema}.{self.name}" - def refresh_sql(self, concurrently: bool = True) -> str: + def refresh_sql(self, concurrently: bool = True, with_data: bool | None = None) -> str: + """Return the SQL to refresh the view. + + Args: + concurrently: Whether to refresh the view CONCURRENTLY. + with_data: Whether to refresh the view WITH DATA. If None, uses the + view's ``with_data`` attribute. + """ mode = "CONCURRENTLY " if concurrently else "" - return f"REFRESH MATERIALIZED VIEW {mode}{self.fqn} WITH DATA;" + data_present = "DATA" if with_data or self.with_data else "NO DATA" + return f"REFRESH MATERIALIZED VIEW {mode}{self.fqn} WITH {data_present};" def __repr__(self) -> str: # Avoid dumping the full SQL string (some views are 100+ lines) in logs. From 90ce0e9dbed327bcd263ddef09868f7b58e75b8d Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 12 Oct 2026 16:18:21 -0400 Subject: [PATCH 10/23] put to_pg_view() back so that the materialized views generate cleanly changes made above what sonnet did: - remove a numbered list from the docs and clean up the language a little - change the with_data param to be an override for what we already defined on self, not a straight up setting of the value Generated-by: Sonnet 4.6 Medium Signed-off-by: Adrian Edwards --- .../application/db/materialized_views.py | 23 +++++++++++++++++++ collectoss/application/schema/alembic/env.py | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 8a0a2677d..88438c11b 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -60,6 +60,29 @@ def refresh_sql(self, concurrently: bool = True, with_data: bool | None = None) data_present = "DATA" if with_data or self.with_data else "NO DATA" return f"REFRESH MATERIALIZED VIEW {mode}{self.fqn} WITH {data_present};" + def to_pg_view(self, with_data: bool = None) -> PGMaterializedView: + """Return a plain ``PGMaterializedView`` for use with alembic_utils registration. + + We cannot return *self* here even though ``MaterializedView`` is a + ``PGMaterializedView`` subclass. ``alembic_utils_extended`` uses two + exact-class checks that both fail with a subclass and cause spurious + ``DropOp``s for all materialized views. + + Returning a genuine ``PGMaterializedView`` sidesteps both issues while + all runtime helpers (``fqn``, ``refresh_sql``, ``unique_index_columns``) + remain accessible on *self*. + + Parameters: + with_data: Optional override for ``with_data`` in the returned view. If None, uses the + view's ``with_data`` attribute. + """ + return PGMaterializedView( + schema=self.schema, + signature=self.signature, + definition=self.definition, + with_data=with_data or self.with_data, + ) + def __repr__(self) -> str: # Avoid dumping the full SQL string (some views are 100+ lines) in logs. return ( diff --git a/collectoss/application/schema/alembic/env.py b/collectoss/application/schema/alembic/env.py index a648cf69a..90049330a 100644 --- a/collectoss/application/schema/alembic/env.py +++ b/collectoss/application/schema/alembic/env.py @@ -44,7 +44,8 @@ from alembic_utils_extended.pg_materialized_view import PGMaterializedView from alembic_utils_extended.replaceable_entity import register_entities from collectoss.application.db.materialized_views import MATERIALIZED_VIEWS -register_entities(MATERIALIZED_VIEWS, entity_types=[PGMaterializedView]) +_materialized_view_entities = [view.to_pg_view() for view in MATERIALIZED_VIEWS] +register_entities(_materialized_view_entities, entity_types=[PGMaterializedView]) # other values from the config, defined by the needs of env.py, # can be acquired: From ddd3f6c256bf8473c30c12ad208ac763cdc690ab Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Sun, 18 Oct 2026 10:36:11 -0400 Subject: [PATCH 11/23] add Index argument Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 88438c11b..4767a92a5 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -17,6 +17,7 @@ """ from alembic_utils_extended.pg_materialized_view import PGMaterializedView +from alembic_utils_extended.index import Index class MaterializedView(PGMaterializedView): @@ -35,6 +36,7 @@ def __init__( sql: str, unique_index_columns: tuple[str, ...] = (), with_data: bool = False, + indexes = list[Index] ) -> None: super().__init__( schema=schema, From 908d35d6f39e6a66c2c559c325d0d52a69139bd9 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Sun, 18 Oct 2026 10:46:33 -0400 Subject: [PATCH 12/23] with_data or self.with_data means explicitly passing with_data=False is silently ignored. Assisted-by: GLM 5.3 Low Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 4767a92a5..0f7c5a406 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -58,8 +58,10 @@ def refresh_sql(self, concurrently: bool = True, with_data: bool | None = None) with_data: Whether to refresh the view WITH DATA. If None, uses the view's ``with_data`` attribute. """ + if with_data is None: + with_data = self.with_data mode = "CONCURRENTLY " if concurrently else "" - data_present = "DATA" if with_data or self.with_data else "NO DATA" + data_present = "DATA" if with_data else "NO DATA" return f"REFRESH MATERIALIZED VIEW {mode}{self.fqn} WITH {data_present};" def to_pg_view(self, with_data: bool = None) -> PGMaterializedView: @@ -78,11 +80,13 @@ def to_pg_view(self, with_data: bool = None) -> PGMaterializedView: with_data: Optional override for ``with_data`` in the returned view. If None, uses the view's ``with_data`` attribute. """ + if with_data is None: + with_data = self.with_data return PGMaterializedView( schema=self.schema, signature=self.signature, definition=self.definition, - with_data=with_data or self.with_data, + with_data=with_data, ) def __repr__(self) -> str: From 18a9f3fe5754825ceb4e669ee67812349e6639b1 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Sun, 18 Oct 2026 10:46:59 -0400 Subject: [PATCH 13/23] corrections to and proper passthrough of indexes arg Assisted-by: GLM 5.3 Low Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 0f7c5a406..0cf573620 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -17,7 +17,7 @@ """ from alembic_utils_extended.pg_materialized_view import PGMaterializedView -from alembic_utils_extended.index import Index +from sqlalchemy import Index class MaterializedView(PGMaterializedView): @@ -36,13 +36,14 @@ def __init__( sql: str, unique_index_columns: tuple[str, ...] = (), with_data: bool = False, - indexes = list[Index] + indexes = list[Index] | None = None, ) -> None: super().__init__( schema=schema, signature=name, definition=sql, with_data=with_data, + indexes=indexes, ) self.unique_index_columns = unique_index_columns @@ -87,6 +88,7 @@ def to_pg_view(self, with_data: bool = None) -> PGMaterializedView: signature=self.signature, definition=self.definition, with_data=with_data, + indexes=self.indexes, ) def __repr__(self) -> str: From 2369f34b65c456f47f53d01d7800b8f92aeb3ab7 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Sun, 18 Oct 2026 18:04:33 -0400 Subject: [PATCH 14/23] replace implementation of unique_index_columns Generated-by: GLM 5.3 Low Signed-off-by: Adrian Edwards --- .../application/db/materialized_views.py | 31 +++++++++++++------ .../tasks/db/refresh_materialized_views.py | 2 +- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 0cf573620..84c3f03d9 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -23,10 +23,15 @@ class MaterializedView(PGMaterializedView): """Thin subclass of ``PGMaterializedView`` with CollectOSS-specific helpers. - The constructor uses CollectOSS-style kwargs (``name``, ``sql``, - ``unique_index_columns``) and delegates to ``PGMaterializedView.__init__``, - so every instance *is* a ``PGMaterializedView`` and can be passed directly - to ``alembic_utils_extended.register_entities`` without any conversion. + Indexes are declared as full SQLAlchemy ``Index`` objects (strings for + plain column references), giving complete flexibility (uniqueness, + expressions, opclasses) instead of a bare column list. A unique index is + what makes ``REFRESH MATERIALIZED VIEW ... CONCURRENTLY`` possible; see + ``has_unique_index``. + + Instances must be converted via ``to_pg_view()`` before registration with + ``register_entities``: ``alembic_utils_extended`` performs exact-class + checks that mis-handle subclasses (spurious ``DropOp``s). """ def __init__( @@ -34,9 +39,8 @@ def __init__( name: str, schema: str, sql: str, - unique_index_columns: tuple[str, ...] = (), with_data: bool = False, - indexes = list[Index] | None = None, + indexes: list[Index] | None = None, ) -> None: super().__init__( schema=schema, @@ -45,12 +49,21 @@ def __init__( with_data=with_data, indexes=indexes, ) - self.unique_index_columns = unique_index_columns @property def fqn(self) -> str: return f"{self.schema}.{self.name}" + @property + def unique_indexes(self) -> list[Index]: + """The view's unique indexes (the prerequisite for CONCURRENTLY refresh).""" + return [index for index in (self.indexes or []) if index.unique] + + @property + def has_unique_index(self) -> bool: + """Whether the view can be refreshed CONCURRENTLY.""" + return bool(self.unique_indexes) + def refresh_sql(self, concurrently: bool = True, with_data: bool | None = None) -> str: """Return the SQL to refresh the view. @@ -74,7 +87,7 @@ def to_pg_view(self, with_data: bool = None) -> PGMaterializedView: ``DropOp``s for all materialized views. Returning a genuine ``PGMaterializedView`` sidesteps both issues while - all runtime helpers (``fqn``, ``refresh_sql``, ``unique_index_columns``) + all runtime helpers (``fqn``, ``refresh_sql``, ``has_unique_index``) remain accessible on *self*. Parameters: @@ -95,7 +108,7 @@ def __repr__(self) -> str: # Avoid dumping the full SQL string (some views are 100+ lines) in logs. return ( f"MaterializedView(name={self.name!r}, schema={self.schema!r}, " - f"unique_index_columns={self.unique_index_columns!r})" + f"indexes={[index.name for index in self.indexes or []]!r})" ) diff --git a/collectoss/tasks/db/refresh_materialized_views.py b/collectoss/tasks/db/refresh_materialized_views.py index 0febe34b8..7d1916ae8 100644 --- a/collectoss/tasks/db/refresh_materialized_views.py +++ b/collectoss/tasks/db/refresh_materialized_views.py @@ -20,7 +20,7 @@ def refresh_materialized_views(self): for view in MATERIALIZED_VIEWS: logger.info(f"Refreshing materialized view: {view.fqn}") - if len(view.unique_index_columns) > 0: + if view.has_unique_index: try: conn.execute(s.sql.text(view.refresh_sql(concurrently=True))) continue From 03b88364a8f520001a3fb1dce019e8fa9dab1017 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 30 Oct 2026 17:27:06 -0400 Subject: [PATCH 15/23] first pass at a comparator Generated-by: GLM 5.3 High Signed-off-by: Adrian Edwards --- .../db/matview_index_comparator.py | 285 ++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 collectoss/application/db/matview_index_comparator.py diff --git a/collectoss/application/db/matview_index_comparator.py b/collectoss/application/db/matview_index_comparator.py new file mode 100644 index 000000000..208c413d8 --- /dev/null +++ b/collectoss/application/db/matview_index_comparator.py @@ -0,0 +1,285 @@ +"""Autogenerate comparator for materialized view *index* changes. + +``alembic_utils_extended``'s entity comparator detects view definition changes +and renders index creation alongside create/replace ops +(``render_post_create_entity``), but the registry's ``indexes`` list plays no +role in change *detection*: an index-only change produces an empty migration +(confirmed with the fork maintainer for v1.5.0). + +This module closes that gap as a separate Alembic comparator — the approach +the maintainer suggested — emitting bare ``CreateIndexOp`` / ``DropIndexOp`` +instead of rebuilding the view, so a populated matview keeps its data +(``CREATE INDEX`` on an existing matview is legal; no ``WITH NO DATA`` / +first-refresh-must-be-non-concurrent dance). + +Comparison uses structured signatures — ``(name, unique, ordered column +names)`` — the same approach stock Alembic uses for table indexes. Indexes +bearing expressions (anything other than plain column references) compare by +name only: their DB-side rendering (``pg_get_indexdef``) is not reliably +comparable, so a content change to one requires renaming it or a hand-written +migration. + +Import this module from the alembic ``env.py`` *after* ``register_entities`` +has been called: the entity comparator must be registered first so that, when +a view's definition also changed, its pending CreateOp/ReplaceOp (which +already renders index creation) suppresses this comparator for that view. +Importing the module registers the comparator; there is nothing to call. +""" + +from __future__ import annotations + +import re +from collections import defaultdict +from dataclasses import dataclass + +from alembic.autogenerate import comparators +from alembic.autogenerate.api import AutogenContext +from alembic.operations import ops +from sqlalchemy import Column, text +from sqlalchemy.engine import Connection + +from alembic_utils_extended.pg_materialized_view import PGMaterializedView +from alembic_utils_extended.reversible_op import CreateOp, ReplaceOp + +from collectoss.application.db.materialized_views import MATERIALIZED_VIEWS + + +@dataclass(frozen=True) +class _IndexSig: + """Comparison signature for one index. + + ``columns`` is ``None`` for expression-bearing indexes, which compare by + name only (see module docstring). + """ + + name: str + unique: bool + columns: tuple[str, ...] | None + + +_PLAIN_IDENTIFIER = re.compile(r'^"?(\w+)"?$') + +# "CREATE [UNIQUE] INDEX name ON [schema.]table USING method (cols) ..." +# (pg_get_indexdef always emits the USING clause) +_COL_LIST = re.compile(r"\bON\s+(?:[\w\"]+\.)?[\w\"]+\s+USING\s+\w+\s*\(", re.IGNORECASE) + + +def _local_index_sigs(view: MaterializedView) -> dict[str, _IndexSig]: # noqa: F821 + """Signatures for a registry view's ``Index`` declarations, by name.""" + sigs: dict[str, _IndexSig] = {} + for index in view.indexes or []: + if not index.name: + raise ValueError( + f"Unnamed index on materialized view '{view.schema}.{view.signature}'. " + "All indexes must have a name for autogenerate support." + ) + columns: list[str] | None = [] + for expr in index.expressions: + if isinstance(expr, str): + columns.append(expr.lower()) + elif isinstance(expr, Column): + columns.append(expr.name.lower()) + else: + # Expression index: not reliably comparable, name-only. + columns = None + break + sigs[index.name] = _IndexSig(index.name, bool(index.unique), tuple(columns) if columns else None) + return sigs + + +def _parse_indexdef_columns(indexdef: str) -> tuple[str, ...] | None: + """Extract the ordered plain-column names from a ``pg_get_indexdef`` string. + + Returns ``None`` when the index bears expressions (any element that is + not a plain identifier) or does not parse. + """ + match = _COL_LIST.search(indexdef) + if match is None: + return None + + # Walk from just past the opening paren to its matching close. + start = match.end() + depth = 1 + i = start + while i < len(indexdef) and depth > 0: + if indexdef[i] == "(": + depth += 1 + elif indexdef[i] == ")": + depth -= 1 + i += 1 + col_list = indexdef[start : i - 1] + + # Split on top-level commas only (expressions may contain commas). + elements: list[str] = [] + buf = "" + depth = 0 + for char in col_list: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if char == "," and depth == 0: + elements.append(buf.strip()) + buf = "" + else: + buf += char + if buf.strip(): + elements.append(buf.strip()) + + columns: list[str] = [] + for element in elements: + ident = _PLAIN_IDENTIFIER.match(element) + if ident is None: + return None + columns.append(ident.group(1).lower()) + return tuple(columns) + + +# Matviews cannot have PRIMARY KEYs or table constraints, so — unlike the +# fork's table index comparator — no pg_constraint exclusion is needed. +_DB_INDEX_SQL = text( + """ + SELECT t.relname AS table_name, + c.relname AS index_name, + pg_get_indexdef(i.indexrelid) AS indexdef, + i.indisunique AS is_unique + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_class t ON t.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = :schema + AND t.relname = ANY(:view_names) + """ +) + + +def _db_index_sigs( + connection: Connection, schema: str, view_names: list[str] +) -> dict[str, dict[str, _IndexSig]]: + """Live index signatures per view, from ``pg_index`` directly. + + Returns ``{view_name: {index_name: sig}}``. Views absent from the result + do not exist in the database (the entity comparator handles those). + """ + rows = connection.execute(_DB_INDEX_SQL, {"schema": schema, "view_names": view_names}).fetchall() + + result: dict[str, dict[str, _IndexSig]] = defaultdict(dict) + for row in rows: + result[row.table_name][row.index_name] = _IndexSig( + name=row.index_name, + unique=bool(row.is_unique), + columns=_parse_indexdef_columns(row.indexdef), + ) + return dict(result) + + +@comparators.dispatch_for("schema") +def compare_matview_indexes( + autogen_context: AutogenContext, + upgrade_ops: ops.UpgradeOps, + _schemas: list[str | None], +) -> None: + """Diff registered matview indexes against the live database. + + Emits CreateIndexOp / DropIndexOp for added, removed, renamed, and + changed indexes. A changed index under a stable name is emitted as a + drop + create pair. Views with a pending CreateOp/ReplaceOp from the + entity comparator are skipped: those ops already render index creation + (see module docstring for the import-order requirement this relies on). + """ + connection = autogen_context.connection + if connection is None: + # Offline autogenerate has no live database to diff against. + return + + # Views whose create/replace is already pending from the entity + # comparator — emitting index ops for them here would duplicate the ops + # its rendering produces. + pending_entity_views = { + op.target.signature + for op in upgrade_ops.ops + if isinstance(op, (CreateOp, ReplaceOp)) and isinstance(op.target, PGMaterializedView) + } + views = [view for view in MATERIALIZED_VIEWS if view.signature not in pending_entity_views] + + by_schema: dict[str, list[MaterializedView]] = defaultdict(list) # noqa: F821 + for view in views: + by_schema[view.schema].append(view) + + for schema, schema_views in by_schema.items(): + db_views = _db_index_sigs(connection, schema, [v.signature for v in schema_views]) + + for view in schema_views: + db_sigs = db_views.get(view.signature) + if db_sigs is None: + # View missing from the database and no pending entity op: + # creating indexes now would fail, so leave it alone. + continue + + local_sigs = _local_index_sigs(view) + + for name, local in local_sigs.items(): + db = db_sigs.get(name) + if db is None: + # Added (or renamed-to) index. + upgrade_ops.ops.append( + ops.CreateIndexOp( + index_name=name, + table_name=view.signature, + columns=list(local.columns) if local.columns is not None else [], + schema=schema, + unique=local.unique, + ) + ) + elif (local.unique, local.columns) != (db.unique, db.columns) and not ( + local.columns is None or db.columns is None + ): + # Changed under a stable name (unique flag or column + # list): CREATE INDEX cannot overwrite, so drop + create. + # (If either side is expression-bearing — columns is None + # — the comparison above is name-only; skip.) + upgrade_ops.ops.append( + ops.DropIndexOp( + index_name=name, + table_name=view.signature, + schema=schema, + _reverse=ops.CreateIndexOp( + index_name=name, + table_name=view.signature, + columns=list(db.columns), + schema=schema, + unique=db.unique, + ), + ) + ) + upgrade_ops.ops.append( + ops.CreateIndexOp( + index_name=name, + table_name=view.signature, + columns=list(local.columns), + schema=schema, + unique=local.unique, + ) + ) + + for name, db in db_sigs.items(): + if name not in local_sigs: + # Removed (or renamed-from) index. Columns are attached + # to the reverse op only when parseable, so the + # downgrade recreates it faithfully. + upgrade_ops.ops.append( + ops.DropIndexOp( + index_name=name, + table_name=view.signature, + schema=schema, + _reverse=ops.CreateIndexOp( + index_name=name, + table_name=view.signature, + columns=list(db.columns) if db.columns is not None else [], + schema=schema, + unique=db.unique, + ) + if db.columns is not None + else None, + ) + ) From f97ab20d15dab7e4804edf9422eb470c1e56d464 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 30 Oct 2026 17:54:59 -0400 Subject: [PATCH 16/23] register new comparator in alembic env.py Generated-by: GLM 5.3 High Signed-off-by: Adrian Edwards --- collectoss/application/schema/alembic/env.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/collectoss/application/schema/alembic/env.py b/collectoss/application/schema/alembic/env.py index 90049330a..ff26a3ae6 100644 --- a/collectoss/application/schema/alembic/env.py +++ b/collectoss/application/schema/alembic/env.py @@ -47,6 +47,12 @@ _materialized_view_entities = [view.to_pg_view() for view in MATERIALIZED_VIEWS] register_entities(_materialized_view_entities, entity_types=[PGMaterializedView]) +# Index-only change detection for materialized views, as a separate autogen +# comparator. MUST be imported after register_entities above: the entity +# comparator registers first, so views with a pending create/replace op +# (which already render index creation) are skipped by the index comparator. +from collectoss.application.db import matview_index_comparator # noqa: E402,F401 + # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") From 0f5ccd2568e5689aa70e26703110fb38ae7d9b1c Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 30 Oct 2026 18:32:29 -0400 Subject: [PATCH 17/23] replace noqa hack with an actual import Generated-by: GLM 5.3 High Signed-off-by: Adrian Edwards --- collectoss/application/db/matview_index_comparator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/collectoss/application/db/matview_index_comparator.py b/collectoss/application/db/matview_index_comparator.py index 208c413d8..4a302c6b6 100644 --- a/collectoss/application/db/matview_index_comparator.py +++ b/collectoss/application/db/matview_index_comparator.py @@ -41,7 +41,7 @@ from alembic_utils_extended.pg_materialized_view import PGMaterializedView from alembic_utils_extended.reversible_op import CreateOp, ReplaceOp -from collectoss.application.db.materialized_views import MATERIALIZED_VIEWS +from collectoss.application.db.materialized_views import MATERIALIZED_VIEWS, MaterializedView @dataclass(frozen=True) @@ -64,7 +64,7 @@ class _IndexSig: _COL_LIST = re.compile(r"\bON\s+(?:[\w\"]+\.)?[\w\"]+\s+USING\s+\w+\s*\(", re.IGNORECASE) -def _local_index_sigs(view: MaterializedView) -> dict[str, _IndexSig]: # noqa: F821 +def _local_index_sigs(view: MaterializedView) -> dict[str, _IndexSig]: """Signatures for a registry view's ``Index`` declarations, by name.""" sigs: dict[str, _IndexSig] = {} for index in view.indexes or []: @@ -202,7 +202,7 @@ def compare_matview_indexes( } views = [view for view in MATERIALIZED_VIEWS if view.signature not in pending_entity_views] - by_schema: dict[str, list[MaterializedView]] = defaultdict(list) # noqa: F821 + by_schema: dict[str, list[MaterializedView]] = defaultdict(list) for view in views: by_schema[view.schema].append(view) From 7f37cead833ad3808846c4b206ff799bfc52d150 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 30 Oct 2026 18:33:48 -0400 Subject: [PATCH 18/23] logging of whats happening Generated-by: GLM 5.3 High Signed-off-by: Adrian Edwards --- .../application/db/matview_index_comparator.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/collectoss/application/db/matview_index_comparator.py b/collectoss/application/db/matview_index_comparator.py index 4a302c6b6..621760bf4 100644 --- a/collectoss/application/db/matview_index_comparator.py +++ b/collectoss/application/db/matview_index_comparator.py @@ -28,6 +28,7 @@ from __future__ import annotations +import logging import re from collections import defaultdict from dataclasses import dataclass @@ -43,6 +44,8 @@ from collectoss.application.db.materialized_views import MATERIALIZED_VIEWS, MaterializedView +logger = logging.getLogger(__name__) + @dataclass(frozen=True) class _IndexSig: @@ -222,6 +225,11 @@ def compare_matview_indexes( db = db_sigs.get(name) if db is None: # Added (or renamed-to) index. + logger.info( + "Detected CreateIndexOp for %s.%s", + schema, + name, + ) upgrade_ops.ops.append( ops.CreateIndexOp( index_name=name, @@ -238,6 +246,11 @@ def compare_matview_indexes( # list): CREATE INDEX cannot overwrite, so drop + create. # (If either side is expression-bearing — columns is None # — the comparison above is name-only; skip.) + logger.info( + "Detected changed index %s.%s (drop + create)", + schema, + name, + ) upgrade_ops.ops.append( ops.DropIndexOp( index_name=name, @@ -267,6 +280,11 @@ def compare_matview_indexes( # Removed (or renamed-from) index. Columns are attached # to the reverse op only when parseable, so the # downgrade recreates it faithfully. + logger.info( + "Detected DropIndexOp for %s.%s", + schema, + name, + ) upgrade_ops.ops.append( ops.DropIndexOp( index_name=name, From be04c4df8b4e30b3e07ae4e9a8d28b20cec941a9 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Fri, 30 Oct 2026 18:35:50 -0400 Subject: [PATCH 19/23] render expressions via `render_ddl_sql_expr` like alembic_utils_extended expression-bearing indexes would otherwise render as `op.create_index(..., [])` (broken SQL) Generated-by: GLM 5.3 High Signed-off-by: Adrian Edwards --- .../db/matview_index_comparator.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/collectoss/application/db/matview_index_comparator.py b/collectoss/application/db/matview_index_comparator.py index 621760bf4..3169b13cf 100644 --- a/collectoss/application/db/matview_index_comparator.py +++ b/collectoss/application/db/matview_index_comparator.py @@ -17,7 +17,9 @@ bearing expressions (anything other than plain column references) compare by name only: their DB-side rendering (``pg_get_indexdef``) is not reliably comparable, so a content change to one requires renaming it or a hand-written -migration. +migration. Newly-added expression indexes are still rendered faithfully (via +Alembic's ``render_ddl_sql_expr``, the same mechanism the fork's table index +comparator uses). Import this module from the alembic ``env.py`` *after* ``register_entities`` has been called: the entity comparator must be registered first so that, when @@ -38,6 +40,7 @@ from alembic.operations import ops from sqlalchemy import Column, text from sqlalchemy.engine import Connection +from sqlalchemy.sql.elements import TextClause from alembic_utils_extended.pg_materialized_view import PGMaterializedView from alembic_utils_extended.reversible_op import CreateOp, ReplaceOp @@ -90,6 +93,27 @@ def _local_index_sigs(view: MaterializedView) -> dict[str, _IndexSig]: return sigs +def _index_columns_for_op(index, autogen_context: AutogenContext) -> list[str | TextClause]: + """Render an ``Index``'s elements as ``CreateIndexOp`` columns. + + Plain strings and Columns pass through by name; expressions are rendered + via Alembic's ``render_ddl_sql_expr`` — the same mechanism the fork's + table index comparator uses — so newly-declared expression indexes come + out as ``text(...)`` in the generated migration. + """ + columns: list[str | TextClause] = [] + for expr in index.expressions: + if isinstance(expr, str): + columns.append(expr) + elif isinstance(expr, Column): + columns.append(expr.name) + else: + columns.append( + text(autogen_context.migration_context.impl.render_ddl_sql_expr(expr, is_index=True)) + ) + return columns + + def _parse_indexdef_columns(indexdef: str) -> tuple[str, ...] | None: """Extract the ordered plain-column names from a ``pg_get_indexdef`` string. @@ -220,6 +244,7 @@ def compare_matview_indexes( continue local_sigs = _local_index_sigs(view) + local_indexes_by_name = {index.name: index for index in (view.indexes or [])} for name, local in local_sigs.items(): db = db_sigs.get(name) @@ -234,7 +259,7 @@ def compare_matview_indexes( ops.CreateIndexOp( index_name=name, table_name=view.signature, - columns=list(local.columns) if local.columns is not None else [], + columns=_index_columns_for_op(local_indexes_by_name[name], autogen_context), schema=schema, unique=local.unique, ) @@ -269,7 +294,7 @@ def compare_matview_indexes( ops.CreateIndexOp( index_name=name, table_name=view.signature, - columns=list(local.columns), + columns=_index_columns_for_op(local_indexes_by_name[name], autogen_context), schema=schema, unique=local.unique, ) @@ -293,7 +318,7 @@ def compare_matview_indexes( _reverse=ops.CreateIndexOp( index_name=name, table_name=view.signature, - columns=list(db.columns) if db.columns is not None else [], + columns=list(db.columns), schema=schema, unique=db.unique, ) From e6e6be40ae99b494099ed4ae1b5186020119e084 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Sat, 31 Oct 2026 22:05:44 -0400 Subject: [PATCH 20/23] Transition single-column indexes to the new syntax Unsure about whether the remaining ones should be added individually in multiple indexes or together in one compound index Generated-by: GLM 5.3 Low Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 84c3f03d9..acf206ad5 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -786,20 +786,22 @@ def __repr__(self) -> str: name="issue_reporter_created_at", schema="data", sql=_ISSUE_REPORTER_CREATED_AT, - unique_index_columns=(), # only a non-unique btree on repo_id + # Non-unique btree only: this view cannot be refreshed CONCURRENTLY + # (the refresh task falls back to non-concurrent). + indexes=[Index("issue_reporter_created_at_repo_id_idx", "repo_id")], ), # --- Views 2-6: from migration 4, indexes from migration 25 --- MaterializedView( name="api_get_all_repo_prs", schema="data", sql=_API_GET_ALL_REPO_PRS, - unique_index_columns=("repo_id",), + indexes=[Index("api_get_all_repo_prs_uniq", "repo_id", unique=True)], ), MaterializedView( name="explorer_entry_list", schema="data", sql=_EXPLORER_ENTRY_LIST, - unique_index_columns=("repo_id",), + indexes=[Index("explorer_entry_list_uniq", "repo_id", unique=True)], ), MaterializedView( name="explorer_commits_and_committers_daily_count", @@ -811,13 +813,13 @@ def __repr__(self) -> str: name="api_get_all_repos_commits", schema="data", sql=_API_GET_ALL_REPOS_COMMITS, - unique_index_columns=("repo_id",), + indexes=[Index("api_get_all_repos_commits_uniq", "repo_id", unique=True)], ), MaterializedView( name="api_get_all_repos_issues", schema="data", sql=_API_GET_ALL_REPOS_ISSUES, - unique_index_columns=("repo_id",), + indexes=[Index("api_get_all_repos_issues_uniq", "repo_id", unique=True)], ), # --- Views 6-8: from migration 25, recreated --- MaterializedView( From 876378f6cd4a8882c10f68409fea717cc7b1125c Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 2 Nov 2026 08:48:39 -0500 Subject: [PATCH 21/23] Transition multi-column indexes to the new syntax collectoss-new-schema.sql does these as compound indexes. This is likely quite inefficient and could use review Generated-by: GLM 5.3 Low Signed-off-by: Adrian Edwards --- .../application/db/materialized_views.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index acf206ad5..735146966 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -807,7 +807,7 @@ def __repr__(self) -> str: name="explorer_commits_and_committers_daily_count", schema="data", sql=_EXPLORER_COMMITS_AND_COMMITTERS_DAILY_COUNT, - unique_index_columns=("repo_id", "cmt_committer_date",), + indexes=[Index("explorer_commits_and_committers_daily_count_uniq", "repo_id", "cmt_committer_date", unique=True)], ), MaterializedView( name="api_get_all_repos_commits", @@ -826,56 +826,56 @@ def __repr__(self) -> str: name="augur_new_contributors", schema="data", sql=_AUGUR_NEW_CONTRIBUTORS, - unique_index_columns=("cntrb_id", "created_at", "repo_id", "repo_name", "login", "rank",), + indexes=[Index("augur_new_contributors_uniq", "cntrb_id", "created_at", "repo_id", "repo_name", "login", "rank", unique=True)], ), MaterializedView( name="explorer_contributor_actions", schema="data", sql=_EXPLORER_CONTRIBUTOR_ACTIONS, - unique_index_columns=("cntrb_id", "created_at", "repo_id", "action", "repo_name", "login", "rank",), + indexes=[Index("explorer_contributor_actions_uniq", "cntrb_id", "created_at", "repo_id", "action", "repo_name", "login", "rank", unique=True)], ), MaterializedView( name="explorer_new_contributors", schema="data", sql=_EXPLORER_NEW_CONTRIBUTORS, - unique_index_columns=("cntrb_id", "created_at", "month", "year", "repo_id", "full_name", "repo_name", "login", "rank",), + indexes=[Index("explorer_new_contributors_uniq", "cntrb_id", "created_at", "month", "year", "repo_id", "full_name", "repo_name", "login", "rank", unique=True)], ), # --- Views 9-13: from migration 26 --- MaterializedView( name="explorer_pr_assignments", schema="data", sql=_EXPLORER_PR_ASSIGNMENTS, - unique_index_columns=("pull_request_id", "id", "node_id",), + indexes=[Index("explorer_pr_assignments_uniq", "pull_request_id", "id", "node_id", unique=True)], ), MaterializedView( name="explorer_pr_response", schema="data", sql=_EXPLORER_PR_RESPONSE, - unique_index_columns=("pull_request_id", "id", "cntrb_id", "msg_cntrb_id", "msg_timestamp",), + indexes=[Index("explorer_pr_response_uniq", "pull_request_id", "id", "cntrb_id", "msg_cntrb_id", "msg_timestamp", unique=True)], ), MaterializedView( name="explorer_user_repos", schema="data", sql=_EXPLORER_USER_REPOS, - unique_index_columns=("login_name", "user_id", "group_id", "repo_id",), + indexes=[Index("explorer_user_repos_uniq", "login_name", "user_id", "group_id", "repo_id", unique=True)], ), MaterializedView( name="explorer_pr_response_times", schema="data", sql=_EXPLORER_PR_RESPONSE_TIMES, - unique_index_columns=("repo_id", "pr_src_id", "pr_src_meta_label",), + indexes=[Index("explorer_pr_response_times_uniq", "repo_id", "pr_src_id", "pr_src_meta_label", unique=True)], ), MaterializedView( name="explorer_issue_assignments", schema="data", sql=_EXPLORER_ISSUE_ASSIGNMENTS, - unique_index_columns=("issue_id", "id", "node_id",), + indexes=[Index("explorer_issue_assignments_uniq", "issue_id", "id", "node_id", unique=True)], ), # --- View 15: from migration 28 --- MaterializedView( name="explorer_repo_languages", schema="data", sql=_EXPLORER_REPO_LANGUAGES, - unique_index_columns=("repo_id", "programming_language",), + indexes=[Index("explorer_repo_languages_uniq", "repo_id", "programming_language", unique=True)], ), ] From af750b70851ff88a8e0fd67552362d8ecc6f7814 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 2 Nov 2026 09:01:05 -0500 Subject: [PATCH 22/23] rename indexes to match whats already in the DB Signed-off-by: Adrian Edwards --- .../application/db/materialized_views.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index 735146966..bbc3a0579 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -795,87 +795,87 @@ def __repr__(self) -> str: name="api_get_all_repo_prs", schema="data", sql=_API_GET_ALL_REPO_PRS, - indexes=[Index("api_get_all_repo_prs_uniq", "repo_id", unique=True)], + indexes=[Index("api_get_all_repo_prs_repo_id_idx", "repo_id", unique=True)], ), MaterializedView( name="explorer_entry_list", schema="data", sql=_EXPLORER_ENTRY_LIST, - indexes=[Index("explorer_entry_list_uniq", "repo_id", unique=True)], + indexes=[Index("explorer_entry_list_repo_id_idx", "repo_id", unique=True)], ), MaterializedView( name="explorer_commits_and_committers_daily_count", schema="data", sql=_EXPLORER_COMMITS_AND_COMMITTERS_DAILY_COUNT, - indexes=[Index("explorer_commits_and_committers_daily_count_uniq", "repo_id", "cmt_committer_date", unique=True)], + indexes=[Index("explorer_commits_and_committers__repo_id_cmt_committer_date_idx", "repo_id", "cmt_committer_date", unique=True)], ), MaterializedView( name="api_get_all_repos_commits", schema="data", sql=_API_GET_ALL_REPOS_COMMITS, - indexes=[Index("api_get_all_repos_commits_uniq", "repo_id", unique=True)], + indexes=[Index("api_get_all_repos_commits_repo_id_idx", "repo_id", unique=True)], ), MaterializedView( name="api_get_all_repos_issues", schema="data", sql=_API_GET_ALL_REPOS_ISSUES, - indexes=[Index("api_get_all_repos_issues_uniq", "repo_id", unique=True)], + indexes=[Index("api_get_all_repos_issues_repo_id_idx", "repo_id", unique=True)], ), # --- Views 6-8: from migration 25, recreated --- MaterializedView( name="augur_new_contributors", schema="data", sql=_AUGUR_NEW_CONTRIBUTORS, - indexes=[Index("augur_new_contributors_uniq", "cntrb_id", "created_at", "repo_id", "repo_name", "login", "rank", unique=True)], + indexes=[Index("augur_new_contributors_cntrb_id_created_at_repo_id_repo_nam_idx", "cntrb_id", "created_at", "repo_id", "repo_name", "login", "rank", unique=True)], ), MaterializedView( name="explorer_contributor_actions", schema="data", sql=_EXPLORER_CONTRIBUTOR_ACTIONS, - indexes=[Index("explorer_contributor_actions_uniq", "cntrb_id", "created_at", "repo_id", "action", "repo_name", "login", "rank", unique=True)], + indexes=[Index("explorer_contributor_actions_cntrb_id_created_at_repo_id_ac_idx", "cntrb_id", "created_at", "repo_id", "action", "repo_name", "login", "rank", unique=True)], ), MaterializedView( name="explorer_new_contributors", schema="data", sql=_EXPLORER_NEW_CONTRIBUTORS, - indexes=[Index("explorer_new_contributors_uniq", "cntrb_id", "created_at", "month", "year", "repo_id", "full_name", "repo_name", "login", "rank", unique=True)], + indexes=[Index("explorer_new_contributors_cntrb_id_created_at_month_year_re_idx", "cntrb_id", "created_at", "month", "year", "repo_id", "full_name", "repo_name", "login", "rank", unique=True)], ), # --- Views 9-13: from migration 26 --- MaterializedView( name="explorer_pr_assignments", schema="data", sql=_EXPLORER_PR_ASSIGNMENTS, - indexes=[Index("explorer_pr_assignments_uniq", "pull_request_id", "id", "node_id", unique=True)], + indexes=[Index("explorer_pr_assignments_pull_request_id_id_node_id_idx", "pull_request_id", "id", "node_id", unique=True)], ), MaterializedView( name="explorer_pr_response", schema="data", sql=_EXPLORER_PR_RESPONSE, - indexes=[Index("explorer_pr_response_uniq", "pull_request_id", "id", "cntrb_id", "msg_cntrb_id", "msg_timestamp", unique=True)], + indexes=[Index("explorer_pr_response_pull_request_id_id_cntrb_id_msg_cntrb__idx", "pull_request_id", "id", "cntrb_id", "msg_cntrb_id", "msg_timestamp", unique=True)], ), MaterializedView( name="explorer_user_repos", schema="data", sql=_EXPLORER_USER_REPOS, - indexes=[Index("explorer_user_repos_uniq", "login_name", "user_id", "group_id", "repo_id", unique=True)], + indexes=[Index("explorer_user_repos_login_name_user_id_group_id_repo_id_idx", "login_name", "user_id", "group_id", "repo_id", unique=True)], ), MaterializedView( name="explorer_pr_response_times", schema="data", sql=_EXPLORER_PR_RESPONSE_TIMES, - indexes=[Index("explorer_pr_response_times_uniq", "repo_id", "pr_src_id", "pr_src_meta_label", unique=True)], + indexes=[Index("explorer_pr_response_times_repo_id_pr_src_id_pr_src_meta_la_idx", "repo_id", "pr_src_id", "pr_src_meta_label", unique=True)], ), MaterializedView( name="explorer_issue_assignments", schema="data", sql=_EXPLORER_ISSUE_ASSIGNMENTS, - indexes=[Index("explorer_issue_assignments_uniq", "issue_id", "id", "node_id", unique=True)], + indexes=[Index("explorer_issue_assignments_issue_id_id_node_id_idx", "issue_id", "id", "node_id", unique=True)], ), # --- View 15: from migration 28 --- MaterializedView( name="explorer_repo_languages", schema="data", sql=_EXPLORER_REPO_LANGUAGES, - indexes=[Index("explorer_repo_languages_uniq", "repo_id", "programming_language", unique=True)], + indexes=[Index("explorer_repo_languages_repo_id_programming_language_idx", "repo_id", "programming_language", unique=True)], ), ] From a700a559becde963bf6f04e10c9c34194d675886 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 2 Nov 2026 09:06:51 -0500 Subject: [PATCH 23/23] handle when indexes is None Signed-off-by: Adrian Edwards --- collectoss/application/db/materialized_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectoss/application/db/materialized_views.py b/collectoss/application/db/materialized_views.py index bbc3a0579..7bd4a2e6e 100644 --- a/collectoss/application/db/materialized_views.py +++ b/collectoss/application/db/materialized_views.py @@ -40,7 +40,7 @@ def __init__( schema: str, sql: str, with_data: bool = False, - indexes: list[Index] | None = None, + indexes: list[Index] = [], ) -> None: super().__init__( schema=schema,