From 9c10edab4d6e6d0514b19e98500750595317794f Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Fri, 10 Jul 2026 19:56:11 +0100 Subject: [PATCH 1/8] fix: resolve DOI folder lookups with colons in MaterialsFlow processing --- CHANGELOG.md | 2 ++ docs/about/changelog.md | 2 ++ src/comproscanner/extract_flow/main_extraction_flow.py | 2 +- src/comproscanner/extract_flow/tools/equation_tool.py | 2 +- src/comproscanner/extract_flow/tools/graph_extractor_tool.py | 2 +- src/comproscanner/extract_flow/tools/rag_tool.py | 2 +- 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ba6f7ff..e8f06699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). +- Fixed DOI-to-folder-name lookups in `RAGTool`, `GraphExtractorTool`, `EquationTool`, and `DataExtractionFlow` to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1004776418521` correctly resolve to their saved figure/vector-DB directories. + - Previously, a new `MultiModelEmbeddings` instance (and thus a fresh copy of the PhysBERT model) was loaded onto the GPU for every paper processed, because `RAGTool → VectorDatabaseManager → MultiModelEmbeddings` were all re-instantiated per paper. After certain number of papers this exhausted VRAM with `cudaErrorMemoryAllocation` (Refer to issue [#6](https://github.com/slimeslab/ComProScanner/issues/6)). This fix introduces a class-level `_hf_model_cache` dict on MultiModelEmbeddings so the tokenizer and model are loaded onto the GPU exactly once and shared as references across all subsequent instances. Also explicitly delete intermediate CUDA tensors and call `torch.cuda.empty_cache()` after each embedding call to prevent activation memory from accumulating within a paper's processing. Added the same cache flush in `VectorDatabaseManager.create_database` and `query_database` after `gc.collect()`. This fix is associated with PR [#7](https://github.com/slimeslab/ComProScanner/pull/7). diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 3317088c..c0dfe4be 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -10,6 +10,8 @@ - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). +- Fixed DOI-to-folder-name lookups in `RAGTool`, `GraphExtractorTool`, `EquationTool`, and `DataExtractionFlow` to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1004776418521` correctly resolve to their saved figure/vector-DB directories. + - Previously, a new `MultiModelEmbeddings` instance (and thus a fresh copy of the PhysBERT model) was loaded onto the GPU for every paper processed, because `RAGTool → VectorDatabaseManager → MultiModelEmbeddings` were all re-instantiated per paper. After certain number of papers this exhausted VRAM with `cudaErrorMemoryAllocation` (Refer to issue [#6](https://github.com/slimeslab/ComProScanner/issues/6)). This fix introduces a class-level `_hf_model_cache` dict on MultiModelEmbeddings so the tokenizer and model are loaded onto the GPU exactly once and shared as references across all subsequent instances. Also explicitly delete intermediate CUDA tensors and call `torch.cuda.empty_cache()` after each embedding call to prevent activation memory from accumulating within a paper's processing. Added the same cache flush in `VectorDatabaseManager.create_database` and `query_database` after `gc.collect()`. This fix is associated with PR [#7](https://github.com/slimeslab/ComProScanner/pull/7). --- diff --git a/src/comproscanner/extract_flow/main_extraction_flow.py b/src/comproscanner/extract_flow/main_extraction_flow.py index 5c4aa474..809cc991 100644 --- a/src/comproscanner/extract_flow/main_extraction_flow.py +++ b/src/comproscanner/extract_flow/main_extraction_flow.py @@ -570,7 +570,7 @@ def _check_figures_for_data(self) -> bool: Returns True if any figure contains relevant data, False otherwise. """ doi = self.state.doi - doi_folder = doi.replace("/", "_") + doi_folder = doi.replace("/", "_").replace(":", "_") fig_dir = os.path.join(self.state.related_figures_base_path, doi_folder) if not os.path.isdir(fig_dir): diff --git a/src/comproscanner/extract_flow/tools/equation_tool.py b/src/comproscanner/extract_flow/tools/equation_tool.py index 5addccaa..c23c9344 100644 --- a/src/comproscanner/extract_flow/tools/equation_tool.py +++ b/src/comproscanner/extract_flow/tools/equation_tool.py @@ -252,7 +252,7 @@ def _get_crystal_structure_images(self, doi: str) -> list: list: List of dicts with keys `"caption"` (str) and `"b64"` (str, base64 JPEG). Empty list if the figure directory does not exist or no matching images are found. """ - doi_folder = doi.replace("/", "_") + doi_folder = doi.replace("/", "_").replace(":", "_") fig_dir = os.path.join(self.related_figures_base_path, doi_folder) if not os.path.isdir(fig_dir): diff --git a/src/comproscanner/extract_flow/tools/graph_extractor_tool.py b/src/comproscanner/extract_flow/tools/graph_extractor_tool.py index 8ca88cd3..b6ee84ae 100644 --- a/src/comproscanner/extract_flow/tools/graph_extractor_tool.py +++ b/src/comproscanner/extract_flow/tools/graph_extractor_tool.py @@ -69,7 +69,7 @@ def _run(self, doi: str) -> str: Returns: str: JSON string with extracted data per figure, or an error message. """ - doi_folder = doi.replace("/", "_") + doi_folder = doi.replace("/", "_").replace(":", "_") fig_dir = os.path.join(self.related_figures_base_path, doi_folder) if not os.path.isdir(fig_dir): diff --git a/src/comproscanner/extract_flow/tools/rag_tool.py b/src/comproscanner/extract_flow/tools/rag_tool.py index 0bb746d1..cddd2717 100644 --- a/src/comproscanner/extract_flow/tools/rag_tool.py +++ b/src/comproscanner/extract_flow/tools/rag_tool.py @@ -221,7 +221,7 @@ def _run(self, doi: str, query: str) -> str: logger.info(f"\nDOI: {doi}") logger.info(f"Query: {query}") - db_name = doi.replace("/", "_") + db_name = doi.replace("/", "_").replace(":", "_") logger.info(f"Database name: {db_name}") logger.info(f"Top K: {self.rag_config.rag_top_k}") From b70fef70935dd24b909cf5fe39ef67682bebb422 Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Sun, 12 Jul 2026 12:47:40 +0100 Subject: [PATCH 2/8] fix: sanitize ':' in DOI folder paths across extract_flow crews --- CHANGELOG.md | 2 +- docs/about/changelog.md | 2 +- .../composition_extraction_crew.py | 6 +++--- .../composition_format_crew/composition_format_crew.py | 4 ++-- .../materials_data_identifier_crew.py | 4 ++-- .../synthesis_extraction_crew/synthesis_extraction_crew.py | 4 ++-- .../synthesis_format_crew/synthesis_format_crew.py | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8f06699..b4897ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). -- Fixed DOI-to-folder-name lookups in `RAGTool`, `GraphExtractorTool`, `EquationTool`, and `DataExtractionFlow` to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1004776418521` correctly resolve to their saved figure/vector-DB directories. +- Fixed DOI-to-folder-name conversion across `extract_flow` (`RAGTool`, `GraphExtractorTool`, `EquationTool`, `DataExtractionFlow`, and all crew log/output folder paths) to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1015522900295` no longer raise `WinError 267: The directory name is invalid` on Windows and correctly resolve to their saved figure/vector-DB/log directories. - Previously, a new `MultiModelEmbeddings` instance (and thus a fresh copy of the PhysBERT model) was loaded onto the GPU for every paper processed, because `RAGTool → VectorDatabaseManager → MultiModelEmbeddings` were all re-instantiated per paper. After certain number of papers this exhausted VRAM with `cudaErrorMemoryAllocation` (Refer to issue [#6](https://github.com/slimeslab/ComProScanner/issues/6)). This fix introduces a class-level `_hf_model_cache` dict on MultiModelEmbeddings so the tokenizer and model are loaded onto the GPU exactly once and shared as references across all subsequent instances. Also explicitly delete intermediate CUDA tensors and call `torch.cuda.empty_cache()` after each embedding call to prevent activation memory from accumulating within a paper's processing. Added the same cache flush in `VectorDatabaseManager.create_database` and `query_database` after `gc.collect()`. This fix is associated with PR [#7](https://github.com/slimeslab/ComProScanner/pull/7). diff --git a/docs/about/changelog.md b/docs/about/changelog.md index c0dfe4be..9aaaf6f9 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -10,7 +10,7 @@ - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). -- Fixed DOI-to-folder-name lookups in `RAGTool`, `GraphExtractorTool`, `EquationTool`, and `DataExtractionFlow` to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1004776418521` correctly resolve to their saved figure/vector-DB directories. +- Fixed DOI-to-folder-name conversion across `extract_flow` (`RAGTool`, `GraphExtractorTool`, `EquationTool`, `DataExtractionFlow`, and all crew log/output folder paths) to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1015522900295` no longer raise `WinError 267: The directory name is invalid` on Windows and correctly resolve to their saved figure/vector-DB/log directories. - Previously, a new `MultiModelEmbeddings` instance (and thus a fresh copy of the PhysBERT model) was loaded onto the GPU for every paper processed, because `RAGTool → VectorDatabaseManager → MultiModelEmbeddings` were all re-instantiated per paper. After certain number of papers this exhausted VRAM with `cudaErrorMemoryAllocation` (Refer to issue [#6](https://github.com/slimeslab/ComProScanner/issues/6)). This fix introduces a class-level `_hf_model_cache` dict on MultiModelEmbeddings so the tokenizer and model are loaded onto the GPU exactly once and shared as references across all subsequent instances. Also explicitly delete intermediate CUDA tensors and call `torch.cuda.empty_cache()` after each embedding call to prevent activation memory from accumulating within a paper's processing. Added the same cache flush in `VectorDatabaseManager.create_database` and `query_database` after `gc.collect()`. This fix is associated with PR [#7](https://github.com/slimeslab/ComProScanner/pull/7). diff --git a/src/comproscanner/extract_flow/crews/composition_crew/composition_extraction_crew/composition_extraction_crew.py b/src/comproscanner/extract_flow/crews/composition_crew/composition_extraction_crew/composition_extraction_crew.py index 63a50172..322d151d 100644 --- a/src/comproscanner/extract_flow/crews/composition_crew/composition_extraction_crew/composition_extraction_crew.py +++ b/src/comproscanner/extract_flow/crews/composition_crew/composition_extraction_crew/composition_extraction_crew.py @@ -89,10 +89,10 @@ def __init__( self.output_log_file = None self.task_output_file = None - final_task_output_folder = f"{task_output_folder}/{self.doi.replace('/', '_')}" + final_task_output_folder = f"{task_output_folder}/{self.doi.replace('/', '_').replace(':', '_')}" if self.output_log_folder: final_output_log_folder = ( - f"{output_log_folder}/{self.doi.replace('/', '_')}" + f"{output_log_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_output_log_folder): os.makedirs(final_output_log_folder) @@ -106,7 +106,7 @@ def __init__( ) if self.task_output_folder: final_task_output_folder = ( - f"{task_output_folder}/{self.doi.replace('/', '_')}" + f"{task_output_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_task_output_folder): os.makedirs(final_task_output_folder) diff --git a/src/comproscanner/extract_flow/crews/composition_crew/composition_format_crew/composition_format_crew.py b/src/comproscanner/extract_flow/crews/composition_crew/composition_format_crew/composition_format_crew.py index e9278d33..d32fa1a4 100644 --- a/src/comproscanner/extract_flow/crews/composition_crew/composition_format_crew/composition_format_crew.py +++ b/src/comproscanner/extract_flow/crews/composition_crew/composition_format_crew/composition_format_crew.py @@ -75,7 +75,7 @@ def __init__( if self.output_log_folder: final_output_log_folder = ( - f"{output_log_folder}/{self.doi.replace('/', '_')}" + f"{output_log_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_output_log_folder): os.makedirs(final_output_log_folder) @@ -89,7 +89,7 @@ def __init__( ) if self.task_output_folder: final_task_output_folder = ( - f"{task_output_folder}/{self.doi.replace('/', '_')}" + f"{task_output_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_task_output_folder): os.makedirs(final_task_output_folder) diff --git a/src/comproscanner/extract_flow/crews/materials_data_identifier_crew/materials_data_identifier_crew.py b/src/comproscanner/extract_flow/crews/materials_data_identifier_crew/materials_data_identifier_crew.py index 5c588053..413c95b9 100644 --- a/src/comproscanner/extract_flow/crews/materials_data_identifier_crew/materials_data_identifier_crew.py +++ b/src/comproscanner/extract_flow/crews/materials_data_identifier_crew/materials_data_identifier_crew.py @@ -76,7 +76,7 @@ def __init__( if self.output_log_folder: final_output_log_folder = ( - f"{output_log_folder}/{self.doi.replace('/', '_')}" + f"{output_log_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_output_log_folder): os.makedirs(final_output_log_folder) @@ -90,7 +90,7 @@ def __init__( ) if self.task_output_folder: final_task_output_folder = ( - f"{task_output_folder}/{self.doi.replace('/', '_')}" + f"{task_output_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_task_output_folder): os.makedirs(final_task_output_folder) diff --git a/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_extraction_crew/synthesis_extraction_crew.py b/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_extraction_crew/synthesis_extraction_crew.py index 6222473a..6f076134 100644 --- a/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_extraction_crew/synthesis_extraction_crew.py +++ b/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_extraction_crew/synthesis_extraction_crew.py @@ -73,7 +73,7 @@ def __init__( if self.output_log_folder: final_output_log_folder = ( - f"{output_log_folder}/{self.doi.replace('/', '_')}" + f"{output_log_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_output_log_folder): os.makedirs(final_output_log_folder) @@ -87,7 +87,7 @@ def __init__( ) if self.task_output_folder: final_task_output_folder = ( - f"{task_output_folder}/{self.doi.replace('/', '_')}" + f"{task_output_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_task_output_folder): os.makedirs(final_task_output_folder) diff --git a/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_format_crew/synthesis_format_crew.py b/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_format_crew/synthesis_format_crew.py index 1d0a4b99..e36ce27e 100644 --- a/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_format_crew/synthesis_format_crew.py +++ b/src/comproscanner/extract_flow/crews/synthesis_crew/synthesis_format_crew/synthesis_format_crew.py @@ -72,7 +72,7 @@ def __init__( if self.output_log_folder: final_output_log_folder = ( - f"{output_log_folder}/{self.doi.replace('/', '_')}" + f"{output_log_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_output_log_folder): os.makedirs(final_output_log_folder) @@ -86,7 +86,7 @@ def __init__( ) if self.task_output_folder: final_task_output_folder = ( - f"{task_output_folder}/{self.doi.replace('/', '_')}" + f"{task_output_folder}/{self.doi.replace('/', '_').replace(':', '_')}" ) if not os.path.exists(final_task_output_folder): os.makedirs(final_task_output_folder) From 02ce0db5867219eb1e3116ee89bfb67c10a47ef7 Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Wed, 15 Jul 2026 12:43:50 +0100 Subject: [PATCH 3/8] refactor: replace cleaning_strategy/apply_advanced_cleaning with a unified cleaning_steps parameter --- CHANGELOG.md | 9 + docs/about/changelog.md | 10 + docs/usage/data-cleaning.md | 140 +++--- src/comproscanner/__init__.py | 9 +- src/comproscanner/comproscanner.py | 47 +- .../post_processing/data_cleaner.py | 416 +++++++++++------- .../test_post_processing/test_data_cleaner.py | 305 ++++++++++--- tests/test_public_api.py | 22 +- 8 files changed, 656 insertions(+), 302 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4897ce7..34327db8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,17 @@ - Centralised non-keyword default file paths (`results/failed_automated_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. +### Changed + +- Replaced the `cleaning_strategy` (`"full"`/`"basic"`) and `apply_advanced_cleaning` parameters on `DataCleaner`, `ComProScanner.clean_data()`, and the top-level `clean_data()` function with a single `cleaning_steps` parameter accepting either `"all"` (default) or a list of individually selectable step names: `abbreviation_filtering`, `element_validation`, `text_normalization`, `miller_indices`, `coefficient_expansion` (exposed via the new `CleaningStep` enum). Unicode subscript conversion and arithmetic/fraction resolution remain always-on, since the other steps depend on their output. `normalization` and `zero_coefficient` are no longer separate steps — both are folded into `coefficient_expansion`, which already performs them internally. + + ### Fixed +- Replaced the blind space-stripping behaviour (`key.replace(" ", "")`, which mangled descriptive composition text such as `"Bi4Ti3O12 ultrathin with oxygen vacancies"` into `"Bi4Ti3O12ultrathinwithoxygenvacancies"`) with a new optional `text_normalization` step that strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens, while leaving formula segments and element-symbol sequences untouched. + +- Fixed Miller-index handling in data cleaning: a bare 3-digit parenthetical crystal-plane notation (e.g. `"AlN (002)"`) was previously fed straight into the mandatory arithmetic/bracket resolver, which misread `(002)` as a bare-number coefficient bracket and merged its digits into the preceding element — silently producing `"AlN2"` instead of recognising it as a Miller index at all. The `miller_indices` cleaning step now detects this notation before arithmetic/bracket resolution runs, and — rather than stripping the notation and keeping the bare formula, which would collapse distinct surface-orientation entries for the same material onto the same dict key (e.g. `"AlN (002)"` and `"AlN (110)"` both becoming `"AlN"` and silently overwriting one property value with the other when merged) — drops the whole composition entry, tracking it in `filtered_compositions` the same way `abbreviation_filtering`/`element_validation` drops are tracked. When `miller_indices` is *not* selected, the same 3-digit-bracket detection now also stops both the arithmetic resolver and `coefficient_expansion` from silently merging the digits into a wrong formula; the composition is instead dropped later as an unresolved composition, the same as any other leftover bracket. + - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). - Fixed DOI-to-folder-name conversion across `extract_flow` (`RAGTool`, `GraphExtractorTool`, `EquationTool`, `DataExtractionFlow`, and all crew log/output folder paths) to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1015522900295` no longer raise `WinError 267: The directory name is invalid` on Windows and correctly resolve to their saved figure/vector-DB/log directories. diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 9aaaf6f9..5b0421a1 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -6,8 +6,18 @@ - Centralised non-keyword default file paths (`results/failed_automated_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. + +### Changed + +- Replaced the `cleaning_strategy` (`"full"`/`"basic"`) and `apply_advanced_cleaning` parameters on `DataCleaner`, `ComProScanner.clean_data()`, and the top-level `clean_data()` function with a single `cleaning_steps` parameter accepting either `"all"` (default) or a list of individually selectable step names: `abbreviation_filtering`, `element_validation`, `text_normalization`, `miller_indices`, `coefficient_expansion` (exposed via the new `CleaningStep` enum). Unicode subscript conversion and arithmetic/fraction resolution remain always-on, since the other steps depend on their output. `normalization` and `zero_coefficient` are no longer separate steps — both are folded into `coefficient_expansion`, which already performs them internally. + + ### Fixed +- Replaced the blind space-stripping behaviour (`key.replace(" ", "")`, which mangled descriptive composition text such as `"Bi4Ti3O12 ultrathin with oxygen vacancies"` into `"Bi4Ti3O12ultrathinwithoxygenvacancies"`) with a new optional `text_normalization` step that strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens, while leaving formula segments and element-symbol sequences untouched. + +- Fixed Miller-index handling in data cleaning: a bare 3-digit parenthetical crystal-plane notation (e.g. `"AlN (002)"`) was previously fed straight into the mandatory arithmetic/bracket resolver, which misread `(002)` as a bare-number coefficient bracket and merged its digits into the preceding element — silently producing `"AlN2"` instead of recognising it as a Miller index at all. The `miller_indices` cleaning step now detects this notation before arithmetic/bracket resolution runs, and — rather than stripping the notation and keeping the bare formula, which would collapse distinct surface-orientation entries for the same material onto the same dict key (e.g. `"AlN (002)"` and `"AlN (110)"` both becoming `"AlN"` and silently overwriting one property value with the other when merged) — drops the whole composition entry, tracking it in `filtered_compositions` the same way `abbreviation_filtering`/`element_validation` drops are tracked. When `miller_indices` is *not* selected, the same 3-digit-bracket detection now also stops both the arithmetic resolver and `coefficient_expansion` from silently merging the digits into a wrong formula; the composition is instead dropped later as an unresolved composition, the same as any other leftover bracket. + - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). - Fixed DOI-to-folder-name conversion across `extract_flow` (`RAGTool`, `GraphExtractorTool`, `EquationTool`, `DataExtractionFlow`, and all crew log/output folder paths) to also replace `:` with `_` (not just `/`), so DOIs like `10.1023/A:1015522900295` no longer raise `WinError 267: The directory name is invalid` on Windows and correctly resolve to their saved figure/vector-DB/log directories. diff --git a/docs/usage/data-cleaning.md b/docs/usage/data-cleaning.md index 8562dda7..9de3991b 100644 --- a/docs/usage/data-cleaning.md +++ b/docs/usage/data-cleaning.md @@ -42,17 +42,21 @@ Whether to save composition-property values to a separate file as a dictionary. Path to the cleaned composition-property file containing a dictionary of composition-property data. -#### :material-square-medium:`cleaning_strategy` _(str)_ +#### :material-square-medium:`cleaning_steps` _(Union[str, List[str]])_ -The cleaning strategy to be used. It can be either `full` or `basic`. While comprehensive cleaning including abbreviation removal, arithmetic resolution, bracket standardization, etc., are done for both strategies, the `full` strategy ensures entries with only periodic elements in the composition. +Either the string `"all"` (default, every optional step enabled) or a list of step names selecting exactly which optional steps run: -#### :material-square-medium:`apply_advanced_cleaning` _(bool)_ +- **`abbreviation_filtering`** — drops composition keys containing 2+ consecutive capital letters (abbreviations/junk keys, e.g. `"PVDF"`). +- **`element_validation`** — keep only compositions whose key fully resolves to valid periodic-table element symbols. +- **`text_normalization`** — deterministic cleanup: strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens (e.g. `" Bi4Ti3O12 ultrathin with oxygen vacancies "` → `"Bi4Ti3O12 Ultrathin with Oxygen Vacancies"`). Tokens containing digits (real formula segments) and tokens that fully parse as element symbols (e.g. `"NaCl"`) are left untouched, as are all-caps abbreviations (e.g. `"PVDF"`). +- **`miller_indices`** — drops compositions carrying a crystal-plane notation like `(002)`, `(111)`, `(100)`, etc. entirely, rather than stripping the notation and keeping the bare formula. Stripping and keeping would collapse distinct surface-orientation entries for the same material onto the same dict key — e.g. `"AlN (002)"` and `"AlN (110)"` would both become `"AlN"`, silently overwriting one value with the other when merged. +- **`coefficient_expansion`** — expands leading/trailing/nested bracket coefficient patterns; internally also normalizes trailing zeros and removes zero-coefficient elements as part of expansion. -Flag to indicate if advanced composition cleaning transformations should be applied. When `True`, applies all advanced cleaning processes including Miller indices removal, coefficient expansion, normalization, and zero-coefficient element removal. When `False`, returns basic cleaned compositions only. +Pass an empty list (`cleaning_steps=[]`) to skip all the optional steps. #### :material-square-medium:`is_store_unresolved_compositions` _(bool)_ -When `True`, logs a split statistics line showing `source`, `filtered`, `unresolved`, and `resolved` composition-property pair counts, and saves both filtered compositions (dropped by invalid-key or element-validation checks) and unresolved compositions (still containing parentheses, brackets, or multiplication operators after the full cleaning pipeline) to a JSON file keyed by DOI. Requires `is_save_composition_property_file=True`. +When `True`, logs a split statistics line showing `source`, `filtered`, `unresolved`, and `resolved` composition-property pair counts, and saves both filtered compositions (dropped by `abbreviation_filtering`, `element_validation`, or `miller_indices`) and unresolved compositions (still containing parentheses, brackets, or multiplication operators after cleaning) to a JSON file keyed by DOI. Requires `is_save_composition_property_file=True`. #### :material-square-medium:`unresolved_compositions_file` _(str)_ @@ -60,7 +64,7 @@ Path to the JSON file where filtered and unresolved composition keys are saved, !!! info "Default Values" - :material-square-small:**`is_save_separate_results`** = True
:material-square-small:**`cleaned_json_results_file`** = "cleaned_results.json"
:material-square-small:**`is_save_composition_property_file`** = True
:material-square-small:**`composition_property_file`** = "composition_property.json"
:material-square-small:**`cleaning_strategy`** = "full"
:material-square-small:**`apply_advanced_cleaning`** = True
:material-square-small:**`is_store_unresolved_compositions`** = False
:material-square-small:**`unresolved_compositions_file`** = "unresolved_compositions.json" + :material-square-small:**`is_save_separate_results`** = True
:material-square-small:**`cleaned_json_results_file`** = "cleaned_results.json"
:material-square-small:**`is_save_composition_property_file`** = True
:material-square-small:**`composition_property_file`** = "composition_property.json"
:material-square-small:**`cleaning_steps`** = "all"
:material-square-small:**`is_store_unresolved_compositions`** = False
:material-square-small:**`unresolved_compositions_file`** = "unresolved_compositions.json" ## Cleaning Process Flow @@ -68,85 +72,95 @@ The data cleaning process follows this workflow: ```mermaid graph TD - A[Start: Raw Extracted Data] --> B[Basic Validation] - B --> C[Element Validation] - C --> D[Unicode Conversion] - D --> E[Arithmetic Resolution] - E --> F{apply_advanced_cleaning?} - - F -->|True| G[Advanced Cleaning Pipeline] - F -->|False| M[Basic Cleaned Data] - - G --> H[Miller Indices Removal] - H --> I[Coefficient Expansion] - I --> J[Coefficient Normalization] - J --> K[Zero-Coefficient Removal] - K --> L[Resolved Composition Data] - - M --> N[End: Cleaned Results] - L --> N + A[Start: Raw Extracted Data] --> B{abbreviation_filtering?} + B -->|selected| B1[Abbreviation Filtering] + B -->|skipped| C + B1 --> C{element_validation?} + C -->|selected| C1[Element Validation] + C -->|skipped| D + C1 --> D{text_normalization?} + D -->|selected| D1[Text Normalization] + D -->|skipped| M + D1 --> M{miller_indices?} + M -->|selected| M1[Drop compositions with Miller indices] + M -->|skipped| F + M1 --> F[Unicode Conversion - always] + F --> G[Arithmetic/Fraction Resolution - always] + G --> I{coefficient_expansion?} + I -->|selected| I1[Coefficient Expansion - incl. normalization and zero-coefficient removal] + I -->|skipped| J + I1 --> J[End: Cleaned Results] style A fill:#e1f5ff - style N fill:#e7f5e1 + style J fill:#e7f5e1 + style F fill:#fff4e1 style G fill:#fff4e1 - style H fill:#ffe1f5 - style I fill:#ffe1f5 - style J fill:#ffe1f5 - style K fill:#ffe1f5 ``` +!!! note "miller_indices drops entries, it does not strip-and-keep" + + `miller_indices` removes the **entire composition entry**, not just the `(002)`-style notation. Stripping the notation and keeping the bare formula would collapse distinct surface-orientation measurements for the same material onto the same dict key — e.g. `"AlN (002)": 3` and `"AlN (110)": 6` would both resolve to `"AlN"`, and merging the results (`_return_in_dict`) would silently overwrite one value with the other. Dropping both entirely avoids that data loss; dropped keys are tracked in `filtered_compositions`, the same as `abbreviation_filtering`/`element_validation` drops. + + This detection must also happen **before** the mandatory arithmetic/bracket resolution: a bare 3-digit parenthetical like `(002)` is indistinguishable from a "coefficient bracket" to that resolver and to `coefficient_expansion`'s own bracket-stripping logic — left unremoved, either would fold its digits straight into the preceding element (e.g. `AlN (002)` → `AlN2`/`AlN 002`). Running the `miller_indices` check first, while the brackets are still in their original shape, avoids this. If `miller_indices` is **not** selected, a bare 3-digit-only bracket is instead left untouched by both of those steps and is dropped later as an unresolved composition, the same as any other leftover bracket — so it is never silently merged into a wrong formula either way. + ### Process Stages -#### 1. Basic Validation +#### 1. Abbreviation Filtering _(optional — `abbreviation_filtering`)_ -Removes invalid keys, abbreviations, and special characters from compositions. +Drops composition keys containing 2+ consecutive capital letters (abbreviations, junk keys). -#### 2. Element Validation +#### 2. Element Validation _(optional — `element_validation`)_ -Verifies compositions contain only valid periodic elements (for `full` strategy only). +Verifies compositions contain only valid periodic elements. -#### 3. Unicode Conversion +#### 3. Text Normalization _(optional — `text_normalization`)_ -Converts subscript Unicode characters to regular digits for standardization. +Strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens, leaving formula segments (tokens with digits, or tokens that fully parse as element symbols) and all-caps abbreviations untouched. -#### 4. Arithmetic Resolution +#### 4. Miller Indices Filtering _(optional — `miller_indices`)_ -Evaluates mathematical expressions and fractional compositions. +Drops any composition entry carrying a crystal plane notation like `(002)`, `(111)`, `(100)`, etc. entirely (not just the notation). Runs before Unicode/arithmetic resolution — see the note above. -#### 5. Advanced Cleaning Pipeline +#### 5. Unicode Conversion _(always runs)_ -When `apply_advanced_cleaning=True`, the following sub-processes are executed sequentially: +Converts subscript Unicode characters to regular digits for standardization. -##### Miller Indices Removal +#### 6. Arithmetic Resolution _(always runs)_ -Removes crystal plane notations like `(002)`, `(111)`, `(100)`, etc. from chemical formulas. +Evaluates mathematical expressions and fractional compositions. -##### Coefficient Expansion +#### 7. Coefficient Expansion _(optional — `coefficient_expansion`)_ -Expands coefficient patterns in chemical formulas including: +Expands coefficient patterns in chemical formulas, including: - **Leading coefficients**: Multiplies all elements inside parentheses by leading coefficient - **Trailing coefficients**: Multiplies all elements inside parentheses by trailing coefficient - **Parenthetical coefficients**: Expands nested brackets with complex coefficient multiplication -##### Coefficient Normalization +It also normalizes trailing zeros and removes zero-coefficient elements internally as part of expansion — there are no separate steps for those. -Removes trailing zeros from element coefficients for cleaner representation. +## Text Normalization Examples -##### Zero-Coefficient Removal +Normalizes whitespace and title-cases descriptive words; formula segments are untouched: -Removes elements with coefficient values of 0 or 0.0 from formulas. +| Input Composition | Output Composition | +| --- | --- | +| `Bi4Ti3O12 ultrathin with oxygen vacancies` | `Bi4Ti3O12 Ultrathin with Oxygen Vacancies` | +| ` Bi4Ti3O12 ultrathin with oxygen vacancies ` | `Bi4Ti3O12 Ultrathin with Oxygen Vacancies` | +| `BaTiO3 XRD pattern` | `BaTiO3 XRD Pattern` | +| `NaCl` | `NaCl` (unchanged — fully parses as elements) | ## Advanced Cleaning Examples -### Miller Indices Removal +### Miller Indices Filtering -Removes crystal plane notations from chemical formulas: +Drops any composition entry carrying a crystal plane notation entirely — it is not stripped down to the bare formula, since that would collapse distinct surface-orientation entries for the same material onto the same key: -| Input Formula | Output Formula | -| ------------- | -------------- | -| `AlN (002)` | `AlN` | -| `ZnO (101)` | `ZnO` | +| Input Composition | Result | +| ------------------ | ------ | +| `AlN (002)` | dropped (tracked in `filtered_compositions`) | +| `ZnO (101)` | dropped (tracked in `filtered_compositions`) | +| `BaTiO3` | kept, unchanged | ### Coefficient Expansion @@ -173,31 +187,15 @@ Multiplies all elements inside parentheses by the coefficient after the closing Handles nested brackets and complex coefficient multiplication: | Input Formula | Output Formula | -| ----------------------------- | ----------------------- | +| ----------------------------- | ------------------------ | | `[(K0.5Na0.5)0.96Bi0.04]NbO3` | `K0.48Na0.48Bi0.04NbO3` | | `[Ba0.85Ca0.15]0.99TiO3` | `Ba0.8415Ca0.1485TiO3` | -### Coefficient Normalization - -Removes trailing zeros from element coefficients: - -| Input Formula | Output Formula | -| ------------------ | -------------- | -| `Pb0.90La0.10` | `Pb0.9La0.1` | -| `Zr0.200Ti0.800O2` | `Zr0.2Ti0.8O2` | - -### Zero-Coefficient Element Removal - -Removes elements with zero coefficients: - -| Input Formula | Output Formula | -| ---------------- | -------------- | -| `BaTiZr0O3` | `BaTiO3` | -| `K0.5Na0.5Nb0O3` | `K0.5Na0.5O3` | +Coefficient expansion also normalizes trailing zeros (`Pb0.90La0.10` → `Pb0.9La0.1`) and removes zero-coefficient elements (`BaTiZr0O3` → `BaTiO3`) internally as part of the same step. !!! tip "Original vs Resolved Compositions" - The advanced cleaning process transforms raw extracted compositions into standardized, resolved forms. Both versions can be preserved for traceability in custom implementations using the `DataCleaner` class directly with the `apply_advanced_cleaning` parameter. This allows you to maintain both the original extracted composition (for reference and validation) and the fully resolved composition (for analysis and database storage). + The optional cleaning steps transform raw extracted compositions into standardized, resolved forms. Both versions can be preserved for traceability in custom implementations using the `DataCleaner` class directly with the `cleaning_steps` parameter. This allows you to maintain both the original extracted composition (for reference and validation) and the fully resolved composition (for analysis and database storage). ## Next Steps diff --git a/src/comproscanner/__init__.py b/src/comproscanner/__init__.py index cd5a23f5..96c8a16e 100644 --- a/src/comproscanner/__init__.py +++ b/src/comproscanner/__init__.py @@ -150,7 +150,7 @@ def clean_data( cleaned_json_results_file: str = "cleaned_results.json", is_save_composition_property_file: bool = True, composition_property_file: str = "composition_property.json", - cleaning_strategy: str = "full", + cleaning_steps="all", ): """ Clean the extracted composition-property data. @@ -162,7 +162,10 @@ def clean_data( cleaned_json_results_file (str, optional): Path to the cleaned JSON results file with articles having relevant composition-property data. Defaults to "cleaned_results.json". is_save_composition_property_file (bool, optional): Whether to save composition-property values to a separate file. Defaults to True. composition_property_file (str, optional): Path to the composition-property file containing a dictionary of composition-property data. Defaults to "composition_property.json". - cleaning_strategy (str, optional): The cleaning strategy to use. Defaults to "full" (with periodic element validation). "basic" (without periodic element validation) is the other option. + cleaning_steps (Union[str, List[str]], optional): Either "all" (default, every optional step + enabled) or a list of step names: "abbreviation_filtering", "element_validation", + "text_normalization", "miller_indices", "coefficient_expansion". Unicode subscript + conversion and arithmetic/fraction resolution always run regardless of this parameter. """ scanner = ComProScanner(main_property_keyword=main_property_keyword) return scanner.clean_data( @@ -171,7 +174,7 @@ def clean_data( cleaned_json_results_file=cleaned_json_results_file, is_save_composition_property_file=is_save_composition_property_file, composition_property_file=composition_property_file, - cleaning_strategy=cleaning_strategy, + cleaning_steps=cleaning_steps, ) def evaluate_semantic( diff --git a/src/comproscanner/comproscanner.py b/src/comproscanner/comproscanner.py index 17e9076d..8a9d6f32 100644 --- a/src/comproscanner/comproscanner.py +++ b/src/comproscanner/comproscanner.py @@ -29,7 +29,7 @@ from .utils.get_paper_data import PaperMetadataExtractor from .utils.save_results import SaveResults from .post_processing.data_cleaner import ( - CleaningStrategy, + CleaningStep, DataCleaner, ) @@ -721,8 +721,7 @@ def clean_data( cleaned_json_results_file: str = "cleaned_results.json", is_save_composition_property_file: bool = True, composition_property_file: str = "composition_property.json", - cleaning_strategy: str = "full", - apply_advanced_cleaning: bool = True, + cleaning_steps: Union[str, List[str]] = "all", is_store_unresolved_compositions: bool = False, unresolved_compositions_file: str = "unresolved_compositions.json", ) -> Tuple[Dict[str, Any], Dict[str, Any]]: @@ -735,14 +734,23 @@ def clean_data( cleaned_json_results_file (str, optional): Path to the cleaned JSON results file with articles having relevant composition-property data. Defaults to "cleaned_results.json". is_save_composition_property_file (bool, optional): Whether to save composition-property values to a separate file. Defaults to True. composition_property_file (str, optional): Path to the composition-property file containing a dictionary of composition-property data. Defaults to "composition_property.json". - cleaning_strategy (str, optional): The cleaning strategy to use. Defaults to "full" (with periodic element validation). "basic" (without periodic element validation) is the other option. - apply_advanced_cleaning (bool, optional): Whether to apply advanced composition cleaning transformations (Miller indices removal, coefficient expansion, normalization, zero-coefficient removal). Defaults to True. + cleaning_steps (Union[str, List[str]], optional): Either "all" (default, every optional step + enabled) or a list of step names selecting exactly which optional steps run: + "abbreviation_filtering" (drop keys with 2+ consecutive capital letters), + "element_validation" (keep only compositions resolving to valid periodic-table elements), + "text_normalization" (normalizes whitespace and title-cases descriptive word tokens), + "miller_indices" (drop compositions carrying crystal-plane notations like "(002)" entirely, + to avoid collapsing distinct surface-orientation entries onto the same key), and + "coefficient_expansion" (expand bracket coefficients; also normalizes trailing zeros and + removes zero-coefficient elements internally). Unicode subscript conversion and + arithmetic/fraction resolution always run regardless of this parameter, since the other + steps depend on their output. is_store_unresolved_compositions (bool, optional): Whether to log resolution statistics and save unresolved composition keys to a file. Requires is_save_composition_property_file=True. Defaults to False. unresolved_compositions_file (str, optional): Path to the file where unresolved composition keys will be saved. Used only when is_store_unresolved_compositions=True. Defaults to "unresolved_compositions.json". Returns: tuple: A tuple containing: - - Dict[str, Any]: Cleaned data based on selected strategy with relevant composition-property data. + - Dict[str, Any]: Cleaned data based on selected steps with relevant composition-property data. - Dict[str, Any]: All composition-property values collected from the cleaned data. (Returned only if is_save_composition_property_file is True) """ if json_results_file is None: @@ -759,13 +767,23 @@ def clean_data( raise ValueErrorHandler( message=f"JSON results file {json_results_file} does not exist. Cannot proceed with data cleaning." ) - if cleaning_strategy not in [CleaningStrategy.FULL, CleaningStrategy.BASIC]: - logger.error( - f"Invalid cleaning strategy: {cleaning_strategy}. Please choose either 'full' or 'basic'." - ) - raise ValueErrorHandler( - message=f"Invalid cleaning strategy: {cleaning_strategy}. Please choose either 'full' or 'basic'." - ) + if cleaning_steps != "all": + if not isinstance(cleaning_steps, list) or not all( + isinstance(s, str) for s in cleaning_steps + ): + logger.error( + "Invalid cleaning_steps: must be 'all' or a list of step name strings." + ) + raise ValueErrorHandler( + message="Invalid cleaning_steps: must be 'all' or a list of step name strings." + ) + unknown_steps = set(cleaning_steps) - set(CleaningStep.all()) + if unknown_steps: + logger.error(f"Invalid cleaning step(s): {sorted(unknown_steps)}.") + raise ValueErrorHandler( + message=f"Invalid cleaning step(s): {sorted(unknown_steps)}. " + f"Valid options are {CleaningStep.all()}." + ) data_cleaner = DataCleaner(results_file=json_results_file) if is_store_unresolved_compositions and is_save_composition_property_file: source_composition_count = sum( @@ -778,8 +796,7 @@ def clean_data( if isinstance(article_data, dict) ) final_data = data_cleaner.clean_data_with_relevant_compositions( - strategy=cleaning_strategy, - apply_advanced_cleaning=apply_advanced_cleaning, + cleaning_steps=cleaning_steps, ) # Save the cleaned data back to the cleaned JSON file if is_save_separate_results: diff --git a/src/comproscanner/post_processing/data_cleaner.py b/src/comproscanner/post_processing/data_cleaner.py index 5ef6fb82..fcae2837 100644 --- a/src/comproscanner/post_processing/data_cleaner.py +++ b/src/comproscanner/post_processing/data_cleaner.py @@ -9,7 +9,7 @@ # Standard library imports import json -from typing import List, Dict, Any +from typing import List, Dict, Any, Union, Set import re from enum import Enum import copy @@ -24,11 +24,24 @@ def get_all_elements() -> List[str]: return [Element.from_Z(i).symbol for i in range(1, 119)] -class CleaningStrategy(str, Enum): - """Cleaning strategies for data cleaning.""" +class CleaningStep(str, Enum): + """Individually selectable optional data-cleaning steps. - BASIC = "basic" # Without element validation - FULL = "full" # With element validation + Unicode subscript conversion and arithmetic/fraction resolution are not + part of this enum — they always run, since coefficient_expansion, + miller_indices, and element_validation all assume their output. + """ + + ABBREVIATION_FILTERING = "abbreviation_filtering" + ELEMENT_VALIDATION = "element_validation" + TEXT_NORMALIZATION = "text_normalization" + MILLER_INDICES = "miller_indices" + COEFFICIENT_EXPANSION = "coefficient_expansion" + + @classmethod + def all(cls) -> List[str]: + """Return every valid step name.""" + return [step.value for step in cls] class DataCleaner: @@ -70,6 +83,29 @@ def _filter_invalid_keys(self, dict_list, doi: str = ""): valid.append(d) return valid + def _can_parse_as_elements(self, s: str) -> bool: + """ + Check if a purely-alphabetic string can be completely parsed as a sequence + of valid element symbols. Uses greedy matching, preferring 2-letter + elements over 1-letter (e.g. "Pb", "Sr" before falling back to "P", "O"). + """ + if not s: + return True + + if len(s) >= 2: + two_letter = s[:2] + if two_letter in self.all_elements: + if self._can_parse_as_elements(s[2:]): + return True + + if len(s) >= 1: + one_letter = s[:1] + if one_letter in self.all_elements: + if self._can_parse_as_elements(s[1:]): + return True + + return False + def _is_elements(self, comp_pro_pair: Dict[str, Any]) -> bool: """Check whether the composition key in a pair can be fully parsed as a sequence of valid element symbols. @@ -79,6 +115,7 @@ def _is_elements(self, comp_pro_pair: Dict[str, Any]) -> bool: Returns: bool: True if the key resolves to a valid chemical composition, False otherwise. """ + def _convert_subscript_unicode(string: str) -> str: """Convert Unicode subscript digits to regular digits.""" subscript_unicode = { @@ -108,33 +145,6 @@ def _remove_special_chars(string: str) -> str: # Then remove all non-alphabetic characters return re.sub(r"[^a-zA-Z]+", "", string) - def _can_parse_as_elements(s: str, elements: List[str]) -> bool: - """ - Check if string can be completely parsed as a sequence of valid element symbols. - Uses greedy matching, preferring 2-letter elements over 1-letter. - """ - if not s: - return True - - # Try to match 2-letter element first (e.g., "Pb", "Sr") - if len(s) >= 2: - two_letter = s[:2] - if two_letter in elements: - # Recursively check the rest - if _can_parse_as_elements(s[2:], elements): - return True - - # Try to match 1-letter element (e.g., "P", "O") - if len(s) >= 1: - one_letter = s[:1] - if one_letter in elements: - # Recursively check the rest - if _can_parse_as_elements(s[1:], elements): - return True - - # Cannot parse this string - return False - try: key = next(iter(comp_pro_pair)) # Get the key key = _remove_special_chars(str(key)) @@ -144,56 +154,116 @@ def _can_parse_as_elements(s: str, elements: List[str]) -> bool: return False # CRITICAL FIX: Verify that the entire string can be parsed as valid elements - return _can_parse_as_elements(key, self.all_elements) + return self._can_parse_as_elements(key) except Exception: return False - def _remove_extra_spaces(self, dict_list): - """Remove all spaces from composition keys in a list of {composition: value} dicts. + _TITLE_CASE_STOPWORDS = {"with", "of", "at", "for", "the", "and"} + + def _normalize_text(self, dict_list): + """Normalize whitespace and title-case descriptive word tokens in composition keys. + + Strips leading/trailing whitespace and collapses runs of whitespace + (multiple spaces, tabs) down to a single space, then title-cases + tokens that are purely alphabetic, not already an all-caps + abbreviation (e.g. "XRD" stays "XRD"), and do NOT fully parse as a + sequence of element symbols (so formula-only tokens like "NaCl" or + "Ti" are left exactly as extracted). Tokens containing digits (real + formula segments like "Bi4Ti3O12") are left untouched. Does not + insert a space where none exists in the source string (e.g. words + glued directly to numbers, like "Milling15h", stay glued). Args: dict_list (list): List of single-entry dicts mapping composition strings to values. Returns: - list: Same structure with spaces stripped from every key. + list: Same structure with whitespace normalized and descriptive + word tokens title-cased. """ - # remove any spaces in the key + + def format_key(key: str) -> str: + tokens = key.strip().split() + formatted = [] + for i, tok in enumerate(tokens): + if ( + tok.isalpha() + and not tok.isupper() + and not self._can_parse_as_elements(tok) + ): + lower = tok.lower() + if i > 0 and lower in self._TITLE_CASE_STOPWORDS: + formatted.append(lower) + else: + formatted.append(tok.capitalize()) + else: + formatted.append(tok) + return " ".join(formatted) + return [ - {key.replace(" ", ""): value for key, value in d.items()} for d in dict_list + {format_key(str(key)): value for key, value in d.items()} for d in dict_list ] - def _clean_comp_prop_data_with_element_check( - self, comp_prop_data: Dict[str, Any], doi: str = "" - ) -> Dict[str, Any]: - """Clean composition-property data with element validation from periodic table.""" - comp_prop_data = self._get_comp_prop_pairs(comp_prop_data) - comp_prop_data = self._filter_invalid_keys(comp_prop_data, doi) - valid_comp_prop_pairs = [] - for single_data in comp_prop_data: - if self._is_elements(single_data): - valid_comp_prop_pairs.append(single_data) - else: - if doi: - self.filtered_compositions.setdefault(doi, []).extend(single_data.keys()) - valid_comp_prop_pairs = self._remove_extra_spaces(valid_comp_prop_pairs) - valid_comp_prop_pairs = self._convert_fractions_and_resolve_compositions( - valid_comp_prop_pairs - ) - return valid_comp_prop_pairs + def _clean_comp_prop_pairs( + self, + comp_prop_data: Dict[str, Any], + steps: Set[str], + doi: str = "", + ) -> List[Dict[str, Any]]: + """Clean composition-property pairs according to the selected optional steps. - def _clean_comp_prop_data_without_element_check( - self, comp_prop_data: Dict[str, Any], doi: str = "" - ) -> Dict[str, Any]: - """Clean composition-property data without element validation.""" - comp_prop_data = self._get_comp_prop_pairs(comp_prop_data) - comp_prop_data = self._filter_invalid_keys(comp_prop_data, doi) - valid_comp_prop_pairs = comp_prop_data - valid_comp_prop_pairs = self._remove_extra_spaces(valid_comp_prop_pairs) - valid_comp_prop_pairs = self._convert_fractions_and_resolve_compositions( - valid_comp_prop_pairs + Args: + comp_prop_data (Dict[str, Any]): Raw composition -> property-value mapping. + steps (Set[str]): Selected optional CleaningStep values. + doi (str, optional): DOI used to track filtered compositions. + + Returns: + List[Dict[str, Any]]: List of single-entry {composition: value} dicts after + cleaning. Unicode conversion and arithmetic resolution always run, + regardless of which optional steps are selected. + """ + comp_prop_pairs = self._get_comp_prop_pairs(comp_prop_data) + + if CleaningStep.ABBREVIATION_FILTERING.value in steps: + comp_prop_pairs = self._filter_invalid_keys(comp_prop_pairs, doi) + + if CleaningStep.ELEMENT_VALIDATION.value in steps: + valid_pairs = [] + for pair in comp_prop_pairs: + if self._is_elements(pair): + valid_pairs.append(pair) + elif doi: + self.filtered_compositions.setdefault(doi, []).extend(pair.keys()) + comp_prop_pairs = valid_pairs + + if CleaningStep.TEXT_NORMALIZATION.value in steps: + comp_prop_pairs = self._normalize_text(comp_prop_pairs) + + if CleaningStep.MILLER_INDICES.value in steps: + # Drop (not transform) compositions carrying a crystal-plane notation. + # Stripping "(002)"/"(110)" and keeping the bare formula would collapse + # distinct surface-orientation entries for the same material down to + # the same dict key — e.g. "AlN (002)" and "AlN (110)" would both + # become "AlN", silently overwriting one another when merged. Must + # run before arithmetic/bracket resolution below: the mandatory + # bracket resolver would otherwise treat the bare, purely-numeric + # parenthetical as a coefficient to fold into the formula. + kept_pairs = [] + for d in comp_prop_pairs: + key = next(iter(d)) + if self._remove_miller_indices(key) != key: + if doi: + self.filtered_compositions.setdefault(doi, []).append(key) + else: + kept_pairs.append(d) + comp_prop_pairs = kept_pairs + + # Mandatory, always runs regardless of `steps` — later steps depend on this output. + comp_prop_pairs = self._convert_fractions_and_resolve_compositions( + comp_prop_pairs ) - return valid_comp_prop_pairs + + return comp_prop_pairs def _convert_fractions_and_resolve_compositions(self, dict_list): """ @@ -258,6 +328,15 @@ def _evaluate_all_parenthetical_expressions(formula): # Skip if preceded by * so _multiply_pure_number_coefficients can handle it. if match.start() > 0 and formula[match.start() - 1] == "*": continue + # Skip bare 3-digit integers (no decimal point) — these are + # Miller-index-shaped, e.g. "(002)". If the miller_indices + # step is selected it already removed these earlier in the + # pipeline; if not, silently merging the digits into the + # preceding element (AlN (002) -> AlN2) would be wrong + # regardless, so leave the bracket untouched here — it then + # gets caught as "unresolved" downstream. + if re.match(r"^[0-9]{3}$", expression): + continue formula = ( formula[: match.start()] + expression @@ -413,7 +492,9 @@ def _replace_element_mult(match): return f"{element}{int(result_coeff)}" else: formatted = f"{result_coeff:.8f}".rstrip("0").rstrip(".") - return f"{element}{formatted}" if formatted not in ("0", "0.") else "" + return ( + f"{element}{formatted}" if formatted not in ("0", "0.") else "" + ) return re.sub(element_mult_pattern, _replace_element_mult, formula) @@ -558,7 +639,9 @@ def _convert_subscript_unicode_to_digits(string: str) -> str: processed_key = _convert_subscript_unicode_to_digits(str(key)) # Step 1: Convert simple fractions to decimals (handles both integer and decimal numerators/denominators) - processed_key = re.sub(r"(\d+(?:\.\d+)?)/(\d+(?:\.\d+)?)", _replace_fraction, processed_key) + processed_key = re.sub( + r"(\d+(?:\.\d+)?)/(\d+(?:\.\d+)?)", _replace_fraction, processed_key + ) # Step 2: Resolve compositions processed_key = _resolve_composition(processed_key) @@ -813,10 +896,14 @@ def replace_coeff(match): if coefficient == 0: formula = formula[: match.start()] + formula[close_pos + 1 :] else: - expanded = multiply_element_coefficients(composition, coefficient) + expanded = multiply_element_coefficients( + composition, coefficient + ) # If expansion results in empty string, remove the brackets entirely if not expanded or expanded.strip() == "": - formula = formula[: match.start()] + formula[close_pos + 1 :] + formula = ( + formula[: match.start()] + formula[close_pos + 1 :] + ) else: formula = ( formula[: match.start()] @@ -951,7 +1038,9 @@ def _expand_parenthetical_coefficients( inner_coefficient = ( float(coefficient_str) if coefficient_str else 1.0 ) - new_coefficient = round(inner_coefficient * outer_coefficient, 5) + new_coefficient = round( + inner_coefficient * outer_coefficient, 5 + ) # Skip elements with coefficient 0 if new_coefficient == 0: @@ -962,9 +1051,9 @@ def _expand_parenthetical_coefficients( elif new_coefficient == int(new_coefficient): expanded += f"{element}{int(new_coefficient)}" else: - formatted_coeff = f"{new_coefficient:.4f}".rstrip("0").rstrip( - "." - ) + formatted_coeff = f"{new_coefficient:.4f}".rstrip( + "0" + ).rstrip(".") expanded += f"{element}{formatted_coeff}" formula = ( @@ -977,9 +1066,17 @@ def _expand_parenthetical_coefficients( # Step 2: Remove brackets without coefficients # Make sure we don't match if there's a * followed by a number - no_coeff_match = re.search( + # Skip bare 3-digit integers (Miller-index-shaped, e.g. "(002)") + # — those must be handled by the miller_indices step, not + # silently stripped here. + no_coeff_match = None + for candidate in re.finditer( r"[\[\(]([A-Za-z0-9.]+)[\]\)](?![\*\d.])", formula - ) + ): + if re.match(r"^[0-9]{3}$", candidate.group(1)): + continue + no_coeff_match = candidate + break if no_coeff_match: inner_content = no_coeff_match.group(1) formula = ( @@ -1008,36 +1105,38 @@ def _expand_parenthetical_coefficients( return formula, total_expansion def _apply_advanced_composition_cleaning( - self, comp_prop_dict: Dict[str, Any] + self, comp_prop_dict: Dict[str, Any], steps: Set[str] ) -> Dict[str, Any]: """ - Apply advanced composition cleaning including Miller indices removal, - coefficient expansion, normalization, and zero-coefficient removal. + Apply coefficient expansion, gated by `steps`. + + Miller indices removal is handled earlier in `_clean_comp_prop_pairs`, + before the mandatory arithmetic/bracket resolution — see the comment + there for why the ordering matters. - Uses local cleaning methods to process compositions. + Coefficient expansion internally normalizes trailing zeros and removes + zero-coefficient elements as part of expanding leading/trailing/nested + bracket coefficients — there are no separate steps for those. """ cleaned_dict = {} for composition, property_value in comp_prop_dict.items(): - # Step 1: Remove Miller indices (crystal plane notations) - cleaned_comp = self._remove_miller_indices(composition) + cleaned_comp = composition - # Step 2: Expand leading and trailing coefficients - cleaned_comp = self._expand_leading_and_trailing_coefficients(cleaned_comp) - - # Step 3: Expand parenthetical coefficients - cleaned_comp, _ = self._expand_parenthetical_coefficients(cleaned_comp, 0) - - # Step 4: Normalize coefficients (remove trailing zeros) - cleaned_comp = self._normalize_coefficients(cleaned_comp) - - # Step 5: Remove elements with zero coefficients - cleaned_comp = self._remove_zero_coefficient_elements(cleaned_comp) + if CleaningStep.COEFFICIENT_EXPANSION.value in steps: + cleaned_comp = self._expand_leading_and_trailing_coefficients( + cleaned_comp + ) + cleaned_comp, _ = self._expand_parenthetical_coefficients( + cleaned_comp, 0 + ) cleaned_dict[cleaned_comp] = property_value return cleaned_dict - def _filter_unresolved_compositions(self, comp_prop_dict: Dict[str, Any], doi: str = "") -> Dict[str, Any]: + def _filter_unresolved_compositions( + self, comp_prop_dict: Dict[str, Any], doi: str = "" + ) -> Dict[str, Any]: """Remove individual compositions that still contain unresolved parentheses, brackets, or multiplication operators.""" resolved = {} for comp, val in comp_prop_dict.items(): @@ -1119,103 +1218,103 @@ def get_useful_data(self) -> Dict[str, Any]: return result - def clean_data_based_on_elements(self, apply_advanced_cleaning: bool = True) -> Dict[str, Any]: - """ - Run complete composition analysis with element validation. + def _resolve_cleaning_steps( + self, cleaning_steps: Union[str, List[str]] + ) -> Set[str]: + """Validate and resolve the public `cleaning_steps` argument into a concrete set. Args: - apply_advanced_cleaning: If True (default), applies advanced composition cleaning - (Miller indices removal, coefficient expansion, normalization). - If False, returns basic cleaned compositions only. + cleaning_steps: Either the string "all" or a list of CleaningStep values. + + Returns: + Set[str]: The resolved set of selected optional step names. + + Raises: + ValueError: If `cleaning_steps` is a non-"all" string, or contains + unknown step names. """ - result = {} - for key, value in self.all_data.items(): - comp_prop_data = self._get_comp_prop_data(value) - cleaned_data = self._clean_comp_prop_data_with_element_check(comp_prop_data, doi=key) - # Only include entries with valid compositions - if cleaned_data: - result[key] = value.copy() - comp_prop_dict = self._return_in_dict(cleaned_data) - # Apply advanced composition cleaning if requested - if apply_advanced_cleaning: - comp_prop_dict = self._apply_advanced_composition_cleaning(comp_prop_dict) - # Remove compositions that still have unresolved brackets or math ops - comp_prop_dict = self._filter_unresolved_compositions(comp_prop_dict, doi=key) - result[key]["composition_data"]["compositions_property_values"] = comp_prop_dict - return result + valid_steps = set(CleaningStep.all()) + if cleaning_steps == "all": + return valid_steps + if isinstance(cleaning_steps, str): + raise ValueError( + "Invalid cleaning_steps value. Must be 'all' or a list of step names: " + f"{CleaningStep.all()}." + ) + unknown = set(cleaning_steps) - valid_steps + if unknown: + raise ValueError( + f"Invalid cleaning step(s): {sorted(unknown)}. Valid options: {CleaningStep.all()}." + ) + return set(cleaning_steps) - def clean_data_without_element_filtering(self, apply_advanced_cleaning: bool = True) -> Dict[str, Any]: + def clean_data_with_selected_steps(self, steps: Set[str]) -> Dict[str, Any]: """ - Run composition analysis without element validation. + Run composition cleaning using exactly the given set of optional steps. + + Unicode conversion and arithmetic/fraction resolution always run + regardless of `steps`, since later steps depend on their output. Args: - apply_advanced_cleaning: If True (default), applies advanced composition cleaning - (Miller indices removal, coefficient expansion, normalization). - If False, returns basic cleaned compositions only. + steps: Resolved set of CleaningStep values to apply. """ result = {} for key, value in self.all_data.items(): comp_prop_data = self._get_comp_prop_data(value) - cleaned_data = self._clean_comp_prop_data_without_element_check( - comp_prop_data, doi=key - ) - # Include all entries that passed other cleaning steps + cleaned_data = self._clean_comp_prop_pairs(comp_prop_data, steps, doi=key) if cleaned_data: result[key] = value.copy() comp_prop_dict = self._return_in_dict(cleaned_data) - # Apply advanced composition cleaning if requested - if apply_advanced_cleaning: - comp_prop_dict = self._apply_advanced_composition_cleaning(comp_prop_dict) + if CleaningStep.COEFFICIENT_EXPANSION.value in steps: + comp_prop_dict = self._apply_advanced_composition_cleaning( + comp_prop_dict, steps + ) # Remove compositions that still have unresolved brackets or math ops - comp_prop_dict = self._filter_unresolved_compositions(comp_prop_dict, doi=key) - result[key]["composition_data"]["compositions_property_values"] = comp_prop_dict + comp_prop_dict = self._filter_unresolved_compositions( + comp_prop_dict, doi=key + ) + result[key]["composition_data"][ + "compositions_property_values" + ] = comp_prop_dict return result def clean_data_with_relevant_compositions( - self, strategy: CleaningStrategy = CleaningStrategy.FULL, - apply_advanced_cleaning: bool = True + self, + cleaning_steps: Union[str, List[str]] = "all", ) -> Dict[str, Any]: """ - Clean data using the specified strategy. + Clean data using the given set of optional cleaning steps. Args: - strategy: CleaningStrategy enum value determining the cleaning approach - - BASIC: Basic cleaning without element validation - - FULL: Complete cleaning with element validation (default) - apply_advanced_cleaning: If True (default), applies advanced composition cleaning - (Miller indices removal, coefficient expansion, normalization). - If False, returns basic cleaned compositions only. + cleaning_steps: Either "all" (default, every optional step enabled) or a + list of step names selecting exactly which optional steps run: + abbreviation_filtering, element_validation, text_normalization, + miller_indices, coefficient_expansion. Unicode conversion and + arithmetic/fraction resolution always run regardless of this + parameter. Returns: - Dict[str, Any]: Cleaned data based on selected strategy + Dict[str, Any]: Cleaned data based on selected steps. """ + steps = self._resolve_cleaning_steps(cleaning_steps) self.all_data = self.get_useful_data() - if strategy == CleaningStrategy.BASIC: - # Clean without element validation - return self.clean_data_without_element_filtering(apply_advanced_cleaning=apply_advanced_cleaning) - else: - # Full cleaning with element validation (default) - return self.clean_data_based_on_elements(apply_advanced_cleaning=apply_advanced_cleaning) + return self.clean_data_with_selected_steps(steps) def get_all_composition_property_pairs( self, - strategy: CleaningStrategy = CleaningStrategy.FULL, + cleaning_steps: Union[str, List[str]] = "all", is_return_doi: bool = False, - apply_advanced_cleaning: bool = True, ) -> Dict[str, Any]: """ - Get all composition-property pairs from cleaned data after applying cleaning strategy - and resolving composition calculations. + Get all composition-property pairs from cleaned data after applying the + selected cleaning steps and resolving composition calculations. Args: - strategy: CleaningStrategy enum value determining the cleaning approach - - BASIC: Basic cleaning without element validation - - FULL: Complete cleaning with element validation (default) + cleaning_steps: Either "all" (default, every optional step enabled) or a + list of step names selecting exactly which optional steps run. + See `clean_data_with_relevant_compositions` for the full list. is_return_doi: If True, returns nested dictionary with DOI as keys. If False (default), returns flat composition-property dictionary. - apply_advanced_cleaning: If True (default), applies advanced composition cleaning - (Miller indices removal, coefficient expansion, normalization). - If False, returns basic cleaned compositions only. Returns: Dict[str, Any]: If is_return_doi is False (default): @@ -1230,10 +1329,9 @@ def get_all_composition_property_pairs( Null values are filtered out in both cases. """ - # Clean the data using the specified strategy + # Clean the data using the specified steps cleaned_data = self.clean_data_with_relevant_compositions( - strategy=strategy, - apply_advanced_cleaning=apply_advanced_cleaning + cleaning_steps=cleaning_steps, ) if is_return_doi: diff --git a/tests/test_post_processing/test_data_cleaner.py b/tests/test_post_processing/test_data_cleaner.py index 048680ab..11d230bc 100644 --- a/tests/test_post_processing/test_data_cleaner.py +++ b/tests/test_post_processing/test_data_cleaner.py @@ -17,7 +17,7 @@ # Import the modules to test from comproscanner.post_processing.data_cleaner import ( DataCleaner, - CleaningStrategy, + CleaningStep, get_all_elements, ) @@ -43,19 +43,34 @@ def test_get_all_elements_contains_common_elements(self): assert element in result -class TestCleaningStrategy: - """Test cases for the CleaningStrategy enum.""" - - def test_cleaning_strategy_enum_values(self): - """Test that CleaningStrategy enum has correct values.""" - assert CleaningStrategy.BASIC == "basic" - assert CleaningStrategy.FULL == "full" +class TestCleaningStep: + """Test cases for the CleaningStep enum.""" + + def test_cleaning_step_enum_values(self): + """Test that CleaningStep enum has the expected five values.""" + assert CleaningStep.ABBREVIATION_FILTERING == "abbreviation_filtering" + assert CleaningStep.ELEMENT_VALIDATION == "element_validation" + assert CleaningStep.TEXT_NORMALIZATION == "text_normalization" + assert CleaningStep.MILLER_INDICES == "miller_indices" + assert CleaningStep.COEFFICIENT_EXPANSION == "coefficient_expansion" + + def test_cleaning_step_all(self): + """Test that CleaningStep.all() returns exactly the five expected step names.""" + assert set(CleaningStep.all()) == { + "abbreviation_filtering", + "element_validation", + "text_normalization", + "miller_indices", + "coefficient_expansion", + } - def test_cleaning_strategy_membership(self): - """Test CleaningStrategy enum membership.""" - assert "basic" in CleaningStrategy - assert "full" in CleaningStrategy - assert "invalid" not in CleaningStrategy + def test_cleaning_step_membership(self): + """Test CleaningStep enum membership.""" + assert "element_validation" in CleaningStep + assert "miller_indices" in CleaningStep + assert "invalid" not in CleaningStep + assert "normalization" not in CleaningStep + assert "zero_coefficient" not in CleaningStep class TestDataCleanerInitialization: @@ -212,12 +227,44 @@ def test_is_elements_empty_dict(self, data_cleaner): result = data_cleaner._is_elements({}) assert result is False - def test_remove_extra_spaces(self, data_cleaner): - """Test _remove_extra_spaces method.""" - dict_list = [{"Na Cl": "value1"}, {"Ti O2": "value2"}, {"Ca CO3": "value3"}] - result = data_cleaner._remove_extra_spaces(dict_list) - expected = [{"NaCl": "value1"}, {"TiO2": "value2"}, {"CaCO3": "value3"}] - assert result == expected + def test_normalize_text_title_cases_descriptive_words(self, data_cleaner): + """Test _normalize_text title-cases descriptive word tokens and preserves spaces.""" + dict_list = [ + {"Bi4Ti3O12 ultrathin with oxygen vacancies": "value1"}, + ] + result = data_cleaner._normalize_text(dict_list) + assert result == [{"Bi4Ti3O12 Ultrathin with Oxygen Vacancies": "value1"}] + + def test_normalize_text_leaves_glued_words_untouched(self, data_cleaner): + """Test _normalize_text does not insert spaces where none exist in the source.""" + dict_list = [{"K0.5Na0.5Nb0.9Ta0.1O3-Milling15h": "value1"}] + result = data_cleaner._normalize_text(dict_list) + assert result == [{"K0.5Na0.5Nb0.9Ta0.1O3-Milling15h": "value1"}] + + def test_normalize_text_preserves_all_caps_abbreviations(self, data_cleaner): + """Test _normalize_text leaves all-caps abbreviation tokens (e.g. XRD) unchanged.""" + dict_list = [{"BaTiO3 XRD pattern": "value1"}] + result = data_cleaner._normalize_text(dict_list) + assert result == [{"BaTiO3 XRD Pattern": "value1"}] + + def test_normalize_text_digit_tokens_untouched(self, data_cleaner): + """Test _normalize_text does not modify tokens containing digits.""" + dict_list = [{"Ti O2": "value1"}] + result = data_cleaner._normalize_text(dict_list) + # "Ti" is title-cased (already correct), "O2" contains a digit so is untouched + assert result == [{"Ti O2": "value1"}] + + def test_normalize_text_collapses_multiple_internal_spaces(self, data_cleaner): + """Test _normalize_text collapses runs of multiple spaces down to one.""" + dict_list = [{"Bi4Ti3O12 ultrathin with oxygen vacancies": "value1"}] + result = data_cleaner._normalize_text(dict_list) + assert result == [{"Bi4Ti3O12 Ultrathin with Oxygen Vacancies": "value1"}] + + def test_normalize_text_strips_leading_and_trailing_whitespace(self, data_cleaner): + """Test _normalize_text strips leading/trailing whitespace from composition keys.""" + dict_list = [{" Bi4Ti3O12 ultrathin with oxygen vacancies ": "value1"}] + result = data_cleaner._normalize_text(dict_list) + assert result == [{"Bi4Ti3O12 Ultrathin with Oxygen Vacancies": "value1"}] def test_convert_fractions_and_resolve_compositions_fractions(self, data_cleaner): """Test _convert_fractions_and_resolve_compositions with fractions.""" @@ -349,10 +396,12 @@ def temp_mixed_json_file(self, sample_data_with_mixed_validity): yield temp_file_path os.unlink(temp_file_path) - def test_clean_data_without_element_filtering(self, temp_mixed_json_file): - """Test clean_data_without_element_filtering method.""" + def test_clean_data_without_element_validation(self, temp_mixed_json_file): + """Test clean_data_with_relevant_compositions without the element_validation step.""" cleaner = DataCleaner(temp_mixed_json_file) - result = cleaner.clean_data_without_element_filtering() + result = cleaner.clean_data_with_relevant_compositions( + cleaning_steps=["abbreviation_filtering"] + ) # Should keep papers with valid compositions after basic cleaning assert isinstance(result, dict) @@ -365,16 +414,35 @@ def test_clean_data_without_element_filtering(self, temp_mixed_json_file): for comp_key in comp_values.keys(): assert not re.match(r"(? 0 def test_clean_data_with_relevant_compositions_empty_input(self): """Test clean_data_with_relevant_compositions with empty JSON input.""" @@ -434,15 +530,41 @@ def test_unresolved_compositions_collected(self): "composition_data": { "compositions_property_values": {"BaTiO3": 100, "PbZrO3": 200} }, - "synthesis_data": {"method": "", "precursors": [], "steps": [], "characterization_techniques": []}, - "article_metadata": {"doi": "", "title": "", "journal": "", "year": "", "isOpenAccess": False, "authors": [], "keywords": []}, + "synthesis_data": { + "method": "", + "precursors": [], + "steps": [], + "characterization_techniques": [], + }, + "article_metadata": { + "doi": "", + "title": "", + "journal": "", + "year": "", + "isOpenAccess": False, + "authors": [], + "keywords": [], + }, }, "10.x/bad": { "composition_data": { "compositions_property_values": {"(Ba0.5Na0.5)(0.9*x)TiO3": 50} }, - "synthesis_data": {"method": "", "precursors": [], "steps": [], "characterization_techniques": []}, - "article_metadata": {"doi": "", "title": "", "journal": "", "year": "", "isOpenAccess": False, "authors": [], "keywords": []}, + "synthesis_data": { + "method": "", + "precursors": [], + "steps": [], + "characterization_techniques": [], + }, + "article_metadata": { + "doi": "", + "title": "", + "journal": "", + "year": "", + "isOpenAccess": False, + "authors": [], + "keywords": [], + }, }, } with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: @@ -521,9 +643,9 @@ def test_full_pipeline_integration(self, temp_complex_json_file): # Test the complete pipeline cleaner = DataCleaner(temp_complex_json_file) - # Clean with FULL strategy + # Clean with all steps enabled cleaned_data = cleaner.clean_data_with_relevant_compositions( - CleaningStrategy.FULL + cleaning_steps="all" ) # Verify the integration worked correctly @@ -532,22 +654,26 @@ def test_full_pipeline_integration(self, temp_complex_json_file): for paper_key, paper_data in cleaned_data.items(): comp_values = paper_data["composition_data"]["compositions_property_values"] - # Should have processed fractions, spaces, and parentheses + # Should have processed fractions and parentheses, and dropped invalid patterns for comp_key in comp_values.keys(): - # No spaces should remain - assert " " not in comp_key + # Fractions should be resolved (no bare "/" left) + assert "/" not in comp_key # No invalid patterns should remain assert not re.match(r"(? "AlN"), silently overwriting one + value with the other when merged.""" + data = { + "10.x/miller": { + "composition_data": { + "compositions_property_values": { + "AlN (002)": 1, + "AlN (110)": 2, + "ZnO (101)": 3, + "BaTiO3": 4, + } + }, + "synthesis_data": {}, + "article_metadata": {}, + } + } + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(data, f) + temp_file_path = f.name + + try: + cleaner = DataCleaner(temp_file_path) + result = cleaner.clean_data_with_relevant_compositions(cleaning_steps="all") + comp_values = result["10.x/miller"]["composition_data"][ + "compositions_property_values" + ] + # Only the composition without a Miller index survives + assert comp_values == {"BaTiO3": 4} + assert set(cleaner.filtered_compositions.get("10.x/miller", [])) == { + "AlN (002)", + "AlN (110)", + "ZnO (101)", + } + finally: + os.unlink(temp_file_path) + + def test_miller_index_shaped_bracket_flagged_unresolved_when_not_selected(self): + """Regression test: when miller_indices is NOT selected, a Miller-index-shaped + bracket like "(002)" must NOT be silently merged into the preceding element by + coefficient_expansion's "remove brackets without coefficients" step (which would + otherwise turn AlN (002) into AlN 002/AlN2). Instead it should be left untouched + and dropped as unresolved, so a stray bracket is never silently misinterpreted. + """ + data = { + "10.x/miller2": { + "composition_data": {"compositions_property_values": {"AlN (002)": 1}}, + "synthesis_data": {}, + "article_metadata": {}, + } + } + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(data, f) + temp_file_path = f.name + + try: + cleaner = DataCleaner(temp_file_path) + result = cleaner.clean_data_with_relevant_compositions( + cleaning_steps=["coefficient_expansion"] + ) + comp_values = ( + result.get("10.x/miller2", {}) + .get("composition_data", {}) + .get("compositions_property_values", {}) + ) + # Must not silently resolve to a wrong formula like "AlN2" or "AlN 002" + assert comp_values == {} + assert cleaner.unresolved_compositions.get("10.x/miller2") == ["AlN (002)"] + finally: + os.unlink(temp_file_path) + class TestErrorHandling: """Test cases for error handling scenarios.""" @@ -612,7 +813,11 @@ def test_division_by_zero_in_fractions(self): cleaner = DataCleaner(temp_file_path) # Should handle division by zero gracefully result = cleaner.clean_data_with_relevant_compositions( - CleaningStrategy.BASIC + cleaning_steps=[ + "abbreviation_filtering", + "miller_indices", + "coefficient_expansion", + ] ) # The original fraction should be kept if division by zero assert isinstance(result, dict) diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 25e26750..c31ea8b4 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -4,6 +4,9 @@ from unittest.mock import MagicMock, patch import pandas as pd +import pytest + +from comproscanner.utils.error_handler import ValueErrorHandler # Avoid importing heavy vector DB runtime deps during test module import. if "langchain_chroma" not in sys.modules: @@ -155,16 +158,27 @@ def test_clean_data_public_api_forwards_to_cleaner(tmp_path): json_results_file=str(input_file), is_save_separate_results=False, is_save_composition_property_file=False, - cleaning_strategy="full", + cleaning_steps="all", ) assert result == {"10.x/test": {}} cleaner.clean_data_with_relevant_compositions.assert_called_once_with( - strategy="full", - apply_advanced_cleaning=True, + cleaning_steps="all", ) +def test_clean_data_rejects_unknown_cleaning_step(tmp_path): + scanner = ComProScanner(main_property_keyword="piezoelectric") + input_file = tmp_path / "input.json" + input_file.write_text("{}", encoding="utf-8") + + with pytest.raises(ValueErrorHandler): + scanner.clean_data( + json_results_file=str(input_file), + cleaning_steps=["bogus_step"], + ) + + def test_clean_data_stores_unresolved_compositions(tmp_path): scanner = ComProScanner(main_property_keyword="piezoelectric") input_file = tmp_path / "input.json" @@ -185,7 +199,7 @@ def test_clean_data_stores_unresolved_compositions(tmp_path): is_save_separate_results=False, is_save_composition_property_file=True, composition_property_file=str(tmp_path / "comp_prop.json"), - cleaning_strategy="full", + cleaning_steps="all", is_store_unresolved_compositions=True, unresolved_compositions_file=str(unresolved_file), ) From ca50c4a22a78ca1a91e6a7f418ea8258fa94c422 Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Wed, 15 Jul 2026 12:49:05 +0100 Subject: [PATCH 4/8] update: deepseek model changed from deepseek-chat to deepseek-v4-flash reason: deepseek-chat to be deprecated on 2026/07/24 --- docs/rag-config.md | 2 +- docs/usage/data-extraction.md | 2 +- .../CMEG-IITR_Agentic_data_extraction/run_piezo_agent.py | 2 +- .../comparing_existing_frameworks/Eunomia/eunomia_test.py | 2 +- examples/test_example.py | 2 +- examples/vlm_piezo_test/vlm_test_example.py | 2 +- src/comproscanner/extract_flow/tools/equation_tool.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/rag-config.md b/docs/rag-config.md index 5a37be74..0b80abaa 100644 --- a/docs/rag-config.md +++ b/docs/rag-config.md @@ -123,7 +123,7 @@ scanner.extract_composition_property_data( scanner.extract_composition_property_data( main_extraction_keyword="d33", rag_db_path="embeddings/piezo", - rag_chat_model="deepseek/deepseek-chat", + rag_chat_model="deepseek/deepseek-v4-flash", rag_max_tokens=1024, rag_top_k=4, ) diff --git a/docs/usage/data-extraction.md b/docs/usage/data-extraction.md index b616ff2b..89854017 100644 --- a/docs/usage/data-extraction.md +++ b/docs/usage/data-extraction.md @@ -243,7 +243,7 @@ Is there any material chemical composition and corresponding {main_property_keyw | 1 (default) | `ANTHROPIC_API_KEY` | `anthropic/claude-sonnet-4-6` | | 2 | `GEMINI_API_KEY` | `gemini/gemini-3-flash-preview` | | 3 | `OPENAI_API_KEY` | `openai/gpt-5.4-mini` | - | 4 | `DEEPSEEK_API_KEY` | `deepseek/deepseek-chat` | + | 4 | `DEEPSEEK_API_KEY` | `deepseek/deepseek-v4-flash` | | 5 | `OPENROUTER_API_KEY` | `openrouter/google/gemini-2.0-flash` | | 6 | `TOGETHER_API_KEY` | `together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo` | | 7 | `COHERE_API_KEY` | `cohere/command-r-plus` | diff --git a/examples/piezo_test/comparing_existing_frameworks/CMEG-IITR_Agentic_data_extraction/run_piezo_agent.py b/examples/piezo_test/comparing_existing_frameworks/CMEG-IITR_Agentic_data_extraction/run_piezo_agent.py index ca663c4a..b3a12f31 100644 --- a/examples/piezo_test/comparing_existing_frameworks/CMEG-IITR_Agentic_data_extraction/run_piezo_agent.py +++ b/examples/piezo_test/comparing_existing_frameworks/CMEG-IITR_Agentic_data_extraction/run_piezo_agent.py @@ -44,7 +44,7 @@ # === DeepSeek Configuration === DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY") DEEPSEEK_BASE_URL = "https://api.deepseek.com" -model_name = "deepseek-chat" +model_name = "deepseek-v4-flash" # === State Definition === diff --git a/examples/piezo_test/comparing_existing_frameworks/Eunomia/eunomia_test.py b/examples/piezo_test/comparing_existing_frameworks/Eunomia/eunomia_test.py index ffb2934d..9e3f6b08 100644 --- a/examples/piezo_test/comparing_existing_frameworks/Eunomia/eunomia_test.py +++ b/examples/piezo_test/comparing_existing_frameworks/Eunomia/eunomia_test.py @@ -96,7 +96,7 @@ def process_single_paper(paper_path, paper_id): agent = Eunomia( tools=tools, - model="deepseek-chat", + model="deepseek-v4-flash", get_cost=False, # Disable cost tracking to avoid warnings agent_type=eunomia.AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION, ) diff --git a/examples/test_example.py b/examples/test_example.py index c57133d2..b29264d9 100644 --- a/examples/test_example.py +++ b/examples/test_example.py @@ -79,7 +79,7 @@ # is_only_consider_test_doi_list=True, test_doi_list_file="piezo_test_dois_random.txt", total_test_data=100, - model="deepseek/deepseek-chat", + model="deepseek/deepseek-v4-flash", output_log_folder="piezo_test/model-logs/logs/deepseek/deepseek-v3-0324", task_output_folder="piezo_test/model-logs/task_outputs/deepseek/deepseek-v3-0324", materials_data_identifier_query="Is there any ceramic, composite, or crystal material with its specific chemical composition and corresponding d33 piezoelectric coefficient value (in pC/N or pm/V units) explicitly mentioned in the paper? Give one word answer - either 'yes' or 'no'. Only answer 'yes' if ALL of the following criteria are met: (1) The material is specifically a ceramic, composite, doped, or crystal, or different environments of materials (exclude all polymers including PVDF, PLLA, and similar), (2) A numerical d33 value with units pC/N or pm/V is explicitly stated which is associated with that specific material composition or specific environment.", diff --git a/examples/vlm_piezo_test/vlm_test_example.py b/examples/vlm_piezo_test/vlm_test_example.py index 1583e446..97aaefe0 100644 --- a/examples/vlm_piezo_test/vlm_test_example.py +++ b/examples/vlm_piezo_test/vlm_test_example.py @@ -203,7 +203,7 @@ is_only_consider_test_doi_list=True, test_doi_list_file="random_dois_for_vlm_test.txt", is_extract_synthesis_data=False, # For this test, we are only evaluating the composition-property extraction capability of the VLM, so we set this to False to save time and cost. - model="deepseek/deepseek-chat", + model="deepseek/deepseek-v4-flash", vlm_model="gemini/gemini-3-flash-preview", output_log_folder="vlm_piezo_test/model-logs/logs/google/gemini-3-flash-preview", task_output_folder="vlm_piezo_test/model-logs/task_outputs/google/gemini-3-flash-preview", diff --git a/src/comproscanner/extract_flow/tools/equation_tool.py b/src/comproscanner/extract_flow/tools/equation_tool.py index c23c9344..aa2fe071 100644 --- a/src/comproscanner/extract_flow/tools/equation_tool.py +++ b/src/comproscanner/extract_flow/tools/equation_tool.py @@ -163,7 +163,7 @@ ("ANTHROPIC_API_KEY", "anthropic/claude-sonnet-4-6"), ("GEMINI_API_KEY", "gemini/gemini-3-flash-preview"), ("OPENAI_API_KEY", "openai/gpt-5.4-mini"), - ("DEEPSEEK_API_KEY", "deepseek/deepseek-chat"), + ("DEEPSEEK_API_KEY", "deepseek/deepseek-v4-flash"), ("OPENROUTER_API_KEY", "openrouter/google/gemini-2.0-flash"), ("TOGETHER_API_KEY", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"), ("COHERE_API_KEY", "cohere/command-r-plus"), From 7c067171c216c2c347efc1463d62750ad72f830d Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Mon, 27 Jul 2026 16:43:55 +0100 Subject: [PATCH 5/8] fix: rename failed-articles default path and eliminate coefficient-expansion corruption on non-formula text via a shared formula-boundary check --- CHANGELOG.md | 18 +- docs/about/changelog.md | 20 +- docs/usage/article-processing.md | 6 +- docs/usage/data-cleaning.md | 127 ++-- src/comproscanner/__init__.py | 9 +- src/comproscanner/comproscanner.py | 24 +- .../post_processing/data_cleaner.py | 652 +++++++++++++--- .../utils/configs/paths_config.py | 6 +- .../test_post_processing/test_data_cleaner.py | 709 +++++++++++++++++- 9 files changed, 1373 insertions(+), 198 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34327db8..01617270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,19 +4,29 @@ - Added `is_track_pdfs` and `track_pdfs_report_path` to `process_articles()` for local PDF workflows. When enabled (default), each processed PDF is recorded as a `filenamedoi` entry in `logs/{keyword}_pdf_processed_dois.txt`, allowing re-runs to skip already-processed PDFs before any conversion or API calls. Falls back to scanning the output CSV when the tracking file does not yet exist. -- Centralised non-keyword default file paths (`results/failed_automated_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. +- Centralised non-keyword default file paths (`results/article_processor_failed_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. ### Changed -- Replaced the `cleaning_strategy` (`"full"`/`"basic"`) and `apply_advanced_cleaning` parameters on `DataCleaner`, `ComProScanner.clean_data()`, and the top-level `clean_data()` function with a single `cleaning_steps` parameter accepting either `"all"` (default) or a list of individually selectable step names: `abbreviation_filtering`, `element_validation`, `text_normalization`, `miller_indices`, `coefficient_expansion` (exposed via the new `CleaningStep` enum). Unicode subscript conversion and arithmetic/fraction resolution remain always-on, since the other steps depend on their output. `normalization` and `zero_coefficient` are no longer separate steps — both are folded into `coefficient_expansion`, which already performs them internally. +- Replaced the `cleaning_strategy` (`"full"`/`"basic"`) and `apply_advanced_cleaning` parameters on `DataCleaner`, `ComProScanner.clean_data()`, and the top-level `clean_data()` function with a single `cleaning_steps` parameter accepting either `"all"` (default) or a list of individually selectable step names: `abbreviation_filtering`, `element_validation_strict`/`element_validation_lenient`, `text_normalization`, `miller_indices`, `coefficient_expansion_strict`/`coefficient_expansion_lenient` (exposed via the new `CleaningStep` enum). The `_lenient` variants are weaker companions of their `_strict` counterparts and have no additional effect when both are selected together. ### Fixed - Replaced the blind space-stripping behaviour (`key.replace(" ", "")`, which mangled descriptive composition text such as `"Bi4Ti3O12 ultrathin with oxygen vacancies"` into `"Bi4Ti3O12ultrathinwithoxygenvacancies"`) with a new optional `text_normalization` step that strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens, while leaving formula segments and element-symbol sequences untouched. -- Fixed Miller-index handling in data cleaning: a bare 3-digit parenthetical crystal-plane notation (e.g. `"AlN (002)"`) was previously fed straight into the mandatory arithmetic/bracket resolver, which misread `(002)` as a bare-number coefficient bracket and merged its digits into the preceding element — silently producing `"AlN2"` instead of recognising it as a Miller index at all. The `miller_indices` cleaning step now detects this notation before arithmetic/bracket resolution runs, and — rather than stripping the notation and keeping the bare formula, which would collapse distinct surface-orientation entries for the same material onto the same dict key (e.g. `"AlN (002)"` and `"AlN (110)"` both becoming `"AlN"` and silently overwriting one property value with the other when merged) — drops the whole composition entry, tracking it in `filtered_compositions` the same way `abbreviation_filtering`/`element_validation` drops are tracked. When `miller_indices` is *not* selected, the same 3-digit-bracket detection now also stops both the arithmetic resolver and `coefficient_expansion` from silently merging the digits into a wrong formula; the composition is instead dropped later as an unresolved composition, the same as any other leftover bracket. +- Fixed Miller-index notation (e.g. `"AlN (002)"`) being misread by the mandatory arithmetic resolver as a coefficient bracket. `miller_indices` now detects and drops such compositions instead of silently merging digits into the formula or colliding distinct surface-orientation entries onto the same key; when `miller_indices` isn't selected, these compositions are left as unresolved rather than corrupted. + +- Fixed unresolved-composition filtering dropping compositions when no `coefficient_expansion_strict`/`_lenient` step was selected; the filter now only runs when coefficient expansion is actually requested. + +- Fixed weight/mole/atomic-percent dopant annotations (e.g. `"7 wt% NiO"`, `"1.25 wt% (0.78PbO-0.22CuO)"`) having their number misread as a stoichiometric coefficient and distributed into the annotated compound. Such annotations, including ones with a bracketed target, are now protected with an inert placeholder before coefficient expansion runs and restored verbatim afterward. + +- Fixed coefficient expansion scaling descriptive-word fragments that coincidentally resemble element symbols — e.g. `"Bo"` in `"Bottom"`, `"Re"` in `"Reoxidized"` (`"Re10oxidized"`), or a unit abbreviation like `"h"` (hours) capitalized by `text_normalization` into `"H"` (Hydrogen) — as if they were real stoichiometry. `text_normalization` also no longer capitalizes a token when doing so would manufacture a disguised element in the first place. + +- Fixed a nested multi-term coefficient expression (e.g. `"0.75*(0.89(Bi0.5Na0.5)TiO3-0.11BaTiO3) + 0.25*(...)"`) being shredded into mismatched brackets with un-distributed coefficients instead of correctly distributing the outer coefficient across each inner term, including through arbitrary nesting depth and sign flips when a whole bracket is subtracted. Comma-separated site-occupancy notation (e.g. `"(K,Na,Li)(Nb,Ta)O3"`) remains unsupported by design but is no longer actively corrupted either. + +- Replaced the several ad-hoc "is this really an element" checks above with one shared boundary-detection function, `_formula_prefix_end`, that determines how much of a string is genuinely valid formula content. This also fixed a further case the old checks missed — a trailing annotation with its own real element letter right after a number (e.g. `"C"` for Celsius in `"...PbTiO3 (calcined at 660°C)"`) — and ensures any future case of this shape is handled by the same rule rather than needing another bespoke patch. - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). @@ -55,7 +65,7 @@ - Added `save_failed_pdf_report` and `failed_pdf_report_path` to `process_articles()`, with filename-derived DOI validation and failed-PDF reporting for local PDF workflows. -- Added `save_failed_automated_report` and `failed_automated_report_path` to `process_articles()` for automated publisher sources (Elsevier, Springer Nature, IOP, Wiley), mirroring the existing PDF failure report. Failed articles are written as tab-separated `doi`, `publisher`, `reason` entries to `results/failed_automated_articles.txt` by default. +- Added `save_failed_automated_report` and `failed_automated_report_path` to `process_articles()` for automated publisher sources (Elsevier, Springer Nature, IOP, Wiley), mirroring the existing PDF failure report. Failed articles are written as tab-separated `doi`, `publisher`, `reason` entries to `results/article_processor_failed_articles.txt` by default. - Added image-aware fallback in `DataExtractionFlow.identify_materials_data_presence()`: diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 5b0421a1..640aa448 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -4,19 +4,29 @@ - Added `is_track_pdfs` and `track_pdfs_report_path` to `process_articles()` for local PDF workflows. When enabled (default), each processed PDF is recorded as a `filenamedoi` entry in `logs/{keyword}_pdf_processed_dois.txt`, allowing re-runs to skip already-processed PDFs before any conversion or API calls. Falls back to scanning the output CSV when the tracking file does not yet exist. -- Centralised non-keyword default file paths (`results/failed_automated_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. +- Centralised non-keyword default file paths (`results/article_processor_failed_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. ### Changed -- Replaced the `cleaning_strategy` (`"full"`/`"basic"`) and `apply_advanced_cleaning` parameters on `DataCleaner`, `ComProScanner.clean_data()`, and the top-level `clean_data()` function with a single `cleaning_steps` parameter accepting either `"all"` (default) or a list of individually selectable step names: `abbreviation_filtering`, `element_validation`, `text_normalization`, `miller_indices`, `coefficient_expansion` (exposed via the new `CleaningStep` enum). Unicode subscript conversion and arithmetic/fraction resolution remain always-on, since the other steps depend on their output. `normalization` and `zero_coefficient` are no longer separate steps — both are folded into `coefficient_expansion`, which already performs them internally. +- Replaced the `cleaning_strategy` (`"full"`/`"basic"`) and `apply_advanced_cleaning` parameters on `DataCleaner`, `ComProScanner.clean_data()`, and the top-level `clean_data()` function with a single `cleaning_steps` parameter accepting either `"all"` (default) or a list of individually selectable step names: `abbreviation_filtering`, `element_validation_strict`/`element_validation_lenient`, `text_normalization`, `miller_indices`, `coefficient_expansion_strict`/`coefficient_expansion_lenient` (exposed via the new `CleaningStep` enum). The `_lenient` variants are weaker companions of their `_strict` counterparts and have no additional effect when both are selected together. ### Fixed - Replaced the blind space-stripping behaviour (`key.replace(" ", "")`, which mangled descriptive composition text such as `"Bi4Ti3O12 ultrathin with oxygen vacancies"` into `"Bi4Ti3O12ultrathinwithoxygenvacancies"`) with a new optional `text_normalization` step that strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens, while leaving formula segments and element-symbol sequences untouched. -- Fixed Miller-index handling in data cleaning: a bare 3-digit parenthetical crystal-plane notation (e.g. `"AlN (002)"`) was previously fed straight into the mandatory arithmetic/bracket resolver, which misread `(002)` as a bare-number coefficient bracket and merged its digits into the preceding element — silently producing `"AlN2"` instead of recognising it as a Miller index at all. The `miller_indices` cleaning step now detects this notation before arithmetic/bracket resolution runs, and — rather than stripping the notation and keeping the bare formula, which would collapse distinct surface-orientation entries for the same material onto the same dict key (e.g. `"AlN (002)"` and `"AlN (110)"` both becoming `"AlN"` and silently overwriting one property value with the other when merged) — drops the whole composition entry, tracking it in `filtered_compositions` the same way `abbreviation_filtering`/`element_validation` drops are tracked. When `miller_indices` is *not* selected, the same 3-digit-bracket detection now also stops both the arithmetic resolver and `coefficient_expansion` from silently merging the digits into a wrong formula; the composition is instead dropped later as an unresolved composition, the same as any other leftover bracket. +- Fixed Miller-index notation (e.g. `"AlN (002)"`) being misread by the mandatory arithmetic resolver as a coefficient bracket. `miller_indices` now detects and drops such compositions instead of silently merging digits into the formula or colliding distinct surface-orientation entries onto the same key; when `miller_indices` isn't selected, these compositions are left as unresolved rather than corrupted. + +- Fixed unresolved-composition filtering dropping compositions when no `coefficient_expansion_strict`/`_lenient` step was selected; the filter now only runs when coefficient expansion is actually requested. + +- Fixed weight/mole/atomic-percent dopant annotations (e.g. `"7 wt% NiO"`, `"1.25 wt% (0.78PbO-0.22CuO)"`) having their number misread as a stoichiometric coefficient and distributed into the annotated compound. Such annotations, including ones with a bracketed target, are now protected with an inert placeholder before coefficient expansion runs and restored verbatim afterward. + +- Fixed coefficient expansion scaling descriptive-word fragments that coincidentally resemble element symbols — e.g. `"Bo"` in `"Bottom"`, `"Re"` in `"Reoxidized"` (`"Re10oxidized"`), or a unit abbreviation like `"h"` (hours) capitalized by `text_normalization` into `"H"` (Hydrogen) — as if they were real stoichiometry. `text_normalization` also no longer capitalizes a token when doing so would manufacture a disguised element in the first place. + +- Fixed a nested multi-term coefficient expression (e.g. `"0.75*(0.89(Bi0.5Na0.5)TiO3-0.11BaTiO3) + 0.25*(...)"`) being shredded into mismatched brackets with un-distributed coefficients instead of correctly distributing the outer coefficient across each inner term, including through arbitrary nesting depth and sign flips when a whole bracket is subtracted. Comma-separated site-occupancy notation (e.g. `"(K,Na,Li)(Nb,Ta)O3"`) remains unsupported by design but is no longer actively corrupted either. + +- Replaced the several ad-hoc "is this really an element" checks above with one shared boundary-detection function, `_formula_prefix_end`, that determines how much of a string is genuinely valid formula content. This also fixed a further case the old checks missed — a trailing annotation with its own real element letter right after a number (e.g. `"C"` for Celsius in `"...PbTiO3 (calcined at 660°C)"`) — and ensures any future case of this shape is handled by the same rule rather than needing another bespoke patch. - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). @@ -54,11 +64,11 @@ - Added `save_failed_pdf_report` and `failed_pdf_report_path` to `process_articles()`, with filename-derived DOI validation and failed-PDF reporting for local PDF workflows. -- Added `save_failed_automated_report` and `failed_automated_report_path` to `process_articles()` for automated publisher sources (Elsevier, Springer Nature, IOP, Wiley), mirroring the existing PDF failure report. Failed articles are written as tab-separated `doi`, `publisher`, `reason` entries to `results/failed_automated_articles.txt` by default. +- Added `save_failed_automated_report` and `failed_automated_report_path` to `process_articles()` for automated publisher sources (Elsevier, Springer Nature, IOP, Wiley), mirroring the existing PDF failure report. Failed articles are written as tab-separated `doi`, `publisher`, `reason` entries to `results/article_processor_failed_articles.txt` by default. - Added `is_track_pdfs` and `track_pdfs_report_path` to `process_articles()` for local PDF workflows. When enabled (default), each processed PDF is recorded as a `filenamedoi` entry in `logs/{keyword}_pdf_processed_dois.txt`, allowing re-runs to skip already-processed PDFs before any conversion or API calls. Falls back to scanning the output CSV when the tracking file does not yet exist. -- Centralised default file paths (`results/failed_automated_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. +- Centralised default file paths (`results/article_processor_failed_articles.txt`, `agentic_evaluation_result.json`, `detailed_evaluation.json`) as class-level constants on `DefaultPaths` so they can be changed in one place. - Added image-aware fallback in `DataExtractionFlow.identify_materials_data_presence()`: diff --git a/docs/usage/article-processing.md b/docs/usage/article-processing.md index c192581f..70bac1bc 100644 --- a/docs/usage/article-processing.md +++ b/docs/usage/article-processing.md @@ -154,11 +154,11 @@ For automated publisher sources (`elsevier`, `springer`, `iop`, `wiley`). If `Tr #### :material-square-medium:`failed_automated_report_path` _(str)_ -Custom output path for the automated failure report. If not provided, defaults to `results/failed_automated_articles.txt`. All enabled publisher processors append to the same file, so a single run produces one consolidated report. +Custom output path for the automated failure report. If not provided, defaults to `results/article_processor_failed_articles.txt`. All enabled publisher processors append to the same file, so a single run produces one consolidated report. !!! info "Default Values" - :material-square-small:**`source_list`** = ["elsevier", "wiley", "iop", "springer"]
:material-square-small:**`folder_path`** = None
:material-square-small:**`doi_list`** = None
:material-square-small:**`is_sql_db`** = False
:material-square-small:**`is_save_xml`** = False
:material-square-small:**`is_save_pdf`** = False
:material-square-small:**`rag_db_path`** = "db"
:material-square-small:**`chunk_size`** = 1000
:material-square-small:**`chunk_overlap`** = 25
:material-square-small:**`embedding_model`** = "huggingface:thellert/physbert_cased"
:material-square-small:**`main_figure_keywords`** = `property_keywords`
:material-square-small:**`additional_figure_keywords`** = None
:material-square-small:**`save_failed_pdf_report`** = True
:material-square-small:**`failed_pdf_report_path`** = None (auto: `{folder_path}/failed_pdf_filenames.txt`)
:material-square-small:**`is_track_pdfs`** = True
:material-square-small:**`track_pdfs_report_path`** = None (auto: `logs/{keyword}_pdf_processed_dois.txt`)
:material-square-small:**`save_failed_automated_report`** = True
:material-square-small:**`failed_automated_report_path`** = None (auto: `results/failed_automated_articles.txt`) + :material-square-small:**`source_list`** = ["elsevier", "wiley", "iop", "springer"]
:material-square-small:**`folder_path`** = None
:material-square-small:**`doi_list`** = None
:material-square-small:**`is_sql_db`** = False
:material-square-small:**`is_save_xml`** = False
:material-square-small:**`is_save_pdf`** = False
:material-square-small:**`rag_db_path`** = "db"
:material-square-small:**`chunk_size`** = 1000
:material-square-small:**`chunk_overlap`** = 25
:material-square-small:**`embedding_model`** = "huggingface:thellert/physbert_cased"
:material-square-small:**`main_figure_keywords`** = `property_keywords`
:material-square-small:**`additional_figure_keywords`** = None
:material-square-small:**`save_failed_pdf_report`** = True
:material-square-small:**`failed_pdf_report_path`** = None (auto: `{folder_path}/failed_pdf_filenames.txt`)
:material-square-small:**`is_track_pdfs`** = True
:material-square-small:**`track_pdfs_report_path`** = None (auto: `logs/{keyword}_pdf_processed_dois.txt`)
:material-square-small:**`save_failed_automated_report`** = True
:material-square-small:**`failed_automated_report_path`** = None (auto: `results/article_processor_failed_articles.txt`) ## Processing Workflow @@ -243,7 +243,7 @@ scanner.process_articles( property_keywords=property_keywords, source_list=["elsevier", "springer", "iop", "wiley"], save_failed_automated_report=True, - failed_automated_report_path="results/failed_automated_articles.txt" + failed_automated_report_path="results/article_processor_failed_articles.txt" ) ``` diff --git a/docs/usage/data-cleaning.md b/docs/usage/data-cleaning.md index 9de3991b..946e89d2 100644 --- a/docs/usage/data-cleaning.md +++ b/docs/usage/data-cleaning.md @@ -46,21 +46,23 @@ Path to the cleaned composition-property file containing a dictionary of composi Either the string `"all"` (default, every optional step enabled) or a list of step names selecting exactly which optional steps run: -- **`abbreviation_filtering`** — drops composition keys containing 2+ consecutive capital letters (abbreviations/junk keys, e.g. `"PVDF"`). -- **`element_validation`** — keep only compositions whose key fully resolves to valid periodic-table element symbols. -- **`text_normalization`** — deterministic cleanup: strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens (e.g. `" Bi4Ti3O12 ultrathin with oxygen vacancies "` → `"Bi4Ti3O12 Ultrathin with Oxygen Vacancies"`). Tokens containing digits (real formula segments) and tokens that fully parse as element symbols (e.g. `"NaCl"`) are left untouched, as are all-caps abbreviations (e.g. `"PVDF"`). -- **`miller_indices`** — drops compositions carrying a crystal-plane notation like `(002)`, `(111)`, `(100)`, etc. entirely, rather than stripping the notation and keeping the bare formula. Stripping and keeping would collapse distinct surface-orientation entries for the same material onto the same dict key — e.g. `"AlN (002)"` and `"AlN (110)"` would both become `"AlN"`, silently overwriting one value with the other when merged. -- **`coefficient_expansion`** — expands leading/trailing/nested bracket coefficient patterns; internally also normalizes trailing zeros and removes zero-coefficient elements as part of expansion. +- **`abbreviation_filtering`**: drops composition keys containing 2+ consecutive capital letters (abbreviations/junk keys, e.g. `"PVDF"`). +- **`element_validation_strict`**: keep only compositions whose key fully resolves to valid periodic-table element symbols. +- **`element_validation_lenient`**: a weaker companion of `element_validation_strict`. Instead of requiring the *entire* key to be pure elements, it keeps a composition as long as it contains at least one embedded formula fragment anywhere in the text. For example, `"Cellulose nanofibers/BaTiO3@TiO2/Polyvinylidene fluoride-(%)"` is kept because `"BaTiO3"` and `"TiO2"` each parse as elements, even though the rest is descriptive text. Only compositions with *no* recognizable formula fragment anywhere are dropped. If `element_validation_strict` is also selected, its stricter result wins: selecting both together gives no additional compositions beyond what `element_validation_strict` alone would keep. +- **`text_normalization`**: deterministic cleanup, strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens (e.g. `" Bi4Ti3O12 ultrathin with oxygen vacancies "` → `"Bi4Ti3O12 Ultrathin with Oxygen Vacancies"`). Tokens containing digits (real formula segments) and tokens that fully parse as element symbols (e.g. `"NaCl"`) are left untouched, as are all-caps abbreviations (e.g. `"PVDF"`). +- **`miller_indices`**: drops compositions carrying a crystal-plane notation like `(002)`, `(111)`, `(100)`, etc. entirely, rather than stripping the notation and keeping the bare formula. Stripping and keeping would collapse distinct surface-orientation entries for the same material onto the same dict key, e.g. `"AlN (002)"` and `"AlN (110)"` would both become `"AlN"`, silently overwriting one value with the other when merged. +- **`coefficient_expansion_strict`**: expands leading/trailing/nested bracket coefficient patterns; internally also normalizes trailing zeros and removes zero-coefficient elements as part of expansion. Compositions left with any residual `()`, `[]`, or `*` afterward are dropped as unresolved. +- **`coefficient_expansion_lenient`**: a weaker companion of `coefficient_expansion_strict`. Performs the same bracket expansion, but spares compositions with *balanced* brackets (equal open/close counts, no stray `*`) whose bracket content is genuine text rather than a failed arithmetic expression, e.g. `"(Bi0.5Ag0.5)ZrO3-(as-sintered)"` is kept as `"Bi0.5Ag0.5ZrO3-(as-sintered)"` instead of being dropped as unresolved. If `coefficient_expansion_strict` is also selected, its stricter unresolved-filtering wins: selecting both together reverts to strict behavior. Pass an empty list (`cleaning_steps=[]`) to skip all the optional steps. #### :material-square-medium:`is_store_unresolved_compositions` _(bool)_ -When `True`, logs a split statistics line showing `source`, `filtered`, `unresolved`, and `resolved` composition-property pair counts, and saves both filtered compositions (dropped by `abbreviation_filtering`, `element_validation`, or `miller_indices`) and unresolved compositions (still containing parentheses, brackets, or multiplication operators after cleaning) to a JSON file keyed by DOI. Requires `is_save_composition_property_file=True`. +When `True`, logs a split statistics line showing `source`, `filtered`, `unresolved`, and `resolved` composition-property pair counts, and saves both filtered compositions (dropped by `abbreviation_filtering`, `element_validation_strict`, `element_validation_lenient`, or `miller_indices`) and unresolved compositions (still containing parentheses, brackets, or multiplication operators after cleaning) to a JSON file keyed by DOI. Requires `is_save_composition_property_file=True`. #### :material-square-medium:`unresolved_compositions_file` _(str)_ -Path to the JSON file where filtered and unresolved composition keys are saved, with `"filtered"` and `"unresolved"` as top-level keys and DOIs as sub-keys mapping to lists of composition strings. Used only when `is_store_unresolved_compositions=True`. +Path to the JSON file where filtered and unresolved composition keys are saved, with `"filtered"` and `"unresolved"` as top-level keys and DOIs as sub-keys mapping to lists of `{"composition": ..., "reason": ...}` entries. `reason` names the step that dropped the composition (e.g. `"element_validation_strict"`, `"miller_indices"`, `"unresolved_brackets_or_operators"`). Used only when `is_store_unresolved_compositions=True`. !!! info "Default Values" @@ -75,18 +77,21 @@ graph TD A[Start: Raw Extracted Data] --> B{abbreviation_filtering?} B -->|selected| B1[Abbreviation Filtering] B -->|skipped| C - B1 --> C{element_validation?} - C -->|selected| C1[Element Validation] - C -->|skipped| D - C1 --> D{text_normalization?} + B1 --> C{element_validation_strict?} + C -->|selected| C1[Element Validation - strict] + C -->|skipped| C2 + C1 --> C2{element_validation_lenient?} + C2 -->|selected| C3[Element Validation - lenient] + C2 -->|skipped| D + C3 --> D{text_normalization?} D -->|selected| D1[Text Normalization] D -->|skipped| M D1 --> M{miller_indices?} M -->|selected| M1[Drop compositions with Miller indices] M -->|skipped| F - M1 --> F[Unicode Conversion - always] - F --> G[Arithmetic/Fraction Resolution - always] - G --> I{coefficient_expansion?} + M1 --> F["Unicode Conversion (always)"] + F --> G["Arithmetic/Fraction Resolution (always)"] + G --> I{coefficient_expansion_strict or coefficient_expansion_lenient?} I -->|selected| I1[Coefficient Expansion - incl. normalization and zero-coefficient removal] I -->|skipped| J I1 --> J[End: Cleaned Results] @@ -97,39 +102,41 @@ graph TD style G fill:#fff4e1 ``` -!!! note "miller_indices drops entries, it does not strip-and-keep" - - `miller_indices` removes the **entire composition entry**, not just the `(002)`-style notation. Stripping the notation and keeping the bare formula would collapse distinct surface-orientation measurements for the same material onto the same dict key — e.g. `"AlN (002)": 3` and `"AlN (110)": 6` would both resolve to `"AlN"`, and merging the results (`_return_in_dict`) would silently overwrite one value with the other. Dropping both entirely avoids that data loss; dropped keys are tracked in `filtered_compositions`, the same as `abbreviation_filtering`/`element_validation` drops. - - This detection must also happen **before** the mandatory arithmetic/bracket resolution: a bare 3-digit parenthetical like `(002)` is indistinguishable from a "coefficient bracket" to that resolver and to `coefficient_expansion`'s own bracket-stripping logic — left unremoved, either would fold its digits straight into the preceding element (e.g. `AlN (002)` → `AlN2`/`AlN 002`). Running the `miller_indices` check first, while the brackets are still in their original shape, avoids this. If `miller_indices` is **not** selected, a bare 3-digit-only bracket is instead left untouched by both of those steps and is dropped later as an unresolved composition, the same as any other leftover bracket — so it is never silently merged into a wrong formula either way. - ### Process Stages -#### 1. Abbreviation Filtering _(optional — `abbreviation_filtering`)_ +##### 1. Abbreviation Filtering _(optional — `abbreviation_filtering`)_ Drops composition keys containing 2+ consecutive capital letters (abbreviations, junk keys). -#### 2. Element Validation _(optional — `element_validation`)_ +##### 2. Element Validation _(optional — `element_validation_strict`)_ Verifies compositions contain only valid periodic elements. -#### 3. Text Normalization _(optional — `text_normalization`)_ +##### 3. Element Validation (Lenient) _(optional — `element_validation_lenient`)_ + +Keeps a composition if it contains at least one embedded formula fragment anywhere in the text (letters-only runs, split at digits/punctuation, that individually parse as element symbols), instead of requiring the whole key to be pure elements. Drops only compositions with no recognizable formula fragment anywhere. Has no additional effect when `element_validation_strict` is also selected: the stricter result wins. + +##### 4. Text Normalization _(optional — `text_normalization`)_ Strips leading/trailing whitespace, collapses runs of multiple spaces down to one, and title-cases descriptive word tokens, leaving formula segments (tokens with digits, or tokens that fully parse as element symbols) and all-caps abbreviations untouched. -#### 4. Miller Indices Filtering _(optional — `miller_indices`)_ +##### 5. Miller Indices Filtering _(optional — `miller_indices`)_ -Drops any composition entry carrying a crystal plane notation like `(002)`, `(111)`, `(100)`, etc. entirely (not just the notation). Runs before Unicode/arithmetic resolution — see the note above. +Drops any composition entry carrying a crystal plane notation like `(002)`, `(111)`, `(100)`, etc. entirely (not just the notation). Runs before Unicode/arithmetic resolution, see the following note. -#### 5. Unicode Conversion _(always runs)_ +!!! note "miller_indices drops entries, it does not strip-and-keep" + + `miller_indices` removes the **entire composition entry**, not just the `(002)`-style notation. Stripping the notation and keeping the bare formula would collapse distinct surface-orientation measurements for the same material onto the same dict key, e.g. `"AlN (002)": 3` and `"AlN (110)": 6` would both resolve to `"AlN"`, and merging the results (`_return_in_dict`) would silently overwrite one value with the other. Dropped keys are tracked in `filtered_compositions`, the same as `abbreviation_filtering`/`element_validation_strict` drops. + +##### 6. Unicode Conversion _(always runs)_ Converts subscript Unicode characters to regular digits for standardization. -#### 6. Arithmetic Resolution _(always runs)_ +##### 7. Arithmetic Resolution _(always runs)_ Evaluates mathematical expressions and fractional compositions. -#### 7. Coefficient Expansion _(optional — `coefficient_expansion`)_ +##### 8. Coefficient Expansion _(optional — `coefficient_expansion_strict` / `coefficient_expansion_lenient`)_ Expands coefficient patterns in chemical formulas, including: @@ -137,9 +144,26 @@ Expands coefficient patterns in chemical formulas, including: - **Trailing coefficients**: Multiplies all elements inside parentheses by trailing coefficient - **Parenthetical coefficients**: Expands nested brackets with complex coefficient multiplication -It also normalizes trailing zeros and removes zero-coefficient elements internally as part of expansion — there are no separate steps for those. +Both `coefficient_expansion_strict` and `coefficient_expansion_lenient` trigger this same expansion logic. They differ only in what happens to compositions still carrying brackets/`*` afterward: `coefficient_expansion_strict` drops any of them as unresolved, while `coefficient_expansion_lenient` (when selected without `coefficient_expansion_strict`) spares compositions whose leftover brackets are balanced and contain genuine text rather than a failed arithmetic expression. + +It also normalizes trailing zeros and removes zero-coefficient elements internally as part of expansion. There are no separate steps for those. + + +### Examples of Various Cleaning Steps -## Text Normalization Examples +#### Element Validation + +Keeps only compositions that resolve to elements, or, in the lenient variant, contain at least one embedded formula fragment: + +| Input Composition | `element_validation_strict` alone | `element_validation_lenient` alone | +| --- | --- | --- | +| `BaTiO3` | kept, unchanged | kept, unchanged | +| `Cellulose nanofibers/BaTiO3@TiO2/Polyvinylidene fluoride-(%)` | dropped (tracked in `filtered_compositions`) | kept, unchanged | +| `beta-glycine-polydimethylsiloxane` | dropped (tracked in `filtered_compositions`) | dropped (tracked in `filtered_compositions`) | + +Selecting both `element_validation_strict` and `element_validation_lenient` together reverts to the strict column above. + +#### Text Normalization Examples Normalizes whitespace and title-cases descriptive words; formula segments are untouched: @@ -148,13 +172,11 @@ Normalizes whitespace and title-cases descriptive words; formula segments are un | `Bi4Ti3O12 ultrathin with oxygen vacancies` | `Bi4Ti3O12 Ultrathin with Oxygen Vacancies` | | ` Bi4Ti3O12 ultrathin with oxygen vacancies ` | `Bi4Ti3O12 Ultrathin with Oxygen Vacancies` | | `BaTiO3 XRD pattern` | `BaTiO3 XRD Pattern` | -| `NaCl` | `NaCl` (unchanged — fully parses as elements) | - -## Advanced Cleaning Examples +| `NaCl` | `NaCl` (unchanged, fully parses as elements) | -### Miller Indices Filtering +#### Miller Indices Filtering -Drops any composition entry carrying a crystal plane notation entirely — it is not stripped down to the bare formula, since that would collapse distinct surface-orientation entries for the same material onto the same key: +Drops any composition entry carrying a crystal plane notation entirely. It is not stripped down to the bare formula, since that would collapse distinct surface-orientation entries for the same material onto the same key: | Input Composition | Result | | ------------------ | ------ | @@ -162,36 +184,17 @@ Drops any composition entry carrying a crystal plane notation entirely — it is | `ZnO (101)` | dropped (tracked in `filtered_compositions`) | | `BaTiO3` | kept, unchanged | -### Coefficient Expansion - -#### Leading Coefficient Expansion - -Multiplies all elements inside parentheses by the coefficient before the opening bracket: - -| Input Formula | Output Formula | -| ---------------------- | ------------------- | -| `0.7(K0.48Na0.52NbO3)` | `K0.336Na0.364NbO3` | -| `(0.15)Dy2O3` | `Dy0.3O0.45` | - -#### Trailing Coefficient Expansion - -Multiplies all elements inside parentheses by the coefficient after the closing bracket: - -| Input Formula | Output Formula | -| ----------------------- | ------------------- | -| `(K0.5Na0.5)(0.97)NbO3` | `K0.485Na0.485NbO3` | -| `(Bi0.5Na0.5)0.94TiO3` | `Bi0.47Na0.47TiO3` | - -#### Parenthetical Coefficient Expansion +#### Coefficient Expansion -Handles nested brackets and complex coefficient multiplication: +Expands coefficients exactly like `coefficient_expansion_strict`, but keeps compositions whose leftover brackets are balanced and contain genuine text instead of dropping them as unresolved: -| Input Formula | Output Formula | -| ----------------------------- | ------------------------ | -| `[(K0.5Na0.5)0.96Bi0.04]NbO3` | `K0.48Na0.48Bi0.04NbO3` | -| `[Ba0.85Ca0.15]0.99TiO3` | `Ba0.8415Ca0.1485TiO3` | +| Input Composition | `coefficient_expansion_strict` alone | `coefficient_expansion_lenient` alone | +| --- | --- | --- | +| `(Bi0.5Ag0.5)ZrO3-(as-sintered)` | dropped (tracked in `unresolved_compositions`) | `Bi0.5Ag0.5ZrO3-(as-sintered)` | +| `0.03*(Bi0.5Ag0.5)ZrO3` (stray `*` never expanded) | dropped (tracked in `unresolved_compositions`) | dropped (tracked in `unresolved_compositions`); a stray `*` is always treated as a genuine failure | +| `0.7(K0.48Na0.52NbO3)` | `K0.336Na0.364Nb0.7O2.1` | `K0.336Na0.364Nb0.7O2.1` | -Coefficient expansion also normalizes trailing zeros (`Pb0.90La0.10` → `Pb0.9La0.1`) and removes zero-coefficient elements (`BaTiZr0O3` → `BaTiO3`) internally as part of the same step. +Selecting both `coefficient_expansion_strict` and `coefficient_expansion_lenient` together reverts to the strict column above. !!! tip "Original vs Resolved Compositions" diff --git a/src/comproscanner/__init__.py b/src/comproscanner/__init__.py index 96c8a16e..38891fb1 100644 --- a/src/comproscanner/__init__.py +++ b/src/comproscanner/__init__.py @@ -163,9 +163,12 @@ def clean_data( is_save_composition_property_file (bool, optional): Whether to save composition-property values to a separate file. Defaults to True. composition_property_file (str, optional): Path to the composition-property file containing a dictionary of composition-property data. Defaults to "composition_property.json". cleaning_steps (Union[str, List[str]], optional): Either "all" (default, every optional step - enabled) or a list of step names: "abbreviation_filtering", "element_validation", - "text_normalization", "miller_indices", "coefficient_expansion". Unicode subscript - conversion and arithmetic/fraction resolution always run regardless of this parameter. + enabled) or a list of step names: "abbreviation_filtering", "element_validation_strict", + "element_validation_lenient", "text_normalization", "miller_indices", + "coefficient_expansion_strict", "coefficient_expansion_lenient". The "_lenient" steps are + weaker companions of their "_strict" counterparts and have no additional effect when both + are selected together. Unicode subscript conversion and arithmetic/fraction resolution + always run regardless of this parameter. """ scanner = ComProScanner(main_property_keyword=main_property_keyword) return scanner.clean_data( diff --git a/src/comproscanner/comproscanner.py b/src/comproscanner/comproscanner.py index 8a9d6f32..f76b09e0 100644 --- a/src/comproscanner/comproscanner.py +++ b/src/comproscanner/comproscanner.py @@ -157,7 +157,7 @@ def process_articles( save_failed_pdf_report (bool, optional): For `pdfs` source only. If True, save skipped/failed filename-based DOI fallback cases to a text report. Defaults to True. failed_pdf_report_path (str, optional): For `pdfs` source only. Custom path for failed PDF filename report. Defaults to None (uses `{folder_path}/failed_pdf_filenames.txt`). save_failed_automated_report (bool, optional): For automated publisher sources (elsevier, springer, iop, wiley). If True, save failed/unparseable articles to a report. Defaults to True. - failed_automated_report_path (str, optional): Custom path for the automated failure report. Defaults to None (uses `results/failed_automated_articles.txt`).. + failed_automated_report_path (str, optional): Custom path for the automated failure report. Defaults to None (uses `results/article_processor_failed_articles.txt`).. Raises: ValueErrorHandler: If property_keywords is not provided. @@ -737,14 +737,24 @@ def clean_data( cleaning_steps (Union[str, List[str]], optional): Either "all" (default, every optional step enabled) or a list of step names selecting exactly which optional steps run: "abbreviation_filtering" (drop keys with 2+ consecutive capital letters), - "element_validation" (keep only compositions resolving to valid periodic-table elements), + "element_validation_strict" (keep only compositions resolving to valid periodic-table + elements), + "element_validation_lenient" (weaker companion of element_validation_strict: keeps a + composition if it contains at least one embedded formula fragment, e.g. "BaTiO3" inside + "Cellulose nanofibers/BaTiO3@TiO2/...", instead of requiring the whole key to be pure + elements), "text_normalization" (normalizes whitespace and title-cases descriptive word tokens), "miller_indices" (drop compositions carrying crystal-plane notations like "(002)" entirely, - to avoid collapsing distinct surface-orientation entries onto the same key), and - "coefficient_expansion" (expand bracket coefficients; also normalizes trailing zeros and - removes zero-coefficient elements internally). Unicode subscript conversion and - arithmetic/fraction resolution always run regardless of this parameter, since the other - steps depend on their output. + to avoid collapsing distinct surface-orientation entries onto the same key), + "coefficient_expansion_strict" (expand bracket coefficients; also normalizes trailing + zeros and removes zero-coefficient elements internally), and + "coefficient_expansion_lenient" (weaker companion of coefficient_expansion_strict: also + expands bracket coefficients, but spares compositions with balanced brackets containing + genuine text/annotations, e.g. "...-(as-sintered)", from being dropped as unresolved). + element_validation_lenient/coefficient_expansion_lenient have no additional effect when + their strict counterpart is also selected — the strict step's result wins. Unicode + subscript conversion and arithmetic/fraction resolution always run regardless of this + parameter, since the other steps depend on their output. is_store_unresolved_compositions (bool, optional): Whether to log resolution statistics and save unresolved composition keys to a file. Requires is_save_composition_property_file=True. Defaults to False. unresolved_compositions_file (str, optional): Path to the file where unresolved composition keys will be saved. Used only when is_store_unresolved_compositions=True. Defaults to "unresolved_compositions.json". diff --git a/src/comproscanner/post_processing/data_cleaner.py b/src/comproscanner/post_processing/data_cleaner.py index fcae2837..0dfb00ac 100644 --- a/src/comproscanner/post_processing/data_cleaner.py +++ b/src/comproscanner/post_processing/data_cleaner.py @@ -28,15 +28,18 @@ class CleaningStep(str, Enum): """Individually selectable optional data-cleaning steps. Unicode subscript conversion and arithmetic/fraction resolution are not - part of this enum — they always run, since coefficient_expansion, - miller_indices, and element_validation all assume their output. + part of this enum — they always run, since coefficient_expansion_strict/ + _lenient, miller_indices, and element_validation_strict/_lenient all + assume their output. """ ABBREVIATION_FILTERING = "abbreviation_filtering" - ELEMENT_VALIDATION = "element_validation" + ELEMENT_VALIDATION_STRICT = "element_validation_strict" + ELEMENT_VALIDATION_LENIENT = "element_validation_lenient" TEXT_NORMALIZATION = "text_normalization" MILLER_INDICES = "miller_indices" - COEFFICIENT_EXPANSION = "coefficient_expansion" + COEFFICIENT_EXPANSION_STRICT = "coefficient_expansion_strict" + COEFFICIENT_EXPANSION_LENIENT = "coefficient_expansion_lenient" @classmethod def all(cls) -> List[str]: @@ -55,8 +58,11 @@ def __init__(self, results_file: str): self.results_file = results_file self.all_data = self._load_results() self.all_elements = get_all_elements() - self.filtered_compositions: Dict[str, List[str]] = {} - self.unresolved_compositions: Dict[str, List[str]] = {} + self.filtered_compositions: Dict[str, List[Dict[str, str]]] = {} + self.unresolved_compositions: Dict[str, List[Dict[str, str]]] = {} + # Placeholder -> original-text map for protected percent annotations + # (e.g. "7 wt% NiO", "0.1%MgO"), reset at the start of every clean run. + self._pct_annotation_map: Dict[str, str] = {} def _load_results(self) -> Dict[str, Any]: """Load results from JSON file.""" @@ -78,7 +84,13 @@ def _filter_invalid_keys(self, dict_list, doi: str = ""): for d in dict_list: if any(re.search(pattern, key) for key in d.keys()): if doi: - self.filtered_compositions.setdefault(doi, []).extend(d.keys()) + self.filtered_compositions.setdefault(doi, []).extend( + { + "composition": key, + "reason": CleaningStep.ABBREVIATION_FILTERING.value, + } + for key in d.keys() + ) else: valid.append(d) return valid @@ -106,6 +118,89 @@ def _can_parse_as_elements(self, s: str) -> bool: return False + # Matches the numeric+percent-sign prefix of weight/mole/atomic-percent + # dopant annotations such as "7 wt% NiO", "0.1 mol% Fe2O3", or "0.1%MgO". + # What follows this prefix (a plain token or a bracketed expression) is + # captured separately in _protect_percent_annotations, since a bracket's + # length is variable and can't be matched by a fixed-width regex group. + _PERCENT_ANNOTATION_PATTERN = re.compile( + r"(?::\s*)?(\d+(?:\.\d+)?)\s*(?:wt\.?|mol\.?|at\.?)?\s*%\s*" + ) + _PERCENT_ANNOTATION_TARGET_TOKEN = re.compile(r"[A-Za-z][A-Za-z0-9]*") + + # Shared prefix for the inert placeholders substituted in for percent-annotations. + # Lowercase-letters-only is intentional (keeps bracket-content char-class checks elsewhere + # treating it as plain text), but that same lowercase-ness means the "reject an element match + # immediately followed by more lowercase word text" anti-corruption checks below must special-case + # it explicitly, otherwise a genuine element right before a protected annotation gets skipped. + _PCT_PLACEHOLDER_PREFIX = "zzzpctannot" + + def _protect_percent_annotations(self, text: str) -> str: + """Replace percent-based dopant annotations with inert placeholders. + + The target being annotated (what follows the "%") may be a plain + token (e.g. "NiO") or a bracketed multi-term expression (e.g. + "(0.78PbO-0.22CuO)") — the latter's length is variable, so it can't + be captured by _PERCENT_ANNOTATION_PATTERN itself; it's located here + via _find_matching_close_bracket instead. + + Args: + text (str): Raw composition key, before arithmetic/coefficient processing. + + Returns: + str: Same text with each percent annotation replaced by a placeholder. + """ + result = [] + pos = 0 + for match in self._PERCENT_ANNOTATION_PATTERN.finditer(text): + if match.start() < pos: + continue # already consumed by a previous annotation's span + + end = match.end() + if end < len(text) and text[end] in "([": + close_pos = self._find_matching_close_bracket(text, end) + if close_pos == -1: + continue # unbalanced bracket — leave unprotected + end = close_pos + 1 + else: + token_match = self._PERCENT_ANNOTATION_TARGET_TOKEN.match(text, end) + if not token_match: + continue # nothing sensible to protect + end = token_match.end() + + result.append(text[pos : match.start()]) + placeholder = ( + f"{self._PCT_PLACEHOLDER_PREFIX}{len(self._pct_annotation_map)}zzz" + ) + self._pct_annotation_map[placeholder] = text[match.start() : end] + result.append(placeholder) + pos = end + + result.append(text[pos:]) + return "".join(result) + + def _restore_percent_annotations_in_dict( + self, comp_prop_dict: Dict[str, Any] + ) -> Dict[str, Any]: + """Restore any protected percent annotations in composition keys. + + Args: + comp_prop_dict (Dict[str, Any]): Composition -> property-value mapping, potentially still containing placeholders from `_protect_percent_annotations`. + + Returns: + Dict[str, Any]: Same mapping with placeholders replaced by their original percent-annotation text. + """ + if not self._pct_annotation_map: + return comp_prop_dict + restored = {} + for comp, val in comp_prop_dict.items(): + restored_comp = comp + for placeholder, original in self._pct_annotation_map.items(): + if placeholder in restored_comp: + restored_comp = restored_comp.replace(placeholder, original) + restored[restored_comp] = val + return restored + def _is_elements(self, comp_pro_pair: Dict[str, Any]) -> bool: """Check whether the composition key in a pair can be fully parsed as a sequence of valid element symbols. @@ -159,21 +254,251 @@ def _remove_special_chars(string: str) -> str: except Exception: return False + def _contains_element_token(self, comp_pro_pair: Dict[str, Any]) -> bool: + """Check whether the composition key contains at least one embedded formula fragment, even if the key as a whole is not purely elements. + + Args: + comp_pro_pair (Dict[str, Any]): A single {composition: value} dictionary. + + Returns: + bool: True if any letter-run in the key parses as element symbols. + """ + try: + key = str(next(iter(comp_pro_pair))) + except Exception: + return False + + for token in re.findall(r"[A-Za-z]+", key): + if re.search(r"[A-Z]", token) and self._can_parse_as_elements(token): + return True + return False + + def _has_balanced_annotated_brackets(self, comp: str) -> bool: + """Check whether a composition's leftover brackets are balanced and + contain genuine text (an annotation), rather than a failed/partial + arithmetic or coefficient expression. + + Used by the coefficient_expansion_lenient step to spare compositions + like "...-(%)" or "...(as-sintered)" from being dropped as + unresolved, while still treating unmatched brackets, a stray "*", or + brackets containing only leftover numbers/operators as genuine + cleaning failures. + + Args: + comp (str): Composition key to check. + + Returns: + bool: True if brackets are balanced and at least one bracketed + span contains non-arithmetic content. + """ + if "*" in comp: + return False + if comp.count("(") != comp.count(")"): + return False + if comp.count("[") != comp.count("]"): + return False + + for open_content, bracket_content in re.findall( + r"\(([^()]*)\)|\[([^\[\]]*)\]", comp + ): + content = open_content or bracket_content + if content and not re.match(r"^[0-9.\s+\-]*$", content): + return True + return False + + def _find_matching_close_bracket(self, text: str, open_pos: int) -> int: + """Given text[open_pos] is '(' or '[', return the index of its + depth-aware matching close bracket (handles mixed ()/[] nesting), or + -1 if unbalanced.""" + depth = 0 + for i in range(open_pos, len(text)): + if text[i] in "([": + depth += 1 + elif text[i] in ")]": + depth -= 1 + if depth == 0: + return i + return -1 + + def _split_top_level_terms(self, text: str) -> List[str]: + """Split text on +/-/– that occur at bracket depth 0, keeping each + sign attached to the term that follows it. Unlike a plain regex + split, this does not split on a +/- hidden inside a bracket's own + multi-term content (e.g. the "-" inside "(0.89A-0.11B)").""" + parts: List[str] = [] + depth = 0 + start = 0 + for i, ch in enumerate(text): + if ch in "([": + depth += 1 + elif ch in ")]": + depth -= 1 + elif ch in "+-–" and depth == 0 and i > start: + parts.append(text[start:i]) + start = i + parts.append(text[start:]) + return parts + + def _formula_prefix_end(self, text: str) -> int: + """Length of the longest prefix of `text` that is valid chemical- + formula content (Element[coefficient] tokens, +/-/– joins, and + brackets whose content is itself fully valid by this same rule). + + Everything from the first point that breaks this — a non-element + token, an element that's actually the start of a longer descriptive + word, a bracket containing free text, plain whitespace, etc. — is + not formula content and must never be scaled by a coefficient that + appeared earlier in the string. This is the single shared rule + behind why e.g. "Bottom", "Reoxidized", and "PbTiO3 (calcined at + 660C)" must each stop being touched at a specific point rather than + needing a bespoke check for every new shape of non-formula text. + """ + element_pattern = re.compile(r"[A-Z][a-z]?") + number_pattern = re.compile(r"[0-9]+(?:\.[0-9]+)?") + pos = 0 + n = len(text) + while pos < n: + ch = text[pos] + if ch in "([": + close = self._find_matching_close_bracket(text, pos) + if close == -1: + break + inner = text[pos + 1 : close] + if self._formula_prefix_end(inner) != len(inner): + break + pos = close + 1 + continue + if ch in "+-–": + pos += 1 + continue + m = element_pattern.match(text, pos) + if not m or m.group(0) not in self.all_elements: + break + end = m.end() + num_m = number_pattern.match(text, end) + if num_m: + end = num_m.end() + next_text = text[end:] + if ( + next_text[:1].isalpha() + and next_text[:1].islower() + and not next_text.startswith(self._PCT_PLACEHOLDER_PREFIX) + ): + break + pos = end + return pos + + def _distribute_multiterm_brackets(self, formula: str) -> str: + """Resolve nested multi-term coefficient*(term1±term2±...) expressions + (e.g. "0.75*(0.89(Bi0.5Na0.5)TiO3-0.11BaTiO3)") by distributing the + outer coefficient into each of the bracket's own top-level +/- + separated sub-terms, multiplying it into that sub-term's own leading + coefficient. Only the numeric coefficients are combined here — + element-level scaling (e.g. expanding "Bi0.5Na0.5") is left to the + existing coefficient_expansion pipeline, which already scales + elements correctly through a bracket wrapper regardless of what's + nested inside it. This purely eliminates +/- signs that would + otherwise be hidden inside a bracket, which the (bracket-depth- + unaware) top-level formula splitting elsewhere cannot handle safely. + + No explicit recursion is needed for deeper nesting: each successful + splice restarts the scan over the updated formula, so a spliced-in + sub-term that is itself a further-nested coeff*(multi-term) shape + gets picked up and resolved on a later pass. + """ + if not formula or not isinstance(formula, str): + return formula + + # Anchored to term-start (start-of-string or right after a top-level + # +/-/–), mirroring the leading-coefficient-before-bracket pattern in + # _expand_leading_and_trailing_coefficients — not just "any digit run + # before a bracket", which would false-positive-match an element's + # own trailing coefficient sitting next to an unrelated bracket (e.g. + # the "0.15" in "Ca0.15(Zr0.1-Ti0.9)O3" belongs to Ca, not a + # multiplier for the bracket). + pattern = re.compile( + r"(?:^|(?P[+\-–]))\s*(?P\d+(?:\.\d+)?)\*?\s*(?P[\(\[])" + ) + iterations = 0 + while iterations < 20: + iterations += 1 + progressed = False + for m in pattern.finditer(formula): + open_pos = m.start("open") + close_pos = self._find_matching_close_bracket(formula, open_pos) + if close_pos == -1: + continue + inner = formula[open_pos + 1 : close_pos] + + # Defer pure-numeric arithmetic parens (e.g. "0.5*(0.2+0.3)") + # to the existing arithmetic-evaluation machinery — this + # function only handles brackets containing actual + # composition text. + if re.match(r"^[0-9.\s+\-*/]+$", inner): + continue + + sub_terms = self._split_top_level_terms(inner) + if len(sub_terms) <= 1: + # Single-term bracket — leave to the existing + # coefficient_expansion pipeline. + continue + + outer_coeff = float(m.group("coeff")) + negate = m.group("sign") in ("-", "–") + # When negating, consume the leading "-" into the splice so + # it can be folded into each sub-term's flipped sign; + # otherwise leave any "+" (or start-of-string) untouched, + # matching how the rest of the formula is joined. + splice_start = m.start("sign") if negate else m.start("coeff") + + resolved_terms = [] + for term in sub_terms: + sign = "" + body = term + if body[:1] in ("+", "-", "–"): + sign = "-" if body[0] in ("-", "–") else "+" + body = body[1:] + body = body.strip() + + inner_match = re.match(r"^(\d+(?:\.\d+)?)\*?", body) + if inner_match: + inner_coeff = float(inner_match.group(1)) + remainder = body[inner_match.end() :].strip() + else: + inner_coeff = 1.0 + remainder = body + + if negate: + sign = "+" if sign == "-" else "-" + + new_coeff = round(outer_coeff * inner_coeff, 8) + new_coeff_str = ( + str(int(new_coeff)) + if new_coeff == int(new_coeff) + else f"{new_coeff:.8f}".rstrip("0").rstrip(".") + ) + resolved_terms.append(f"{sign}{new_coeff_str}{remainder}") + + replacement = "".join(resolved_terms) + if replacement.startswith("+"): + replacement = replacement[1:] + + formula = ( + formula[:splice_start] + replacement + formula[close_pos + 1 :] + ) + progressed = True + break + + if not progressed: + break + + return formula + _TITLE_CASE_STOPWORDS = {"with", "of", "at", "for", "the", "and"} def _normalize_text(self, dict_list): """Normalize whitespace and title-case descriptive word tokens in composition keys. - Strips leading/trailing whitespace and collapses runs of whitespace - (multiple spaces, tabs) down to a single space, then title-cases - tokens that are purely alphabetic, not already an all-caps - abbreviation (e.g. "XRD" stays "XRD"), and do NOT fully parse as a - sequence of element symbols (so formula-only tokens like "NaCl" or - "Ti" are left exactly as extracted). Tokens containing digits (real - formula segments like "Bi4Ti3O12") are left untouched. Does not - insert a space where none exists in the source string (e.g. words - glued directly to numbers, like "Milling15h", stay glued). - Args: dict_list (list): List of single-entry dicts mapping composition strings to values. @@ -194,6 +519,8 @@ def format_key(key: str) -> str: lower = tok.lower() if i > 0 and lower in self._TITLE_CASE_STOPWORDS: formatted.append(lower) + elif self._can_parse_as_elements(tok.capitalize()): + formatted.append(tok) else: formatted.append(tok.capitalize()) else: @@ -227,13 +554,41 @@ def _clean_comp_prop_pairs( if CleaningStep.ABBREVIATION_FILTERING.value in steps: comp_prop_pairs = self._filter_invalid_keys(comp_prop_pairs, doi) - if CleaningStep.ELEMENT_VALIDATION.value in steps: + if CleaningStep.ELEMENT_VALIDATION_STRICT.value in steps: valid_pairs = [] for pair in comp_prop_pairs: if self._is_elements(pair): valid_pairs.append(pair) elif doi: - self.filtered_compositions.setdefault(doi, []).extend(pair.keys()) + self.filtered_compositions.setdefault(doi, []).extend( + { + "composition": key, + "reason": CleaningStep.ELEMENT_VALIDATION_STRICT.value, + } + for key in pair.keys() + ) + comp_prop_pairs = valid_pairs + + if CleaningStep.ELEMENT_VALIDATION_LENIENT.value in steps: + # Weaker than element_validation_strict: keeps a composition as long as + # it contains at least one embedded formula fragment (e.g. "BaTiO3" inside + # "Cellulose nanofibers/BaTiO3@TiO2/..."), instead of requiring the + # entire key to be pure elements. Applied as an additional sequential + # filter, so if element_validation_strict is also selected, its stricter + # result already excludes everything this step alone would allow — + # this step gives no extra ground back when both are selected. + valid_pairs = [] + for pair in comp_prop_pairs: + if self._contains_element_token(pair): + valid_pairs.append(pair) + elif doi: + self.filtered_compositions.setdefault(doi, []).extend( + { + "composition": key, + "reason": CleaningStep.ELEMENT_VALIDATION_LENIENT.value, + } + for key in pair.keys() + ) comp_prop_pairs = valid_pairs if CleaningStep.TEXT_NORMALIZATION.value in steps: @@ -253,7 +608,12 @@ def _clean_comp_prop_pairs( key = next(iter(d)) if self._remove_miller_indices(key) != key: if doi: - self.filtered_compositions.setdefault(doi, []).append(key) + self.filtered_compositions.setdefault(doi, []).append( + { + "composition": key, + "reason": CleaningStep.MILLER_INDICES.value, + } + ) else: kept_pairs.append(d) comp_prop_pairs = kept_pairs @@ -506,6 +866,10 @@ def _resolve_arithmetic_and_multiply(formula): if not formula or not isinstance(formula, str): return str(formula) if formula is not None else "" + # Step 0: Distribute nested multi-term coefficient*(term1±term2±...) + # expressions before anything else touches the bracket structure. + formula = self._distribute_multiterm_brackets(formula) + # Step 1: Evaluate ALL parenthetical expressions with arithmetic first formula = _evaluate_all_parenthetical_expressions(formula) @@ -531,8 +895,10 @@ def _add_composition_brackets(formula): - Don't add brackets at the very beginning if no coefficient - Don't add brackets after - if no coefficient follows the - """ - # Split formula by +/- operators while preserving them - parts = re.split(r"(?=[+\-])", formula) + # Split formula by +/- operators while preserving them, without + # splitting on a +/- hidden inside a bracket's own multi-term + # content (depth-aware — see _split_top_level_terms). + parts = self._split_top_level_terms(formula) processed_parts = [] for _, part in enumerate(parts): @@ -561,27 +927,56 @@ def _add_composition_brackets(formula): processed_parts.append(sign + coefficient) continue - # Check if composition part already has proper brackets at the outermost level + # Split off any trailing content that isn't valid formula + # text (per _formula_prefix_end), e.g. a descriptive + # annotation like "PbTiO3 (calcined at 660C)" — the + # annotation must never share the composition's + # coefficient/bracket scope, since it can incidentally + # contain single-letter unit symbols that are also valid + # periodic-table elements (e.g. "C" for Celsius, "K" for + # Kelvin), which coefficient_expansion would otherwise + # scale as if they were real stoichiometry. + boundary = self._formula_prefix_end(composition_part) + formula_part = composition_part[:boundary] + trailing_annotation = composition_part[boundary:] + + if not formula_part: + # Nothing formula-shaped follows the coefficient at + # all — leave everything untouched. + processed_parts.append( + sign + coefficient + " " + composition_part + ) + continue + + # Check if the formula part already has proper brackets at the outermost level if ( - composition_part.startswith("(") - and composition_part.endswith(")") - ) or ( - composition_part.startswith("[") - and composition_part.endswith("]") - ): + formula_part.startswith("(") and formula_part.endswith(")") + ) or (formula_part.startswith("[") and formula_part.endswith("]")): # Already properly bracketed - processed_parts.append(sign + coefficient + composition_part) + processed_parts.append( + sign + coefficient + formula_part + trailing_annotation + ) else: - # Determine bracket type based on whether parentheses exist in composition - if "(" in composition_part or ")" in composition_part: + # Determine bracket type based on whether parentheses exist in the formula part + if "(" in formula_part or ")" in formula_part: # Use square brackets processed_parts.append( - sign + coefficient + "[" + composition_part + "]" + sign + + coefficient + + "[" + + formula_part + + "]" + + trailing_annotation ) else: # Use round brackets processed_parts.append( - sign + coefficient + "(" + composition_part + ")" + sign + + coefficient + + "(" + + formula_part + + ")" + + trailing_annotation ) else: # No coefficient found at the beginning @@ -635,9 +1030,11 @@ def _convert_subscript_unicode_to_digits(string: str) -> str: for d in dict_list: new_dict = {} for key, value in d.items(): - # Step 0: Convert Unicode subscripts to regular digits + # Step 0: Convert Unicode subscripts to regular digits and protect percent annotations before any arithmetic or bracket processing processed_key = _convert_subscript_unicode_to_digits(str(key)) + processed_key = self._protect_percent_annotations(processed_key) + # Step 1: Convert simple fractions to decimals (handles both integer and decimal numerators/denominators) processed_key = re.sub( r"(\d+(?:\.\d+)?)/(\d+(?:\.\d+)?)", _replace_fraction, processed_key @@ -769,9 +1166,14 @@ def _expand_leading_and_trailing_coefficients(self, formula: str) -> str: return formula def multiply_element_coefficients(composition: str, multiplier: float) -> str: - """Multiply all element coefficients in a composition by a multiplier.""" - # Match element symbol followed by optional valid decimal number + """Multiply all element coefficients within composition's valid + formula prefix by multiplier; everything from the first + non-formula point onward (e.g. a trailing descriptive + annotation like " (calcined at 660C)") is left completely + untouched, per _formula_prefix_end.""" element_pattern = r"([A-Z][a-z]?)([0-9]+(?:\.[0-9]+)?)?" + boundary = self._formula_prefix_end(composition) + scannable, trailing = composition[:boundary], composition[boundary:] def replace_coeff(match): element = match.group(1) @@ -796,7 +1198,7 @@ def replace_coeff(match): return "" return f"{element}{formatted_coeff}" - return re.sub(element_pattern, replace_coeff, composition) + return re.sub(element_pattern, replace_coeff, scannable) + trailing max_iterations = 50 iteration = 0 @@ -863,23 +1265,29 @@ def replace_coeff(match): continue # Handle pattern 1a: Leading coefficient before bracket - # Match: 0.7(composition) or 0.7[composition] at start or after +/-/− - match = re.search(r"(?:^|([+\-\u2013]))(\d+(?:\.\d+)?)([\[\(])", formula) - if match: - operator = match.group(1) or "" - try: - coefficient = float(match.group(2)) - except ValueError: - iteration += 1 - continue - open_bracket = match.group(3) - close_bracket = "]" if open_bracket == "[" else ")" - - # Find the matching closing bracket + # Match: 0.7(composition), 0.7[composition], or the same with a + # space before the bracket (e.g. "0.7 (composition)") - the space + # is purely a formatting artifact and must not change the result. + # + # A candidate is only acted on if its bracket contents are at + # least partly genuine formula content (per _formula_prefix_end), + # or the coefficient is exactly 0 (always removable regardless of + # content). A bracket that's actually free text - e.g. + # "0.1 (Sandwich-structured)" or "20 (0 Phr hydroxyapatite, ...)" + # - must be left completely untouched, coefficient included, + # rather than having its digits silently dropped as if they'd + # been consumed into a scaling that never happened. Candidates + # are scanned left to right so an earlier non-formula bracket + # doesn't block a later, genuinely actionable one. + match = None + match_close_pos = -1 + for candidate in re.finditer( + r"(?:^|([+\-–]))(\d+(?:\.\d+)?)\s*([\[\(])", formula + ): + open_bracket = candidate.group(3) bracket_depth = 1 - start_pos = match.end() + start_pos = candidate.end() close_pos = -1 - for i in range(start_pos, len(formula)): if formula[i] in "([": bracket_depth += 1 @@ -888,33 +1296,47 @@ def replace_coeff(match): if bracket_depth == 0: close_pos = i break + if close_pos == -1: + continue + try: + coefficient = float(candidate.group(2)) + except ValueError: + continue + composition = formula[start_pos:close_pos] + if coefficient != 0 and self._formula_prefix_end(composition) == 0: + continue + match = candidate + match_close_pos = close_pos + break - if close_pos != -1: - composition = formula[start_pos:close_pos] + if match: + operator = match.group(1) or "" + coefficient = float(match.group(2)) + open_bracket = match.group(3) + close_bracket = "]" if open_bracket == "[" else ")" + start_pos = match.end() + close_pos = match_close_pos + composition = formula[start_pos:close_pos] - # If coefficient is zero, remove the entire bracketed section including operator - if coefficient == 0: + # If coefficient is zero, remove the entire bracketed section including operator + if coefficient == 0: + formula = formula[: match.start()] + formula[close_pos + 1 :] + else: + expanded = multiply_element_coefficients(composition, coefficient) + # If expansion results in empty string, remove the brackets entirely + if not expanded or expanded.strip() == "": formula = formula[: match.start()] + formula[close_pos + 1 :] else: - expanded = multiply_element_coefficients( - composition, coefficient + formula = ( + formula[: match.start()] + + operator + + open_bracket + + expanded + + close_bracket + + formula[close_pos + 1 :] ) - # If expansion results in empty string, remove the brackets entirely - if not expanded or expanded.strip() == "": - formula = ( - formula[: match.start()] + formula[close_pos + 1 :] - ) - else: - formula = ( - formula[: match.start()] - + operator - + open_bracket - + expanded - + close_bracket - + formula[close_pos + 1 :] - ) - changed = True - continue + changed = True + continue # Handle pattern 1b: (coefficient)composition # Match: (0.15)Dy2O3 where parenthesis contains only a number @@ -1028,10 +1450,18 @@ def _expand_parenthetical_coefficients( outer_coefficient = float(bracket_match.group(2)) element_pattern = r"([A-Z][a-z]?)([\d.]*)" - elements = re.findall(element_pattern, inner_content) + + # Only scale within the bracket content's valid formula + # prefix (per _formula_prefix_end) — anything from the + # first non-formula point onward is appended untouched + # rather than scanned for element-like matches at all. + boundary = self._formula_prefix_end(inner_content) + scannable = inner_content[:boundary] + trailing = inner_content[boundary:] expanded = "" - for element, coefficient_str in elements: + for m in re.finditer(element_pattern, scannable): + element, coefficient_str = m.group(1), m.group(2) if not element: continue @@ -1056,6 +1486,8 @@ def _expand_parenthetical_coefficients( ).rstrip(".") expanded += f"{element}{formatted_coeff}" + expanded += trailing + formula = ( formula[: bracket_match.start()] + expanded @@ -1122,7 +1554,10 @@ def _apply_advanced_composition_cleaning( for composition, property_value in comp_prop_dict.items(): cleaned_comp = composition - if CleaningStep.COEFFICIENT_EXPANSION.value in steps: + if steps & { + CleaningStep.COEFFICIENT_EXPANSION_STRICT.value, + CleaningStep.COEFFICIENT_EXPANSION_LENIENT.value, + }: cleaned_comp = self._expand_leading_and_trailing_coefficients( cleaned_comp ) @@ -1135,14 +1570,39 @@ def _apply_advanced_composition_cleaning( return cleaned_dict def _filter_unresolved_compositions( - self, comp_prop_dict: Dict[str, Any], doi: str = "" + self, comp_prop_dict: Dict[str, Any], doi: str = "", steps: Set[str] = None ) -> Dict[str, Any]: - """Remove individual compositions that still contain unresolved parentheses, brackets, or multiplication operators.""" + """Remove individual compositions that still contain unresolved parentheses, brackets, or multiplication operators. + + If coefficient_expansion_lenient is selected without + coefficient_expansion_strict, compositions with balanced + brackets/braces containing genuine text (not a stray "*" or leftover + arithmetic) are spared instead of being dropped — see + `_has_balanced_annotated_brackets`. Selecting both steps together + reverts to the strict behavior below. + + Args: + steps (Set[str], optional): Resolved set of selected CleaningStep + values, used to decide whether the lenient carve-out applies. + """ + steps = steps or set() + lenient_active = ( + CleaningStep.COEFFICIENT_EXPANSION_LENIENT.value in steps + and CleaningStep.COEFFICIENT_EXPANSION_STRICT.value not in steps + ) resolved = {} for comp, val in comp_prop_dict.items(): + if lenient_active and self._has_balanced_annotated_brackets(comp): + resolved[comp] = val + continue if re.search(r"[()[\]*]", comp): if doi: - self.unresolved_compositions.setdefault(doi, []).append(comp) + self.unresolved_compositions.setdefault(doi, []).append( + { + "composition": comp, + "reason": "unresolved_brackets_or_operators", + } + ) else: resolved[comp] = val return resolved @@ -1258,6 +1718,7 @@ def clean_data_with_selected_steps(self, steps: Set[str]) -> Dict[str, Any]: Args: steps: Resolved set of CleaningStep values to apply. """ + self._pct_annotation_map = {} result = {} for key, value in self.all_data.items(): comp_prop_data = self._get_comp_prop_data(value) @@ -1265,13 +1726,24 @@ def clean_data_with_selected_steps(self, steps: Set[str]) -> Dict[str, Any]: if cleaned_data: result[key] = value.copy() comp_prop_dict = self._return_in_dict(cleaned_data) - if CleaningStep.COEFFICIENT_EXPANSION.value in steps: + if steps & { + CleaningStep.COEFFICIENT_EXPANSION_STRICT.value, + CleaningStep.COEFFICIENT_EXPANSION_LENIENT.value, + }: comp_prop_dict = self._apply_advanced_composition_cleaning( comp_prop_dict, steps ) - # Remove compositions that still have unresolved brackets or math ops - comp_prop_dict = self._filter_unresolved_compositions( - comp_prop_dict, doi=key + # Only filter out unresolved brackets/operators when coefficient + # expansion was actually attempted. Without it selected, the + # brackets the mandatory arithmetic step adds around coefficient + # segments are expected and unexpanded on purpose — they are not + # "failures", so the composition should pass through untouched + # rather than being dropped. + comp_prop_dict = self._filter_unresolved_compositions( + comp_prop_dict, doi=key, steps=steps + ) + comp_prop_dict = self._restore_percent_annotations_in_dict( + comp_prop_dict ) result[key]["composition_data"][ "compositions_property_values" @@ -1288,10 +1760,14 @@ def clean_data_with_relevant_compositions( Args: cleaning_steps: Either "all" (default, every optional step enabled) or a list of step names selecting exactly which optional steps run: - abbreviation_filtering, element_validation, text_normalization, - miller_indices, coefficient_expansion. Unicode conversion and + abbreviation_filtering, element_validation_strict, element_validation_lenient, + text_normalization, miller_indices, coefficient_expansion_strict, + coefficient_expansion_lenient. Unicode conversion and arithmetic/fraction resolution always run regardless of this - parameter. + parameter. element_validation_lenient/coefficient_expansion_lenient + are weaker companions of element_validation_strict/coefficient_expansion_strict — + selecting both a step and its lenient companion together yields the + stricter step's result. Returns: Dict[str, Any]: Cleaned data based on selected steps. diff --git a/src/comproscanner/utils/configs/paths_config.py b/src/comproscanner/utils/configs/paths_config.py index df97dcc8..efe16f82 100644 --- a/src/comproscanner/utils/configs/paths_config.py +++ b/src/comproscanner/utils/configs/paths_config.py @@ -15,7 +15,7 @@ class DefaultPaths: # Keyword-independent paths — accessible as DefaultPaths.XXX without instantiation - FAILED_AUTOMATED_ARTICLES_FILENAME = "results/failed_automated_articles.txt" + FAILED_AUTOMATED_ARTICLES_FILENAME = "results/article_processor_failed_articles.txt" AGENTIC_EVALUATION_RESULT_FILENAME = "agentic_evaluation_result.json" DETAILED_EVALUATION_FILENAME = "detailed_evaluation.json" @@ -23,5 +23,7 @@ def __init__(self, main_property_keyword): # Keyword-dependent paths — require a keyword, accessed as self.all_paths.XXX self.METADATA_CSV_FILENAME = f"results/{main_property_keyword}_metadata.csv" self.TIMEOUT_DOI_LOG_FILENAME = f"logs/{main_property_keyword}_timeout_dois.txt" - self.PDF_PROCESSED_DOIS_FILENAME = f"logs/{main_property_keyword}_pdf_processed_dois.txt" + self.PDF_PROCESSED_DOIS_FILENAME = ( + f"logs/{main_property_keyword}_pdf_processed_dois.txt" + ) self.IOP_FOLDERPATH = os.getenv("IOP_papers_path") diff --git a/tests/test_post_processing/test_data_cleaner.py b/tests/test_post_processing/test_data_cleaner.py index 11d230bc..72f0e507 100644 --- a/tests/test_post_processing/test_data_cleaner.py +++ b/tests/test_post_processing/test_data_cleaner.py @@ -47,26 +47,34 @@ class TestCleaningStep: """Test cases for the CleaningStep enum.""" def test_cleaning_step_enum_values(self): - """Test that CleaningStep enum has the expected five values.""" + """Test that CleaningStep enum has the expected seven values.""" assert CleaningStep.ABBREVIATION_FILTERING == "abbreviation_filtering" - assert CleaningStep.ELEMENT_VALIDATION == "element_validation" + assert CleaningStep.ELEMENT_VALIDATION_STRICT == "element_validation_strict" + assert CleaningStep.ELEMENT_VALIDATION_LENIENT == "element_validation_lenient" assert CleaningStep.TEXT_NORMALIZATION == "text_normalization" assert CleaningStep.MILLER_INDICES == "miller_indices" - assert CleaningStep.COEFFICIENT_EXPANSION == "coefficient_expansion" + assert CleaningStep.COEFFICIENT_EXPANSION_STRICT == "coefficient_expansion_strict" + assert ( + CleaningStep.COEFFICIENT_EXPANSION_LENIENT == "coefficient_expansion_lenient" + ) def test_cleaning_step_all(self): - """Test that CleaningStep.all() returns exactly the five expected step names.""" + """Test that CleaningStep.all() returns exactly the seven expected step names.""" assert set(CleaningStep.all()) == { "abbreviation_filtering", - "element_validation", + "element_validation_strict", + "element_validation_lenient", "text_normalization", "miller_indices", - "coefficient_expansion", + "coefficient_expansion_strict", + "coefficient_expansion_lenient", } def test_cleaning_step_membership(self): """Test CleaningStep enum membership.""" - assert "element_validation" in CleaningStep + assert "element_validation_strict" in CleaningStep + assert "element_validation_lenient" in CleaningStep + assert "coefficient_expansion_lenient" in CleaningStep assert "miller_indices" in CleaningStep assert "invalid" not in CleaningStep assert "normalization" not in CleaningStep @@ -227,6 +235,41 @@ def test_is_elements_empty_dict(self, data_cleaner): result = data_cleaner._is_elements({}) assert result is False + def test_contains_element_token_finds_embedded_formula(self, data_cleaner): + """Test _contains_element_token detects a formula fragment embedded in text.""" + pair = { + "Cellulose nanofibers/BaTiO3@TiO2/Polyvinylidene fluoride-(%)": "value" + } + assert data_cleaner._contains_element_token(pair) is True + + def test_contains_element_token_pure_formula(self, data_cleaner): + """Test _contains_element_token detects a composition that is already pure elements.""" + assert data_cleaner._contains_element_token({"BaTiO3": "value"}) is True + + def test_contains_element_token_no_element_anywhere(self, data_cleaner): + """Test _contains_element_token returns False when no letter-run parses as elements.""" + assert data_cleaner._contains_element_token({"RandomTextNoElement": "value"}) is False + + def test_contains_element_token_empty_dict(self, data_cleaner): + """Test _contains_element_token with an empty dict.""" + assert data_cleaner._contains_element_token({}) is False + + def test_has_balanced_annotated_brackets_true_for_text_in_parens(self, data_cleaner): + """Test _has_balanced_annotated_brackets True for balanced brackets with text.""" + assert data_cleaner._has_balanced_annotated_brackets("Bi0.5Ag0.5ZrO3-(as-sintered)") is True + + def test_has_balanced_annotated_brackets_false_for_stray_asterisk(self, data_cleaner): + """Test _has_balanced_annotated_brackets False when a stray '*' remains.""" + assert data_cleaner._has_balanced_annotated_brackets("0.03*(Bi0.5Ag0.5)ZrO3") is False + + def test_has_balanced_annotated_brackets_false_for_unbalanced_brackets(self, data_cleaner): + """Test _has_balanced_annotated_brackets False when brackets are unmatched.""" + assert data_cleaner._has_balanced_annotated_brackets("BaTiO3-(unbalanced") is False + + def test_has_balanced_annotated_brackets_false_for_pure_arithmetic_content(self, data_cleaner): + """Test _has_balanced_annotated_brackets False when bracket content is purely numeric/arithmetic.""" + assert data_cleaner._has_balanced_annotated_brackets("BaTiO3(0.04-0.03)") is False + def test_normalize_text_title_cases_descriptive_words(self, data_cleaner): """Test _normalize_text title-cases descriptive word tokens and preserves spaces.""" dict_list = [ @@ -266,6 +309,18 @@ def test_normalize_text_strips_leading_and_trailing_whitespace(self, data_cleane result = data_cleaner._normalize_text(dict_list) assert result == [{"Bi4Ti3O12 Ultrathin with Oxygen Vacancies": "value1"}] + def test_normalize_text_does_not_capitalize_element_lookalike_units( + self, data_cleaner + ): + """Regression test: capitalizing a short unit-abbreviation token like + "h" (hours) would create "H" — a genuine periodic-table element + (Hydrogen) — which a later coefficient-expansion pass could then + scale as if it were real stoichiometry. Such tokens must be left in + their original (lowercase) form instead of being title-cased.""" + dict_list = [{"BaTiO3 sintered for 20 h": "value1"}] + result = data_cleaner._normalize_text(dict_list) + assert result == [{"BaTiO3 Sintered for 20 h": "value1"}] + def test_convert_fractions_and_resolve_compositions_fractions(self, data_cleaner): """Test _convert_fractions_and_resolve_compositions with fractions.""" dict_list = [ @@ -341,6 +396,95 @@ def test_convert_fractions_element_coefficient_multiplications(self, data_cleane assert "Ta" in second_key assert "Nb" in second_key + def test_distribute_multiterm_brackets_user_example(self, data_cleaner): + """Regression test: an outer coefficient multiplying a multi-term + bracket (terms separated by +/- inside, each with its own inner + coefficient) must be distributed correctly instead of shredding the + bracket structure. 0.89 must multiply with (Bi0.5Na0.5)TiO3, 0.11 + with BaTiO3, both further scaled by the outer 0.75, and similarly + for the second half.""" + formula = ( + "0.75*(0.89(Bi0.5Na0.5)TiO3-0.11BaTiO3) + " + "0.25*(0.87(Bi0.5Na0.5)TiO3-0.11BaTiO3-0.02(Sm0.5K0.5)TiO3)" + ) + result = data_cleaner._distribute_multiterm_brackets(formula) + assert result == ( + "0.6675(Bi0.5Na0.5)TiO3-0.0825BaTiO3 + " + "0.2175(Bi0.5Na0.5)TiO3-0.0275BaTiO3-0.005(Sm0.5K0.5)TiO3" + ) + + def test_distribute_multiterm_brackets_two_level_nesting(self, data_cleaner): + """A doubly-nested multi-term bracket must resolve across multiple + passes without explicit recursion.""" + result = data_cleaner._distribute_multiterm_brackets( + "0.5*(0.5*(0.5A-0.5B)-0.5C)" + ) + assert result == "0.125A-0.125B-0.25C" + + def test_distribute_multiterm_brackets_negated_top_level_term( + self, data_cleaner + ): + """A multi-term bracket subtracted at the top level must have its + internal signs flipped correctly (distributing the negation), not + just have the outer coefficient applied blindly.""" + result = data_cleaner._distribute_multiterm_brackets("1-0.5*(0.3A-0.2B)") + assert result == "1-0.15A+0.1B" + + def test_distribute_multiterm_brackets_leaves_single_term_bracket_untouched( + self, data_cleaner + ): + """A bracket with no top-level +/- inside it (single term) is left + for the existing coefficient_expansion pipeline to handle.""" + formula = "0.03*(Bi0.5Ag0.5)ZrO3" + assert data_cleaner._distribute_multiterm_brackets(formula) == formula + + def test_distribute_multiterm_brackets_defers_pure_numeric_arithmetic( + self, data_cleaner + ): + """A bracket containing only numbers/operators (e.g. "0.5*(0.2+0.3)") + must be left for the existing arithmetic-evaluation machinery, not + shredded into dangling additive numeric terms.""" + formula = "0.5*(0.2+0.3)" + assert data_cleaner._distribute_multiterm_brackets(formula) == formula + + def test_formula_prefix_end_stops_at_trailing_annotation(self, data_cleaner): + """A real formula followed by a space-separated descriptive + annotation must have its boundary end right after the formula, + even when the annotation contains a real single-letter element + symbol (e.g. "C" for Celsius) that must never be scaled.""" + text = "PbTiO3 (calcined at 660C)" + assert data_cleaner._formula_prefix_end(text) == len("PbTiO3") + + def test_formula_prefix_end_rejects_word_prefixed_by_real_element( + self, data_cleaner + ): + """An apparent element match that's actually the start of a longer + descriptive word (e.g. "Re" in "Reoxidized") must not be included + in the valid prefix at all.""" + assert data_cleaner._formula_prefix_end("Reoxidized") == 0 + + def test_formula_prefix_end_rejects_fake_element(self, data_cleaner): + """A capitalized 1-2 letter run that isn't a real periodic-table + symbol (e.g. "Bo" in "Bottom") must not be included in the prefix.""" + assert data_cleaner._formula_prefix_end("Bottom") == 0 + + def test_formula_prefix_end_exempts_percent_placeholder(self, data_cleaner): + """A genuine element immediately followed by the internal + percent-annotation placeholder (itself lowercase) must still be + included in the valid prefix, since the placeholder is not real + corrupting text.""" + text = "Li0.5Bi0.5TiO3zzzpctannot0zzz" + assert data_cleaner._formula_prefix_end(text) == len("Li0.5Bi0.5TiO3") + + def test_formula_prefix_end_recurses_into_valid_bracket(self, data_cleaner): + """A legitimate nested formula bracket (e.g. site-occupancy + notation) must be treated as part of the formula, not a break + point, as long as its own content is fully valid.""" + text = "K0.48Na0.52NbO2.7SnO2" + assert data_cleaner._formula_prefix_end(text) == len(text) + text2 = "(Bi0.5Na0.5)TiO3" + assert data_cleaner._formula_prefix_end(text2) == len(text2) + def test_return_in_dict(self, data_cleaner): """Test _return_in_dict method.""" dict_list = [{"key1": "value1"}, {"key2": "value2"}, {"key3": "value3"}] @@ -396,8 +540,8 @@ def temp_mixed_json_file(self, sample_data_with_mixed_validity): yield temp_file_path os.unlink(temp_file_path) - def test_clean_data_without_element_validation(self, temp_mixed_json_file): - """Test clean_data_with_relevant_compositions without the element_validation step.""" + def test_clean_data_without_element_validation_strict(self, temp_mixed_json_file): + """Test clean_data_with_relevant_compositions without the element_validation_strict step.""" cleaner = DataCleaner(temp_mixed_json_file) result = cleaner.clean_data_with_relevant_compositions( cleaning_steps=["abbreviation_filtering"] @@ -433,10 +577,10 @@ def test_abbreviation_filtering_optional_keeps_invalid_keys_when_omitted( for comp_key in all_comp_keys ) - def test_clean_data_with_relevant_compositions_with_element_validation( + def test_clean_data_with_relevant_compositions_with_element_validation_strict( self, temp_mixed_json_file ): - """Test clean_data_with_relevant_compositions with element_validation (cleaning_steps='all').""" + """Test clean_data_with_relevant_compositions with element_validation_strict (cleaning_steps='all').""" cleaner = DataCleaner(temp_mixed_json_file) result = cleaner.clean_data_with_relevant_compositions(cleaning_steps="all") @@ -451,16 +595,16 @@ def test_clean_data_with_relevant_compositions_with_element_validation( # Should only contain valid elements after full cleaning assert cleaner._is_elements(test_dict) is True - def test_clean_data_with_relevant_compositions_without_element_validation( + def test_clean_data_with_relevant_compositions_without_element_validation_strict( self, temp_mixed_json_file ): - """Test clean_data_with_relevant_compositions without element_validation selected.""" + """Test clean_data_with_relevant_compositions without element_validation_strict selected.""" cleaner = DataCleaner(temp_mixed_json_file) result = cleaner.clean_data_with_relevant_compositions( cleaning_steps=[ "abbreviation_filtering", "miller_indices", - "coefficient_expansion", + "coefficient_expansion_strict", ] ) @@ -662,14 +806,14 @@ def test_full_pipeline_integration(self, temp_complex_json_file): assert not re.match(r"(? Ni7O7).""" + data = { + "10.x/percent": { + "composition_data": { + "compositions_property_values": { + "PVDF + 7 wt% NiO + 0.1 wt% ZnO": 1, + "0.845Na0.5Bi0.5TiO3-0.055BaTiO3-0.1Li0.5Bi0.5TiO3:0.1%MgO": 2, + } + }, + "synthesis_data": {}, + "article_metadata": {}, + } + } + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(data, f) + temp_file_path = f.name + + try: + cleaner = DataCleaner(temp_file_path) + result = cleaner.clean_data_with_relevant_compositions(cleaning_steps="all") + comp_values = result["10.x/percent"]["composition_data"][ + "compositions_property_values" + ] + assert comp_values == { + "Na0.4225Bi0.4225Ti0.845O2.535-Ba0.055Ti0.055O0.165-Li0.05Bi0.05Ti0.1O0.3:0.1%MgO": 2, + } finally: os.unlink(temp_file_path) @@ -816,7 +1477,7 @@ def test_division_by_zero_in_fractions(self): cleaning_steps=[ "abbreviation_filtering", "miller_indices", - "coefficient_expansion", + "coefficient_expansion_strict", ] ) # The original fraction should be kept if division by zero From 061df5493598a792f8c490127e4ec32d648b6fff Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Tue, 28 Jul 2026 15:50:06 +0100 Subject: [PATCH 6/8] fix: migrate Springer TDM API to api.springernature.com ahead of old endpoint retirement --- CHANGELOG.md | 2 ++ docs/about/changelog.md | 2 ++ src/comproscanner/utils/configs/base_urls.py | 2 +- tests/test_apis_primary/test_springer.py | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01617270..db89ea6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ - Fixed a nested multi-term coefficient expression (e.g. `"0.75*(0.89(Bi0.5Na0.5)TiO3-0.11BaTiO3) + 0.25*(...)"`) being shredded into mismatched brackets with un-distributed coefficients instead of correctly distributing the outer coefficient across each inner term, including through arbitrary nesting depth and sign flips when a whole bracket is subtracted. Comma-separated site-occupancy notation (e.g. `"(K,Na,Li)(Nb,Ta)O3"`) remains unsupported by design but is no longer actively corrupted either. +- Fixed `SPRINGER_TDM_BASE_URL` pointing at `spdi.public.springernature.app`, which Springer Nature is retiring; requests now go to the new `api.springernature.com/xmldata/jats` endpoint ahead of the old host's retirement on 7th August 2026. + - Replaced the several ad-hoc "is this really an element" checks above with one shared boundary-detection function, `_formula_prefix_end`, that determines how much of a string is genuinely valid formula content. This also fixed a further case the old checks missed — a trailing annotation with its own real element letter right after a number (e.g. `"C"` for Celsius in `"...PbTiO3 (calcined at 660°C)"`) — and ensures any future case of this shape is handled by the same rule rather than needing another bespoke patch. - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 640aa448..069f4ad0 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -26,6 +26,8 @@ - Fixed a nested multi-term coefficient expression (e.g. `"0.75*(0.89(Bi0.5Na0.5)TiO3-0.11BaTiO3) + 0.25*(...)"`) being shredded into mismatched brackets with un-distributed coefficients instead of correctly distributing the outer coefficient across each inner term, including through arbitrary nesting depth and sign flips when a whole bracket is subtracted. Comma-separated site-occupancy notation (e.g. `"(K,Na,Li)(Nb,Ta)O3"`) remains unsupported by design but is no longer actively corrupted either. +- Fixed `SPRINGER_TDM_BASE_URL` pointing at `spdi.public.springernature.app`, which Springer Nature is retiring; requests now go to the new `api.springernature.com/xmldata/jats` endpoint ahead of the old host's retirement on 7th August 2026. + - Replaced the several ad-hoc "is this really an element" checks above with one shared boundary-detection function, `_formula_prefix_end`, that determines how much of a string is genuinely valid formula content. This also fixed a further case the old checks missed — a trailing annotation with its own real element letter right after a number (e.g. `"C"` for Celsius in `"...PbTiO3 (calcined at 660°C)"`) — and ensures any future case of this shape is handled by the same rule rather than needing another bespoke patch. - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). diff --git a/src/comproscanner/utils/configs/base_urls.py b/src/comproscanner/utils/configs/base_urls.py index 68797e55..1afc81ec 100644 --- a/src/comproscanner/utils/configs/base_urls.py +++ b/src/comproscanner/utils/configs/base_urls.py @@ -18,4 +18,4 @@ class BaseUrls: SPRINGER_OPENACCESS_BASE_URL = ( "https://api.springernature.com/openaccess/jats?q=doi:" ) - SPRINGER_TDM_BASE_URL = "https://spdi.public.springernature.app/xmldata/jats?q=doi:" + SPRINGER_TDM_BASE_URL = "https://api.springernature.com/xmldata/jats?q=doi:" diff --git a/tests/test_apis_primary/test_springer.py b/tests/test_apis_primary/test_springer.py index fb0375d9..ef0ccbe0 100644 --- a/tests/test_apis_primary/test_springer.py +++ b/tests/test_apis_primary/test_springer.py @@ -17,7 +17,7 @@ # Specify the URL # url = f"http://api.springernature.com/openaccess/jats?q=doi:10.1007/s42114-024-00879-6&api_key={os.getenv('SPRINGER_OPENACCESS_API_KEY')}" # Used for only open access articles -url = f"https://spdi.public.springernature.app/xmldata/jats?q=doi:10.1007/s42114-024-00879-6&api_key={os.getenv('SPRINGER_TDM_API_KEY')}" +url = f"https://api.springernature.com/xmldata/jats?q=doi:10.1007/s42114-024-00879-6&api_key={os.getenv('SPRINGER_TDM_API_KEY')}" try: print(url) From 85a09d646905192b3dff9b3a3cc05796fd863694 Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Tue, 11 Aug 2026 09:24:11 +0100 Subject: [PATCH 7/8] deepseek default model change details added to changelog --- CHANGELOG.md | 2 ++ docs/about/changelog.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db89ea6c..25e87db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ - Fixed `SPRINGER_TDM_BASE_URL` pointing at `spdi.public.springernature.app`, which Springer Nature is retiring; requests now go to the new `api.springernature.com/xmldata/jats` endpoint ahead of the old host's retirement on 7th August 2026. +- Changed the default DeepSeek model from `deepseek/deepseek-chat` to `deepseek/deepseek-v4-flash` across the RAG chat model docs, the `DEEPSEEK_API_KEY`-based fallback in `EquationTool`, and example scripts, ahead of `deepseek-chat`'s deprecation on 24th July 2026. + - Replaced the several ad-hoc "is this really an element" checks above with one shared boundary-detection function, `_formula_prefix_end`, that determines how much of a string is genuinely valid formula content. This also fixed a further case the old checks missed — a trailing annotation with its own real element letter right after a number (e.g. `"C"` for Celsius in `"...PbTiO3 (calcined at 660°C)"`) — and ensures any future case of this shape is handled by the same rule rather than needing another bespoke patch. - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 069f4ad0..1fb87897 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -28,6 +28,8 @@ - Fixed `SPRINGER_TDM_BASE_URL` pointing at `spdi.public.springernature.app`, which Springer Nature is retiring; requests now go to the new `api.springernature.com/xmldata/jats` endpoint ahead of the old host's retirement on 7th August 2026. +- Changed the default DeepSeek model from `deepseek/deepseek-chat` to `deepseek/deepseek-v4-flash` across the RAG chat model docs, the `DEEPSEEK_API_KEY`-based fallback in `EquationTool`, and example scripts, ahead of `deepseek-chat`'s deprecation on 24th July 2026. + - Replaced the several ad-hoc "is this really an element" checks above with one shared boundary-detection function, `_formula_prefix_end`, that determines how much of a string is genuinely valid formula content. This also fixed a further case the old checks missed — a trailing annotation with its own real element letter right after a number (e.g. `"C"` for Celsius in `"...PbTiO3 (calcined at 660°C)"`) — and ensures any future case of this shape is handled by the same rule rather than needing another bespoke patch. - Handled multi-word property keywords (e.g., _thermal conductivity_) for accurate Scopus search, uniform filename handling (`thermal conductivity` resolves to `thermal_conductivity_metadata.csv` or similar) and restoring the original form `thermal conductivity` in the data extraction RAG search query instead of `thermal_conductivity`. This fix is associated with [#5](https://github.com/slimeslab/ComProScanner/pull/5) and contributed by [@WilmerGaspar](https://github.com/WilmerGaspar). From 913afe9a9bdcbbb7dd7eb627e6ef9490f9df54a8 Mon Sep 17 00:00:00 2001 From: aritraroy24 Date: Tue, 11 Aug 2026 09:28:21 +0100 Subject: [PATCH 8/8] chore: bump version to 2026.08.11 --- CHANGELOG.md | 2 +- docs/about/changelog.md | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e87db3..0b2d6561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Unreleased +# 2026.08.11 ### Added diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 1fb87897..688c3b84 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -1,4 +1,4 @@ -# Unreleased +# 2026.08.11 ### Added diff --git a/pyproject.toml b/pyproject.toml index ab686085..f784b3af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "comproscanner" -version = "2026.05.19" +version = "2026.08.11" description = "Multi-agent system for extracting and processing structured composition-property data from scientific literature" readme = "README.md" authors = [{ name = "Aritra Roy", email = "contact@aritraroy.live" }]