From 2a07b8bc8d9646293c3607b701b0ba8aacd7c786 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 10:16:16 +0200 Subject: [PATCH 01/19] first counter version --- .../alert_aggregation/logid_count.py | 78 +++++++++++++++++++ .../test_logid_count.py | 15 ++++ 2 files changed, 93 insertions(+) create mode 100644 src/detectmatelibrary/alert_aggregation/logid_count.py create mode 100644 tests/test_alert_aggregators/test_logid_count.py diff --git a/src/detectmatelibrary/alert_aggregation/logid_count.py b/src/detectmatelibrary/alert_aggregation/logid_count.py new file mode 100644 index 00000000..09035959 --- /dev/null +++ b/src/detectmatelibrary/alert_aggregation/logid_count.py @@ -0,0 +1,78 @@ +from detectmatelibrary.common.alert_aggregator import ( + CoreAlertAggregatorConfig, CoreAlertAggregator +) + +from detectmatelibrary.schemas import DetectorSchema, AggregateSchema +from detectmatelibrary.utils.data_buffer import BufferMode + +from typing import Any, Optional +from functools import lru_cache +import polars as pl + + +class Counter: + def __init__(self, max_count: int, overall_size: int) -> None: + self.max_count = max_count + + self._df = pl.DataFrame() + self._df = self._df.insert_column(0, pl.repeat("", n=overall_size).alias("LogID")) + self._df = self._df.insert_column(1, pl.zeros(overall_size, dtype=pl.UInt8).alias("Count")) + self._df = self._df.insert_column(2, pl.zeros(overall_size, dtype=pl.Boolean).alias("WasAlert")) + self._df = self._df.with_row_index(name="Index") + + self.idx = 0 + + def __is_new_alert(self, idx: int) -> bool: + if not self._df[idx, "WasAlert"] and self._df[idx, "Count"] >= self.max_count: + self._df[idx, "WasAlert"] = True + return True + return False + + @lru_cache(maxsize=32) + def __get_idx(self, logId: str) -> int: + indices = self._df.filter(pl.col("LogID") == logId).select("Index").to_series() + + if indices.shape[0] == 0: + new_row = pl.DataFrame( + data=[[self.idx, logId, 0, False]], schema=self._df.schema, orient="row" + ) + self._df = self._df.update(new_row, on="Index") + idx = self.idx + self.idx += 1 + else: + idx = indices[0] + + return idx + + def add_and_check(self, logId: str) -> bool: + idx = self.__get_idx(logId) + self._df[idx, "Count"] += 1 + + return self.__is_new_alert(idx) + + +class LogIDCountAggegationConfig(CoreAlertAggregatorConfig): + method_type: str = "logid_count_aggregator" + max_count: int = 3 + + +class LogIDCountAggegation(CoreAlertAggregator): + def __init__( + self, + name: str = "LogIDCountAggregator", + config: Optional[LogIDCountAggegationConfig | dict[str, Any]] = LogIDCountAggegationConfig(), + ) -> None: + if isinstance(config, dict): + config = LogIDCountAggegationConfig.from_dict(config, name) + + super().__init__(name=name, config=config, buffer_mode=BufferMode.NO_BUF) + self.config: LogIDCountAggegationConfig + + self.counter = Counter(max_count=self.config.max_count) + + def aggregate_alerts( + self, input_: DetectorSchema, output_: AggregateSchema # type: ignore + ) -> bool: + + output_["description"] = "Basic aggregation by alert concatenation" + return True diff --git a/tests/test_alert_aggregators/test_logid_count.py b/tests/test_alert_aggregators/test_logid_count.py new file mode 100644 index 00000000..1eb4463c --- /dev/null +++ b/tests/test_alert_aggregators/test_logid_count.py @@ -0,0 +1,15 @@ + +from detectmatelibrary.alert_aggregation.logid_count import Counter + + +class TestCounter: + def test_basic_count(self) -> None: + counter = Counter(max_count=2, overall_size=10) + + assert not counter.add_and_check("a") + assert not counter.add_and_check("b") + assert counter.add_and_check("a") + assert counter.add_and_check("b") + + assert not counter.add_and_check("a") # should not generate new ones + assert not counter.add_and_check("b") # should not generate new ones \ No newline at end of file From a7eb0ef826551c3b029b22a10e0cedf55d9a3116 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 10:32:49 +0200 Subject: [PATCH 02/19] add memory limits --- .../alert_aggregation/logid_count.py | 8 ++++---- .../test_alert_aggregators/test_logid_count.py | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/detectmatelibrary/alert_aggregation/logid_count.py b/src/detectmatelibrary/alert_aggregation/logid_count.py index 09035959..dd76d419 100644 --- a/src/detectmatelibrary/alert_aggregation/logid_count.py +++ b/src/detectmatelibrary/alert_aggregation/logid_count.py @@ -20,6 +20,8 @@ def __init__(self, max_count: int, overall_size: int) -> None: self._df = self._df.insert_column(2, pl.zeros(overall_size, dtype=pl.Boolean).alias("WasAlert")) self._df = self._df.with_row_index(name="Index") + self._cached_search = lru_cache(maxsize=overall_size)(self.__get_idx) + self.idx = 0 def __is_new_alert(self, idx: int) -> bool: @@ -28,7 +30,6 @@ def __is_new_alert(self, idx: int) -> bool: return True return False - @lru_cache(maxsize=32) def __get_idx(self, logId: str) -> int: indices = self._df.filter(pl.col("LogID") == logId).select("Index").to_series() @@ -38,16 +39,15 @@ def __get_idx(self, logId: str) -> int: ) self._df = self._df.update(new_row, on="Index") idx = self.idx - self.idx += 1 + self.idx = self.idx + 1 if self.idx < self._df.shape[0] - 1 else 0 else: idx = indices[0] return idx def add_and_check(self, logId: str) -> bool: - idx = self.__get_idx(logId) + idx = self._cached_search(logId) self._df[idx, "Count"] += 1 - return self.__is_new_alert(idx) diff --git a/tests/test_alert_aggregators/test_logid_count.py b/tests/test_alert_aggregators/test_logid_count.py index 1eb4463c..686f3b57 100644 --- a/tests/test_alert_aggregators/test_logid_count.py +++ b/tests/test_alert_aggregators/test_logid_count.py @@ -12,4 +12,19 @@ def test_basic_count(self) -> None: assert counter.add_and_check("b") assert not counter.add_and_check("a") # should not generate new ones - assert not counter.add_and_check("b") # should not generate new ones \ No newline at end of file + assert not counter.add_and_check("b") # should not generate new ones + + def test_surpass_memory(self) -> None: + counter = Counter(max_count=2, overall_size=2) + + assert not counter.add_and_check("a") + assert not counter.add_and_check("b") + assert not counter.add_and_check("c") + + assert counter.add_and_check("b") + assert not counter.add_and_check("a") # was rename by c + assert counter.add_and_check("c") + assert counter.add_and_check("a") + + assert not counter.add_and_check("a") # should not generate new ones + assert not counter.add_and_check("c") # should not generate new ones \ No newline at end of file From d89240750615e7fa72abd29d92cbe950505843c3 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 12:17:51 +0200 Subject: [PATCH 03/19] functionality ready --- .../alert_aggregation/logid_count.py | 23 ++++-- .../test_logid_count.py | 70 ++++++++++++++++++- 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/detectmatelibrary/alert_aggregation/logid_count.py b/src/detectmatelibrary/alert_aggregation/logid_count.py index dd76d419..5f15d117 100644 --- a/src/detectmatelibrary/alert_aggregation/logid_count.py +++ b/src/detectmatelibrary/alert_aggregation/logid_count.py @@ -53,7 +53,8 @@ def add_and_check(self, logId: str) -> bool: class LogIDCountAggegationConfig(CoreAlertAggregatorConfig): method_type: str = "logid_count_aggregator" - max_count: int = 3 + max_count: int = 4 + max_memory: int = 5000 class LogIDCountAggegation(CoreAlertAggregator): @@ -65,14 +66,26 @@ def __init__( if isinstance(config, dict): config = LogIDCountAggegationConfig.from_dict(config, name) - super().__init__(name=name, config=config, buffer_mode=BufferMode.NO_BUF) + super().__init__(name=name, config=config, buffer_mode=BufferMode.NO_BUF, buffer_size=None) self.config: LogIDCountAggegationConfig - self.counter = Counter(max_count=self.config.max_count) + self.counter = Counter( + max_count=self.config.max_count, overall_size=self.config.max_memory + ) def aggregate_alerts( self, input_: DetectorSchema, output_: AggregateSchema # type: ignore ) -> bool: - output_["description"] = "Basic aggregation by alert concatenation" - return True + alert_logs = [] + for logID in input_["logIDs"]: + if self.counter.add_and_check(logID): + alert_logs.append(logID) + + if (aggreated_alerts := len(alert_logs) > 0): + output_["description"] = ( + "LogID count how many times a log was count as alert." + f"Logs that pass the limit count: {", ".join(alert_logs)}" + ) + + return aggreated_alerts diff --git a/tests/test_alert_aggregators/test_logid_count.py b/tests/test_alert_aggregators/test_logid_count.py index 686f3b57..4bd259ff 100644 --- a/tests/test_alert_aggregators/test_logid_count.py +++ b/tests/test_alert_aggregators/test_logid_count.py @@ -1,5 +1,8 @@ -from detectmatelibrary.alert_aggregation.logid_count import Counter +from detectmatelibrary.alert_aggregation.logid_count import ( + Counter, LogIDCountAggegation, LogIDCountAggegationConfig +) +import detectmatelibrary.schemas as schemas class TestCounter: @@ -27,4 +30,67 @@ def test_surpass_memory(self) -> None: assert counter.add_and_check("a") assert not counter.add_and_check("a") # should not generate new ones - assert not counter.add_and_check("c") # should not generate new ones \ No newline at end of file + assert not counter.add_and_check("c") # should not generate new ones + + +def generate_schema(log_id: str | list[str]) -> schemas.DetectorSchema: + log_id = log_id if isinstance(log_id, list) else [log_id] + return schemas.DetectorSchema({ + "detectorID": "1", + "detectorType": "dummy", + "alertID": "2", + "detectionTimestamp": 0, + "logIDs": log_id, + "score": 0.2, + "extractedTimestamps": [-1], + "description": "hello there", + "receivedTimestamp": 1, + "alertsObtain": {"99 problems": "but logs aint one"} + }) + + +class TestLogIDCount: + def test_basic_count(self): + aggregator = LogIDCountAggegation(config=LogIDCountAggegationConfig(max_count=2)) + + assert aggregator.process(generate_schema("a")) is None + assert aggregator.process(generate_schema("b")) is None + + assert aggregator.process(generate_schema("a")) is not None + assert aggregator.process(generate_schema("b")) is not None + + assert aggregator.process(generate_schema("a")) is None + assert aggregator.process(generate_schema("b")) is None + + def test_basic_count_as_list(self): + aggregator = LogIDCountAggegation(config=LogIDCountAggegationConfig(max_count=2)) + + assert aggregator.process(generate_schema(["a", "b"])) is None + assert aggregator.process(generate_schema(["a", "b"])) is not None + assert aggregator.process(generate_schema(["a", "b"])) is None + + def test_surpass_memory(self): + aggregator = LogIDCountAggegation( + config=LogIDCountAggegationConfig(max_count=2, max_memory=2) + ) + + assert aggregator.process(generate_schema("a")) is None + assert aggregator.process(generate_schema("b")) is None + assert aggregator.process(generate_schema("c")) is None + + assert aggregator.process(generate_schema("b")) is not None + assert aggregator.process(generate_schema("a")) is None # was rename by c + assert aggregator.process(generate_schema("c")) is not None + assert aggregator.process(generate_schema("a")) is not None + + assert aggregator.process(generate_schema("a")) is None # should not generate new ones + assert aggregator.process(generate_schema("c")) is None # should not generate new ones + + def test_description(self): + aggregator = LogIDCountAggegation(config=LogIDCountAggegationConfig(max_count=1)) + alert = aggregator.process(generate_schema(["a", "b"])) + + expected = "LogID count how many times a log was count as alert." + expected += "Logs that pass the limit count: a, b" + + assert alert["description"] == expected \ No newline at end of file From ed928d0ddb3eaa51023694dac649d5ed00828ad6 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 12:20:35 +0200 Subject: [PATCH 04/19] add fields describption --- .../alert_aggregation/logid_count.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/detectmatelibrary/alert_aggregation/logid_count.py b/src/detectmatelibrary/alert_aggregation/logid_count.py index 5f15d117..deea9db2 100644 --- a/src/detectmatelibrary/alert_aggregation/logid_count.py +++ b/src/detectmatelibrary/alert_aggregation/logid_count.py @@ -7,6 +7,7 @@ from typing import Any, Optional from functools import lru_cache +from pydantic import Field import polars as pl @@ -52,9 +53,13 @@ def add_and_check(self, logId: str) -> bool: class LogIDCountAggegationConfig(CoreAlertAggregatorConfig): - method_type: str = "logid_count_aggregator" - max_count: int = 4 - max_memory: int = 5000 + method_type: str = Field( + default="logid_count_aggregator", description="Indicates what type of method it is." + ) + max_count: int = Field(default=4, description="Max count of alerts in a specific logID.") + max_memory: int = Field( + default=5000, description="Max number of logIDs store in the count memory." + ) class LogIDCountAggegation(CoreAlertAggregator): @@ -87,5 +92,5 @@ def aggregate_alerts( "LogID count how many times a log was count as alert." f"Logs that pass the limit count: {", ".join(alert_logs)}" ) - + return aggreated_alerts From dd6be90d3c4f8a489173ec3f6cba352c5f53fbf2 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 12:49:51 +0200 Subject: [PATCH 05/19] updating alert aggregation docs --- docs/alert_aggregator.md | 19 +++++++++++ docs/alert_aggregators/basic_concatenation.md | 24 +++++++++++++ docs/examples/config/update.py | 34 +++++++++++++++++++ .../alert_aggregation/basic_concat.py | 9 +++-- .../common/alert_aggregator.py | 7 ++-- 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/docs/alert_aggregator.md b/docs/alert_aggregator.md index deac6763..d9322853 100644 --- a/docs/alert_aggregator.md +++ b/docs/alert_aggregator.md @@ -19,6 +19,25 @@ This document explains the expected API and how to implement an alert aggregator With the default `buffer_size` of 1, it is called for every alert with a list of one. - Use a typed `Config` class (subclass of `CoreAlertAggregatorConfig`) to hold runtime parameters. + +???+ note "Top level" + + | Field | Type | Default | Description | + |---|---|---|---| + | `auto_config` | boolean | False | Runs the configuration step before the training process. | + +???+ note "params" + + | Field | Type | Default | Description | + |---|---|---|---| + | `start_id` | integer | 10 | Number used to start the unique ID generator. | + | `data_use_training` | integer, null | None | Data used for training, if None, training is not done. | + | `data_use_configure` | integer, null | None | Data used for configuration, if None, configuration is not done. | + | `use_config_data_as_training` | boolean | True | Combine the configured data in the training process if True. | + | `train_buffer_max_records` | integer | 100000 | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | + | `train_buffer_dir` | string, null | None | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | + + ## CoreAlertAggregator: minimal API ```python diff --git a/docs/alert_aggregators/basic_concatenation.md b/docs/alert_aggregators/basic_concatenation.md index 5e37b904..5fc254a9 100644 --- a/docs/alert_aggregators/basic_concatenation.md +++ b/docs/alert_aggregators/basic_concatenation.md @@ -8,6 +8,30 @@ The basic concat alert aggregation approach aggregates x number of alerts into o | **Output** | [AggregateSchema](../schemas.md) | Aggregated alerts | +## Configuration arguments + + +??? note "Top level" + + | Field | Type | Default | Scope | Description | + |---|---|---|---|---| + | `method_type` | string | basic_concat_aggregator | shared | Indicates what type of method it is. | + | `auto_config` | boolean | False | shared | Runs the configuration step before the training process. | + +???+ note "params" + + | Field | Type | Default | Scope | Description | + |---|---|---|---|---| + | `buffer_size` | integer | 3 | specific | Buffer size use in the aggregation. | + | `start_id` | integer | 10 | shared | Number used to start the unique ID generator. | + | `data_use_training` | integer, null | None | shared | Data used for training, if None, training is not done. | + | `data_use_configure` | integer, null | None | shared | Data used for configuration, if None, configuration is not done. | + | `use_config_data_as_training` | boolean | True | shared | Combine the configured data in the training process if True. | + | `train_buffer_max_records` | integer | 100000 | shared | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | + | `train_buffer_dir` | string, null | None | shared | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | + + + # Configuration example ```yaml diff --git a/docs/examples/config/update.py b/docs/examples/config/update.py index 12b8d8aa..c6479a2b 100644 --- a/docs/examples/config/update.py +++ b/docs/examples/config/update.py @@ -25,11 +25,14 @@ from detectmatelibrary.common.core import CoreConfig from detectmatelibrary.common.detector import CoreDetectorConfig +from detectmatelibrary.common.alert_aggregator import CoreAlertAggregatorConfig from detectmatelibrary.common.parser import CoreParserConfig from detectmatelibrary.base_detectors import TrackerDetectorConfig from detectmatelibrary.base_detectors import VariableDetectorConfig from detectmatelibrary.base_detectors import DeepLearningDetectorConfig +from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregationConfig + from typing import Any @@ -184,6 +187,25 @@ def update_shared_args_parsers(doc_path: str) -> None: raise Exception(f"While updating {doc_path} -> {str(e)}") +def update_shared_args_aggregation(doc_path: str) -> None: + """Fill docs/alert_aggregation.md's tables of parameters shared across several + alert_aggregator: fields every parser has (CoreAlertAggregatorConfig).""" + try: + with open(doc_path, "r") as f: + docs = f.readlines() + + docs = append_docs( + docs=docs, + start_cmd="\n", + end_cmd="\n", + add=get_arguments(CoreAlertAggregatorConfig().get_docs(), with_scope=False), + ) + with open(doc_path, "w") as f: + f.writelines(docs) + except Exception as e: + raise Exception(f"While updating {doc_path} -> {str(e)}") + + # %% Documentation update # # Every config below drives its own doc page: whenever a Field is added, @@ -244,11 +266,23 @@ def update_shared_args_parsers(doc_path: str) -> None: ), ] +AGGREGATOR_DOCS: list[tuple[CoreConfig, type[CoreConfig], str]] = [ + ( + BasicConcatAggregationConfig(), + CoreAlertAggregatorConfig, + "docs/alert_aggregators/basic_concatenation.md" + ) +] + update_shared_args_detectors("docs/detectors.md") update_shared_args_parsers("docs/parsers.md") +update_shared_args_aggregation("docs/alert_aggregator.md") for detector_config, shared_base, doc_path in DETECTOR_DOCS: update_docs(detector_config, doc_path=doc_path, shared_base=shared_base) for parser_config, shared_base, doc_path in PARSER_DOCS: update_docs(parser_config, doc_path=doc_path, shared_base=shared_base) + +for aggr_config, shared_base, doc_path in AGGREGATOR_DOCS: + update_docs(aggr_config, doc_path=doc_path, shared_base=shared_base) diff --git a/src/detectmatelibrary/alert_aggregation/basic_concat.py b/src/detectmatelibrary/alert_aggregation/basic_concat.py index 45883cf8..b863af3f 100644 --- a/src/detectmatelibrary/alert_aggregation/basic_concat.py +++ b/src/detectmatelibrary/alert_aggregation/basic_concat.py @@ -5,11 +5,16 @@ from detectmatelibrary.schemas import DetectorSchema, AggregateSchema from typing import Any, Optional +from pydantic import Field class BasicConcatAggregationConfig(CoreAlertAggregatorConfig): - method_type: str = "basic_concat_aggregator" - buffer_size: int = 3 + method_type: str = Field( + default="basic_concat_aggregator", description="Indicates what type of method it is." + ) + buffer_size: int = Field( + default=3, description="Buffer size use in the aggregation." + ) class BasicConcatAggregation(CoreAlertAggregator): diff --git a/src/detectmatelibrary/common/alert_aggregator.py b/src/detectmatelibrary/common/alert_aggregator.py index 44d84fda..3c41e5e9 100644 --- a/src/detectmatelibrary/common/alert_aggregator.py +++ b/src/detectmatelibrary/common/alert_aggregator.py @@ -6,8 +6,9 @@ from detectmatelibrary.schemas import DetectorSchema, AggregateSchema -from typing import Any, Optional from typing_extensions import override +from typing import Any, Optional +from pydantic import Field def _extract_field( @@ -31,8 +32,8 @@ def _extract_list_field(input_: list[DetectorSchema] | DetectorSchema, field: st class CoreAlertAggregatorConfig(CoreConfig): - component_type: str = "alert_aggregators" - method_type: str = "core_alert_aggregator" + component_type: str = Field(default="alert_aggregators", description="<$IGNORE$>") + method_type: str = Field(default="core_alert_aggregator", description="<$IGNORE$>") class CoreAlertAggregator(CoreComponent): From 671deaff956ddf69ad7924b13f7be2892dc9121b Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 12:58:40 +0200 Subject: [PATCH 06/19] update alert aggregation docs --- docs/alert_aggregator.md | 51 ++++++++++++++++----------- docs/alert_aggregators/logid_count.md | 0 2 files changed, 31 insertions(+), 20 deletions(-) create mode 100644 docs/alert_aggregators/logid_count.md diff --git a/docs/alert_aggregator.md b/docs/alert_aggregator.md index d9322853..eaa7a340 100644 --- a/docs/alert_aggregator.md +++ b/docs/alert_aggregator.md @@ -19,24 +19,6 @@ This document explains the expected API and how to implement an alert aggregator With the default `buffer_size` of 1, it is called for every alert with a list of one. - Use a typed `Config` class (subclass of `CoreAlertAggregatorConfig`) to hold runtime parameters. - -???+ note "Top level" - - | Field | Type | Default | Description | - |---|---|---|---| - | `auto_config` | boolean | False | Runs the configuration step before the training process. | - -???+ note "params" - - | Field | Type | Default | Description | - |---|---|---|---| - | `start_id` | integer | 10 | Number used to start the unique ID generator. | - | `data_use_training` | integer, null | None | Data used for training, if None, training is not done. | - | `data_use_configure` | integer, null | None | Data used for configuration, if None, configuration is not done. | - | `use_config_data_as_training` | boolean | True | Combine the configured data in the training process if True. | - | `train_buffer_max_records` | integer | 100000 | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | - | `train_buffer_dir` | string, null | None | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | - ## CoreAlertAggregator: minimal API @@ -81,6 +63,35 @@ fields left for `aggregate_alerts()` to fill are `description` and `alertsObtain unless your `aggregate_alerts()` keeps it, including their `detectionTimestamp` and `receivedTimestamp`. -## Available alert aggregators +## Alert aggregator methods + +The alert aggregator are numbered from simplest to most complex, and the sidebar lists them in the same order. + +| # | Detector | What it detects | +|---|---|---| +| 00 | [Basic Concat Aggregator](alert_aggregators/basic_concatenation.md) | Concat a set of alerts. | +| 01 | [LogID Count Aggregator](alert_aggregators/logid_count.md) | Count the amount of times a logID is in alerts. | + -- [Basic Concat](alert_aggregators/basic_concatenation.md) +### Common parameters (all alerg aggregators) + +There are some parameters, that **every** alert aggregator inherits from `CoreAlertAggregatorConfig`/`CoreConfigCoreAlertAggregator`/`BasicConfig`, regardless of what it does. The other parameters, that are **specific** for the respective detector, are explained right at the detectors documentation page, later on. + + +???+ note "Top level" + + | Field | Type | Default | Description | + |---|---|---|---| + | `auto_config` | boolean | False | Runs the configuration step before the training process. | + +???+ note "params" + + | Field | Type | Default | Description | + |---|---|---|---| + | `start_id` | integer | 10 | Number used to start the unique ID generator. | + | `data_use_training` | integer, null | None | Data used for training, if None, training is not done. | + | `data_use_configure` | integer, null | None | Data used for configuration, if None, configuration is not done. | + | `use_config_data_as_training` | boolean | True | Combine the configured data in the training process if True. | + | `train_buffer_max_records` | integer | 100000 | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | + | `train_buffer_dir` | string, null | None | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | + diff --git a/docs/alert_aggregators/logid_count.md b/docs/alert_aggregators/logid_count.md new file mode 100644 index 00000000..e69de29b From 05af24cdabf44e96e2696f239d2a9d4f2ed0a0eb Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 13:11:22 +0200 Subject: [PATCH 07/19] update basic concat docs --- docs/alert_aggregator.md | 2 +- docs/alert_aggregators/basic_concatenation.md | 51 ++++++++++++------- .../alert_aggregators/basic_concat.py | 16 ++---- .../alert_aggregators/basic_concat.yaml | 5 ++ 4 files changed, 44 insertions(+), 30 deletions(-) create mode 100644 docs/examples/alert_aggregators/basic_concat.yaml diff --git a/docs/alert_aggregator.md b/docs/alert_aggregator.md index eaa7a340..83f73d86 100644 --- a/docs/alert_aggregator.md +++ b/docs/alert_aggregator.md @@ -73,7 +73,7 @@ The alert aggregator are numbered from simplest to most complex, and the sidebar | 01 | [LogID Count Aggregator](alert_aggregators/logid_count.md) | Count the amount of times a logID is in alerts. | -### Common parameters (all alerg aggregators) +### Common parameters (all alert aggregators) There are some parameters, that **every** alert aggregator inherits from `CoreAlertAggregatorConfig`/`CoreConfigCoreAlertAggregator`/`BasicConfig`, regardless of what it does. The other parameters, that are **specific** for the respective detector, are explained right at the detectors documentation page, later on. diff --git a/docs/alert_aggregators/basic_concatenation.md b/docs/alert_aggregators/basic_concatenation.md index 5fc254a9..5ac0b736 100644 --- a/docs/alert_aggregators/basic_concatenation.md +++ b/docs/alert_aggregators/basic_concatenation.md @@ -2,14 +2,48 @@ The basic concat alert aggregation approach aggregates x number of alerts into one output schema. +## In/out + | | Schema | Description | |------------|----------------------------|--------------------| | **Input** | [DetectorSchema](../schemas.md) | Alerts from detectors | | **Output** | [AggregateSchema](../schemas.md) | Aggregated alerts | +## At a glance + +| Learns from training data | Auto-configuration | Needs `events` | Federation | +|---|---|---|---| +| ❌ | ❌ | ❌ | ❌ | + +## Description + +A basic approach that simply concatenates the alerts without verifying anything specific. + + +## Example + +```python +--8<-- "docs/examples/alert_aggregators/basic_concat.py:example_1" +``` + +## Configuration file + +The configuration used by the example above. It sets only what this use case needs; every other parameter keeps its default (see [Configuration arguments](#configuration-arguments)). + +```yaml +--8<-- "docs/examples/alert_aggregators/basic_concat.yaml" +``` + +## Example + +```python +--8<-- "docs/examples/alert_aggregators/basic_concat.py:example_1" +``` ## Configuration arguments +All parameters this detector accepts, grouped by the YAML block they go in. **Scope** tells whether a parameter is `specific` to this detector or `shared` with other detectors (see the [Alert Aggregation overview](../alert_aggregator.md#common-parameters-all-alert-aggregators)). Where this detector changes a shared default, the shared value is shown in brackets. + ??? note "Top level" @@ -30,20 +64,3 @@ The basic concat alert aggregation approach aggregates x number of alerts into o | `train_buffer_max_records` | integer | 100000 | shared | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | | `train_buffer_dir` | string, null | None | shared | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | - - -# Configuration example - -```yaml -alert_aggregators: - BasicConcatAggregator: - method_type: "basic_concat_aggregator" - buffer_size: 3 - auto_config: False -``` - -## Example usage - -```python ---8<-- "docs/examples/alert_aggregators/basic_concat.py:example_1" -``` diff --git a/docs/examples/alert_aggregators/basic_concat.py b/docs/examples/alert_aggregators/basic_concat.py index ea1bc53a..d3d73b54 100644 --- a/docs/examples/alert_aggregators/basic_concat.py +++ b/docs/examples/alert_aggregators/basic_concat.py @@ -4,17 +4,11 @@ import detectmatelibrary.schemas as schemas +import yaml -aggregations_config = { - "alert_aggregators": { - "BasicConcatAggregator": { - "method_type": "basic_concat_aggregator", - "buffer_size": 3, - "auto_config": False, - } - } -} +with open("docs/examples/alert_aggregators/basic_concat.yaml") as f: + config = yaml.safe_load(f) input_ = schemas.DetectorSchema({ "detectorID": "1", @@ -29,9 +23,7 @@ "alertsObtain": {"99 problems": "but logs aint one"} }) -alert_aggregator = BasicConcatAggregation( - "BasicConcatAggregator", aggregations_config -) +alert_aggregator = BasicConcatAggregation("BasicConcatAggregator", config) alert_aggregator.process(input_) # --8<-- [end:example_1] diff --git a/docs/examples/alert_aggregators/basic_concat.yaml b/docs/examples/alert_aggregators/basic_concat.yaml new file mode 100644 index 00000000..8eef83ca --- /dev/null +++ b/docs/examples/alert_aggregators/basic_concat.yaml @@ -0,0 +1,5 @@ +alert_aggregators: + BasicConcatAggregator: + method_type: "basic_concat_aggregator" + buffer_size: 3 + auto_config: False From 8fbae7f0c1a1e6c6fc05cb7119dbabead4434bf9 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 13:33:40 +0200 Subject: [PATCH 08/19] update logid docs --- docs/alert_aggregators/basic_concatenation.md | 32 ++++------ docs/alert_aggregators/logid_count.md | 60 +++++++++++++++++++ .../examples/alert_aggregators/logid_count.py | 35 +++++++++++ .../alert_aggregators/logid_count.yaml | 5 ++ mkdocs.yml | 1 + 5 files changed, 114 insertions(+), 19 deletions(-) create mode 100644 docs/examples/alert_aggregators/logid_count.py create mode 100644 docs/examples/alert_aggregators/logid_count.yaml diff --git a/docs/alert_aggregators/basic_concatenation.md b/docs/alert_aggregators/basic_concatenation.md index 5ac0b736..e3332bbc 100644 --- a/docs/alert_aggregators/basic_concatenation.md +++ b/docs/alert_aggregators/basic_concatenation.md @@ -34,12 +34,6 @@ The configuration used by the example above. It sets only what this use case nee --8<-- "docs/examples/alert_aggregators/basic_concat.yaml" ``` -## Example - -```python ---8<-- "docs/examples/alert_aggregators/basic_concat.py:example_1" -``` - ## Configuration arguments All parameters this detector accepts, grouped by the YAML block they go in. **Scope** tells whether a parameter is `specific` to this detector or `shared` with other detectors (see the [Alert Aggregation overview](../alert_aggregator.md#common-parameters-all-alert-aggregators)). Where this detector changes a shared default, the shared value is shown in brackets. @@ -47,20 +41,20 @@ All parameters this detector accepts, grouped by the YAML block they go in. **Sc ??? note "Top level" - | Field | Type | Default | Scope | Description | - |---|---|---|---|---| - | `method_type` | string | basic_concat_aggregator | shared | Indicates what type of method it is. | - | `auto_config` | boolean | False | shared | Runs the configuration step before the training process. | +| Field | Type | Default | Scope | Description | +|---|---|---|---|---| +| `method_type` | string | basic_concat_aggregator | shared | Indicates what type of method it is. | +| `auto_config` | boolean | False | shared | Runs the configuration step before the training process. | ???+ note "params" - | Field | Type | Default | Scope | Description | - |---|---|---|---|---| - | `buffer_size` | integer | 3 | specific | Buffer size use in the aggregation. | - | `start_id` | integer | 10 | shared | Number used to start the unique ID generator. | - | `data_use_training` | integer, null | None | shared | Data used for training, if None, training is not done. | - | `data_use_configure` | integer, null | None | shared | Data used for configuration, if None, configuration is not done. | - | `use_config_data_as_training` | boolean | True | shared | Combine the configured data in the training process if True. | - | `train_buffer_max_records` | integer | 100000 | shared | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | - | `train_buffer_dir` | string, null | None | shared | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | +| Field | Type | Default | Scope | Description | +|---|---|---|---|---| +| `buffer_size` | integer | 3 | specific | Buffer size use in the aggregation. | +| `start_id` | integer | 10 | shared | Number used to start the unique ID generator. | +| `data_use_training` | integer, null | None | shared | Data used for training, if None, training is not done. | +| `data_use_configure` | integer, null | None | shared | Data used for configuration, if None, configuration is not done. | +| `use_config_data_as_training` | boolean | True | shared | Combine the configured data in the training process if True. | +| `train_buffer_max_records` | integer | 100000 | shared | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | +| `train_buffer_dir` | string, null | None | shared | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | diff --git a/docs/alert_aggregators/logid_count.md b/docs/alert_aggregators/logid_count.md index e69de29b..1a868e22 100644 --- a/docs/alert_aggregators/logid_count.md +++ b/docs/alert_aggregators/logid_count.md @@ -0,0 +1,60 @@ +# LogID Count Alert Aggregation + +The logID count alert aggregation tracks how many times each log ID appears in the alerts, and if it exceeds a specified threshold, it triggers the aggregation. + +## In/out + +| | Schema | Description | +|------------|----------------------------|--------------------| +| **Input** | [DetectorSchema](../schemas.md) | Alerts from detectors | +| **Output** | [AggregateSchema](../schemas.md) | Aggregated alerts | + +## At a glance + +| Learns from training data | Auto-configuration | Needs `events` | Federation | +|---|---|---|---| +| ❌ | ❌ | ❌ | ❌ | + +## Description + +A log ID counter stores, in a fixed-size table, the counts for different log IDs. If the count for any log ID from the Detector Schemas exceeds a defined threshold, it returns an alert aggregation. + +The fixed-size table operates as a FIFO: once it is full, it removes the oldest entry and inserts the new log ID. +## Example + +```python +--8<-- "docs/examples/alert_aggregators/logid_count.py:example_1" +``` + +## Configuration file + +The configuration used by the example above. It sets only what this use case needs; every other parameter keeps its default (see [Configuration arguments](#configuration-arguments)). + +```yaml +--8<-- "docs/examples/alert_aggregators/logid_count.yaml" +``` + +## Configuration arguments + +All parameters this detector accepts, grouped by the YAML block they go in. **Scope** tells whether a parameter is `specific` to this detector or `shared` with other detectors (see the [Alert Aggregation overview](../alert_aggregator.md#common-parameters-all-alert-aggregators)). Where this detector changes a shared default, the shared value is shown in brackets. + + +??? note "Top level" + +| Field | Type | Default | Scope | Description | +|---|---|---|---|---| +| `method_type` | string | basic_concat_aggregator | shared | Indicates what type of method it is. | +| `auto_config` | boolean | False | shared | Runs the configuration step before the training process. | + +???+ note "params" + +| Field | Type | Default | Scope | Description | +|---|---|---|---|---| +| `buffer_size` | integer | 3 | specific | Buffer size use in the aggregation. | +| `start_id` | integer | 10 | shared | Number used to start the unique ID generator. | +| `data_use_training` | integer, null | None | shared | Data used for training, if None, training is not done. | +| `data_use_configure` | integer, null | None | shared | Data used for configuration, if None, configuration is not done. | +| `use_config_data_as_training` | boolean | True | shared | Combine the configured data in the training process if True. | +| `train_buffer_max_records` | integer | 100000 | shared | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | +| `train_buffer_dir` | string, null | None | shared | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | + diff --git a/docs/examples/alert_aggregators/logid_count.py b/docs/examples/alert_aggregators/logid_count.py new file mode 100644 index 00000000..01af127c --- /dev/null +++ b/docs/examples/alert_aggregators/logid_count.py @@ -0,0 +1,35 @@ +# --8<-- [start:example_1] +from detectmatelibrary.alert_aggregation.logid_count import LogIDCountAggegation + +import detectmatelibrary.schemas as schemas + +import yaml + + +with open("docs/examples/alert_aggregators/logid_count.yaml") as f: + config = yaml.safe_load(f) + + +def generate_schema(log_id: str | list[str]) -> schemas.DetectorSchema: + log_id = log_id if isinstance(log_id, list) else [log_id] + return schemas.DetectorSchema({ + "detectorID": "1", + "detectorType": "dummy", + "alertID": "2", + "detectionTimestamp": 0, + "logIDs": log_id, + "score": 0.2, + "extractedTimestamps": [-1], + "description": "hello there", + "receivedTimestamp": 1, + "alertsObtain": {"99 problems": "but logs aint one"} + }) + + +alert_aggregator = LogIDCountAggegation("LogIDCountAggegation", config) + +print(alert_aggregator.process(generate_schema("a")) is None) # True +print(alert_aggregator.process(generate_schema("b")) is None) # True +print(alert_aggregator.process(generate_schema("a")) is None) # False + +# --8<-- [end:example_1] diff --git a/docs/examples/alert_aggregators/logid_count.yaml b/docs/examples/alert_aggregators/logid_count.yaml new file mode 100644 index 00000000..78a96e45 --- /dev/null +++ b/docs/examples/alert_aggregators/logid_count.yaml @@ -0,0 +1,5 @@ +alert_aggregators: + BasicConcatAggregator: + method_type: "logid_count_aggregator" + max_count: 2 + max_memory: 5000 diff --git a/mkdocs.yml b/mkdocs.yml index b5b8fa43..ffe48996 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -66,6 +66,7 @@ nav: - 12_LogBERT Detector: detectors/logbert.md - Alert Aggregation methods: - 00_Basic Concat: alert_aggregators/basic_concatenation.md + - 01_LogID Count: alert_aggregators/logid_count.md - Helpers: - FromTo: helper/from_to.md - Auxiliary: From 241bad683dac5548de98bd03a0b3095676e19d8e Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 13:46:03 +0200 Subject: [PATCH 09/19] solve merge issues --- .../alert_aggregation/basic_concat.py | 17 ++++++++--------- .../test_alert_aggregators/test_basic_concat.py | 11 +++++------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/detectmatelibrary/alert_aggregation/basic_concat.py b/src/detectmatelibrary/alert_aggregation/basic_concat.py index b51c0026..b863af3f 100644 --- a/src/detectmatelibrary/alert_aggregation/basic_concat.py +++ b/src/detectmatelibrary/alert_aggregation/basic_concat.py @@ -1,5 +1,9 @@ -from detectmatelibrary.common.alert_aggregator import (CoreAlertAggregatorConfig, CoreAlertAggregator) +from detectmatelibrary.common.alert_aggregator import ( + CoreAlertAggregatorConfig, CoreAlertAggregator +) + from detectmatelibrary.schemas import DetectorSchema, AggregateSchema + from typing import Any, Optional from pydantic import Field @@ -21,18 +25,13 @@ def __init__( ) -> None: if isinstance(config, dict): config = BasicConcatAggregationConfig.from_dict(config, name) + buffer_size: int = config.buffer_size # type: ignore - self.deduplicate_values: bool = config.deduplicate_values # type: ignore super().__init__(name=name, buffer_size=buffer_size, config=config) def aggregate_alerts( - self, input_: list[DetectorSchema] | DetectorSchema, output_: AggregateSchema + self, input_: list[DetectorSchema], output_: AggregateSchema # type: ignore ) -> bool: + output_["description"] = "Basic aggregation by alert concatenation" - if self.deduplicate_values: - output_["detectorIDs"] = list(dict.fromkeys(output_["detectorIDs"])) - output_["detectorTypes"] = list(dict.fromkeys(output_["detectorTypes"])) - output_["alertIDs"] = list(dict.fromkeys(output_["alertIDs"])) - output_["logIDs"] = list(dict.fromkeys(output_["logIDs"])) - output_["extractedTimestamps"] = list(dict.fromkeys(output_["extractedTimestamps"])) return True diff --git a/tests/test_alert_aggregators/test_basic_concat.py b/tests/test_alert_aggregators/test_basic_concat.py index 221c14ff..1a788856 100644 --- a/tests/test_alert_aggregators/test_basic_concat.py +++ b/tests/test_alert_aggregators/test_basic_concat.py @@ -18,7 +18,6 @@ "Buffer_6": { "method_type": "basic_concat_aggregator", "buffer_size": 6, - "deduplicate_values": True, "auto_config": False, "params": {}, }, @@ -87,10 +86,10 @@ def test_with_buffer_6(self): assert result is not None assert isinstance(result, AggregateSchema) res_dict = result.as_dict() - assert res_dict["detectorIDs"] == ["1", "2", "3"] - assert res_dict["alertIDs"] == ["1", "2", "3"] + assert res_dict["detectorIDs"] == ['1', '2', '3', '1', '1', '1'] + assert res_dict["alertIDs"] == ['1', '2', '3', '1', '1', '1'] assert res_dict["outputTimestamp"] == 0 - assert res_dict["logIDs"] == ["logID1", "logID2", "logID3"] - assert len(res_dict["extractedTimestamps"]) == 3 + assert set(res_dict["logIDs"]) == {"logID1", "logID2", "logID3"} + assert len(res_dict["extractedTimestamps"]) == 6 assert res_dict["description"] == "Basic aggregation by alert concatenation" - assert res_dict["detectorTypes"] == ["dummy"] + assert set(res_dict["detectorTypes"]) == {"dummy"} From cda9547c4511e486d714ebe74ac54ec9b44890a8 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 14:01:37 +0200 Subject: [PATCH 10/19] fix issues after merge --- .../common/alert_aggregator.py | 4 +- .../test_CustomAlertAggregator.py | 11 +- .../test_basic_concat.py | 103 ++++++------------ .../test_common/test_core_alert_aggregator.py | 20 ++-- 4 files changed, 51 insertions(+), 87 deletions(-) diff --git a/src/detectmatelibrary/common/alert_aggregator.py b/src/detectmatelibrary/common/alert_aggregator.py index 3c41e5e9..74213ef5 100644 --- a/src/detectmatelibrary/common/alert_aggregator.py +++ b/src/detectmatelibrary/common/alert_aggregator.py @@ -17,7 +17,7 @@ def _extract_field( if not isinstance(input_, list): input_ = [input_] - return [str(i[field]) for i in input_] + return list(set([str(i[field]) for i in input_])) def _extract_list_field(input_: list[DetectorSchema] | DetectorSchema, field: str) -> list[str]: @@ -28,7 +28,7 @@ def _extract_list_field(input_: list[DetectorSchema] | DetectorSchema, field: st for i in input_: final.extend(i[field]) - return final + return list(set(final)) class CoreAlertAggregatorConfig(CoreConfig): diff --git a/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py b/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py index ff111148..0a1fc343 100644 --- a/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py +++ b/src/detectmatelibrary/tools/workspace/templates/test_templates/test_CustomAlertAggregator.py @@ -39,11 +39,12 @@ def test_run_aggregate_alerts_method(self) -> None: output = alert_aggregator.process(data) if i % 3 == 2: assert getattr(output, "description") == "Custom alert aggregation" - assert getattr(output, "detectorIDs") == ["MyCoolThing"]*3 + assert getattr(output, "detectorIDs") == ["MyCoolThing"] assert getattr(output, "alertsObtain")["type"] == \ "Anomalies aggregated by CustomAlertAggregator" - assert getattr(output, "detectorTypes") == ["MyCoolThing_detector"]*3 - assert getattr(output, "alertIDs") == [ - f"MyCoolThing_{i + 8}", f"MyCoolThing_{i + 9}", f"MyCoolThing_{i + 10}"] - assert getattr(output, "logIDs") == [str(i-2), str(i-1), str(i)] + assert getattr(output, "detectorTypes") == ["MyCoolThing_detector"] + assert set(getattr(output, "alertIDs")) == set([ + f"MyCoolThing_{i + 8}", f"MyCoolThing_{i + 9}", f"MyCoolThing_{i + 10}" + ]) + assert set(getattr(output, "logIDs")) == set([str(i-2), str(i-1), str(i)]) i += 1 diff --git a/tests/test_alert_aggregators/test_basic_concat.py b/tests/test_alert_aggregators/test_basic_concat.py index 1a788856..d07e1b49 100644 --- a/tests/test_alert_aggregators/test_basic_concat.py +++ b/tests/test_alert_aggregators/test_basic_concat.py @@ -1,95 +1,58 @@ from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregation -from detectmatelibrary.schemas import AggregateSchema + + from detectmatelibrary.utils.aux import time_test_mode + import detectmatelibrary.schemas as schemas -from datetime import datetime time_test_mode() aggregations_config = { "alert_aggregators": { - "Buffer_4": { + "Buffer_3": { "method_type": "basic_concat_aggregator", - "buffer_size": 4, - "auto_config": False, - "params": {}, + "buffer_size": 3, }, - "Buffer_6": { + "Buffer_5": { "method_type": "basic_concat_aggregator", - "buffer_size": 6, - "auto_config": False, - "params": {}, + "buffer_size": 5, }, - }, - "auto_config": False + } } -data_global = [schemas.DetectorSchema({ - "detectorID": str(i), - "detectorType": "dummy", - "alertID": str(i), - "detectionTimestamp": 0, - "logIDs": [f"logID{i}"], - "score": 0.2 + i / 10, - "extractedTimestamps": [int(datetime.now().timestamp() + i)], - "description": "hello there", - "receivedTimestamp": 1, - "alertsObtain": {"99 problems": "but logs aint one"} -}) for i in range(1, 4)] - -data_global += [schemas.DetectorSchema({ +dummy_schema = { "detectorID": "1", "detectorType": "dummy", - "alertID": "1", + "alertID": "2", "detectionTimestamp": 0, - "logIDs": [f"logID{1}"], - "score": 0.3, - "extractedTimestamps": [int(datetime.now().timestamp() + 1)], + "logIDs": ["logID1"], + "score": 0.2, + "extractedTimestamps": [-1], "description": "hello there", "receivedTimestamp": 1, "alertsObtain": {"99 problems": "but logs aint one"} -}) for i in range(1, 4)] +} class TestBasicAggregation: - def test_with_buffer_4(self): - alert_aggregator = BasicConcatAggregation("Buffer_4", aggregations_config) - data = data_global - - assert alert_aggregator.process(data[0]) is None - assert alert_aggregator.process(data[1]) is None - assert alert_aggregator.process(data[2]) is None - result = alert_aggregator.process(data[3]) - assert result is not None - assert isinstance(result, AggregateSchema) - res_dict = result.as_dict() - assert res_dict["detectorIDs"] == ["1", "2", "3", "1"] - assert res_dict["alertIDs"] == ["1", "2", "3", "1"] - assert res_dict["outputTimestamp"] == 0 - assert res_dict["logIDs"] == ["logID1", "logID2", "logID3", "logID1"] - assert len(res_dict["extractedTimestamps"]) == 4 - assert res_dict["description"] == "Basic aggregation by alert concatenation" - assert res_dict["detectorTypes"] == ["dummy"]*4 - - def test_with_buffer_6(self): - alert_aggregator = BasicConcatAggregation("Buffer_6", aggregations_config) - data = data_global - - assert alert_aggregator.process(data[0]) is None - assert alert_aggregator.process(data[1]) is None - assert alert_aggregator.process(data[2]) is None - assert alert_aggregator.process(data[3]) is None - assert alert_aggregator.process(data[4]) is None - result = alert_aggregator.process(data[5]) - assert result is not None - assert isinstance(result, AggregateSchema) - res_dict = result.as_dict() - assert res_dict["detectorIDs"] == ['1', '2', '3', '1', '1', '1'] - assert res_dict["alertIDs"] == ['1', '2', '3', '1', '1', '1'] - assert res_dict["outputTimestamp"] == 0 - assert set(res_dict["logIDs"]) == {"logID1", "logID2", "logID3"} - assert len(res_dict["extractedTimestamps"]) == 6 - assert res_dict["description"] == "Basic aggregation by alert concatenation" - assert set(res_dict["detectorTypes"]) == {"dummy"} + def test_with_buffer_3(self): + alert_aggregator = BasicConcatAggregation("Buffer_3", aggregations_config) + data = schemas.DetectorSchema(dummy_schema).serialize() + + assert alert_aggregator.process(data) is None + assert alert_aggregator.process(data) is None + assert (result := alert_aggregator.process(data)) is not None + assert isinstance(result, bytes) + + def test_with_buffer_5(self): + alert_aggregator = BasicConcatAggregation("Buffer_5", aggregations_config) + data = schemas.DetectorSchema(dummy_schema).serialize() + + assert alert_aggregator.process(data) is None + assert alert_aggregator.process(data) is None + assert alert_aggregator.process(data) is None + assert alert_aggregator.process(data) is None + assert (result := alert_aggregator.process(data)) is not None + assert isinstance(result, bytes) diff --git a/tests/test_common/test_core_alert_aggregator.py b/tests/test_common/test_core_alert_aggregator.py index 1aa180dd..2240ef1b 100644 --- a/tests/test_common/test_core_alert_aggregator.py +++ b/tests/test_common/test_core_alert_aggregator.py @@ -125,13 +125,13 @@ def test_process_input_schema_not_serialized_window_3(self) -> None: alert_aggregator = MockupAlertAggregator(name="TestDetector", config=MockupConfig(), buffer_size=3) expected_result = schemas.AggregateSchema({ "__version__": "1.0.0", - "detectorIDs": ["1", "1", "1"], - "detectorTypes": ["dummy", "dummy", "dummy"], - "alertIDs": ["2", "2", "2"], + "detectorIDs": ["1"], + "detectorTypes": ["dummy"], + "alertIDs": ["2"], "outputTimestamp": 0, - "logIDs": ["logID1", "logID1", "logID1"], + "logIDs": ["logID1"], "description": "Mockup aggregator", - "extractedTimestamps": [-1, -1, -1], + "extractedTimestamps": [-1], "alertsObtain": {} }) data = schemas.DetectorSchema(dummy_schema) @@ -148,13 +148,13 @@ def test_process_input_schema_not_serialized_buffer_3(self) -> None: ) expected_result = schemas.AggregateSchema({ "__version__": "1.0.0", - "detectorIDs": ["1", "1", "1"], - "detectorTypes": ["dummy", "dummy", "dummy"], - "alertIDs": ["2", "2", "2"], + "detectorIDs": ["1"], + "detectorTypes": ["dummy"], + "alertIDs": ["2"], "outputTimestamp": 0, - "logIDs": ["logID1", "logID1", "logID1"], + "logIDs": ["logID1"], "description": "Mockup aggregator", - "extractedTimestamps": [-1, -1, -1], + "extractedTimestamps": [-1], "alertsObtain": {} }) data = schemas.DetectorSchema(dummy_schema) From 2876d675a9e69fff1c6a618b2f6f50c6f8c8d299 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 13 Oct 2026 14:02:55 +0200 Subject: [PATCH 11/19] update yaml --- docs/examples/alert_aggregators/logid_count.py | 2 +- docs/examples/alert_aggregators/logid_count.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples/alert_aggregators/logid_count.py b/docs/examples/alert_aggregators/logid_count.py index 01af127c..f3e2f230 100644 --- a/docs/examples/alert_aggregators/logid_count.py +++ b/docs/examples/alert_aggregators/logid_count.py @@ -26,7 +26,7 @@ def generate_schema(log_id: str | list[str]) -> schemas.DetectorSchema: }) -alert_aggregator = LogIDCountAggegation("LogIDCountAggegation", config) +alert_aggregator = LogIDCountAggegation("LogIDCountAggregator", config=config) print(alert_aggregator.process(generate_schema("a")) is None) # True print(alert_aggregator.process(generate_schema("b")) is None) # True diff --git a/docs/examples/alert_aggregators/logid_count.yaml b/docs/examples/alert_aggregators/logid_count.yaml index 78a96e45..bc646f55 100644 --- a/docs/examples/alert_aggregators/logid_count.yaml +++ b/docs/examples/alert_aggregators/logid_count.yaml @@ -1,5 +1,5 @@ alert_aggregators: - BasicConcatAggregator: + LogIDCountAggregator: method_type: "logid_count_aggregator" max_count: 2 max_memory: 5000 From 9830ab4f9936aba2da3ccee08c0d596e017dde07 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:08:20 +0200 Subject: [PATCH 12/19] Automated documentation update --- docs/alert_aggregators/basic_concatenation.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/alert_aggregators/basic_concatenation.md b/docs/alert_aggregators/basic_concatenation.md index e3332bbc..6334c7ef 100644 --- a/docs/alert_aggregators/basic_concatenation.md +++ b/docs/alert_aggregators/basic_concatenation.md @@ -41,20 +41,20 @@ All parameters this detector accepts, grouped by the YAML block they go in. **Sc ??? note "Top level" -| Field | Type | Default | Scope | Description | -|---|---|---|---|---| -| `method_type` | string | basic_concat_aggregator | shared | Indicates what type of method it is. | -| `auto_config` | boolean | False | shared | Runs the configuration step before the training process. | + | Field | Type | Default | Scope | Description | + |---|---|---|---|---| + | `method_type` | string | basic_concat_aggregator | shared | Indicates what type of method it is. | + | `auto_config` | boolean | False | shared | Runs the configuration step before the training process. | ???+ note "params" -| Field | Type | Default | Scope | Description | -|---|---|---|---|---| -| `buffer_size` | integer | 3 | specific | Buffer size use in the aggregation. | -| `start_id` | integer | 10 | shared | Number used to start the unique ID generator. | -| `data_use_training` | integer, null | None | shared | Data used for training, if None, training is not done. | -| `data_use_configure` | integer, null | None | shared | Data used for configuration, if None, configuration is not done. | -| `use_config_data_as_training` | boolean | True | shared | Combine the configured data in the training process if True. | -| `train_buffer_max_records` | integer | 100000 | shared | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | -| `train_buffer_dir` | string, null | None | shared | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | + | Field | Type | Default | Scope | Description | + |---|---|---|---|---| + | `buffer_size` | integer | 3 | specific | Buffer size use in the aggregation. | + | `start_id` | integer | 10 | shared | Number used to start the unique ID generator. | + | `data_use_training` | integer, null | None | shared | Data used for training, if None, training is not done. | + | `data_use_configure` | integer, null | None | shared | Data used for configuration, if None, configuration is not done. | + | `use_config_data_as_training` | boolean | True | shared | Combine the configured data in the training process if True. | + | `train_buffer_max_records` | integer | 100000 | shared | Configure records kept in memory for training (use_config_data_as_training) before the buffer spills to Parquet files on disk, in parts of this many records. | + | `train_buffer_dir` | string, null | None | shared | Local directory for the spilled training buffer. None uses the system temp directory (TMPDIR). Each spill goes to a private detectmate-train-* directory, removed after training reads it; a killed process leaves it behind. | From 2730373fa0f414a0f542154e60a8629ab26e3841 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Sat, 17 Oct 2026 18:54:48 +0200 Subject: [PATCH 13/19] solve bug missmatch timestamp with logIDs --- .../common/alert_aggregator.py | 23 +++++++++---- .../test_common/test_core_alert_aggregator.py | 32 ++++++++++++++++++- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/detectmatelibrary/common/alert_aggregator.py b/src/detectmatelibrary/common/alert_aggregator.py index 74213ef5..f2917210 100644 --- a/src/detectmatelibrary/common/alert_aggregator.py +++ b/src/detectmatelibrary/common/alert_aggregator.py @@ -20,15 +20,23 @@ def _extract_field( return list(set([str(i[field]) for i in input_])) -def _extract_list_field(input_: list[DetectorSchema] | DetectorSchema, field: str) -> list[str]: +def _extract_list_fields( + input_: list[DetectorSchema] | DetectorSchema, field1: str, field2: str +) -> tuple[list[str], list[str]]: + if not isinstance(input_, list): input_ = [input_] - final = [] - for i in input_: - final.extend(i[field]) + final1, final2 = [], [] + already_checked = set() + for in_ in input_: + for i_1, i_2 in zip(in_[field1], in_[field2]): + if i_1 not in already_checked: + final1.append(i_1) + final2.append(i_2) + already_checked.add(i_1) - return list(set(final)) + return final1, final2 class CoreAlertAggregatorConfig(CoreConfig): @@ -64,8 +72,9 @@ def run( output_["detectorIDs"] = _extract_field(input_, "detectorID") output_["detectorTypes"] = _extract_field(input_, "detectorType") output_["alertIDs"] = _extract_field(input_, "alertID") - output_["logIDs"] = _extract_list_field(input_, "logIDs") - output_["extractedTimestamps"] = _extract_list_field(input_, "extractedTimestamps") + output_["logIDs"], output_["extractedTimestamps"] = _extract_list_fields( + input_, "logIDs", "extractedTimestamps" + ) if (alert := self.aggregate_alerts(input_=input_, output_=output_)): output_["outputTimestamp"] = get_timestamp() diff --git a/tests/test_common/test_core_alert_aggregator.py b/tests/test_common/test_core_alert_aggregator.py index 2240ef1b..b7e96a8c 100644 --- a/tests/test_common/test_core_alert_aggregator.py +++ b/tests/test_common/test_core_alert_aggregator.py @@ -1,5 +1,5 @@ from detectmatelibrary.common.alert_aggregator import ( - CoreAlertAggregatorConfig, CoreAlertAggregator, BufferMode + CoreAlertAggregatorConfig, CoreAlertAggregator, BufferMode, _extract_field, _extract_list_fields ) from detectmatelibrary.utils.aux import time_test_mode @@ -74,6 +74,36 @@ def aggregate_alerts(self, input_, output_): time_test_mode() +class TestAuxiliar: + def test_get_field(self) -> None: + expected = {"dummy2", "dummy"} + + dummy_schema2 = dummy_schema.copy() + dummy_schema2["detectorType"] = "dummy2" + input_ = [dummy_schema, dummy_schema2] + input_ = [schemas.DetectorSchema(in_) for in_ in input_] + + assert len(field := _extract_field(input_, "detectorType")) == 2 + assert expected == set(field) + + def test_get_logID_timestamp(self) -> None: + expected_logID = ["logID1", "logID2", "logID3"] + expected_times = [-1, 0, 1] + + dummy_schema2 = dummy_schema.copy() + dummy_schema2["logIDs"] = ["logID2", "logID3", "logID1"] + dummy_schema2["extractedTimestamps"] = [0, 1, 3] + input_ = [dummy_schema, dummy_schema2] + input_ = [schemas.DetectorSchema(in_) for in_ in input_] + + logids, times = _extract_list_fields( + input_, field1="logIDs", field2="extractedTimestamps" + ) + + assert expected_logID == logids + assert times == expected_times + + class TestCoreAlertAggregator: def test_initialize_default(self) -> None: detector = MockupAlertAggregator(name="TestAAG", config=dummy_config) From d08ac336e4d541e1ebb5a0f50c52227b1d436477 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Sat, 17 Oct 2026 18:59:26 +0200 Subject: [PATCH 14/19] improve testing --- .../test_common/test_core_alert_aggregator.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/test_common/test_core_alert_aggregator.py b/tests/test_common/test_core_alert_aggregator.py index b7e96a8c..bf033693 100644 --- a/tests/test_common/test_core_alert_aggregator.py +++ b/tests/test_common/test_core_alert_aggregator.py @@ -75,7 +75,14 @@ def aggregate_alerts(self, input_, output_): class TestAuxiliar: - def test_get_field(self) -> None: + def test_get_field_single(self) -> None: + expected = {"dummy"} + input_ = schemas.DetectorSchema(dummy_schema) + + assert len(field := _extract_field(input_, "detectorType")) == 1 + assert expected == set(field) + + def test_get_field_multiple(self) -> None: expected = {"dummy2", "dummy"} dummy_schema2 = dummy_schema.copy() @@ -86,7 +93,23 @@ def test_get_field(self) -> None: assert len(field := _extract_field(input_, "detectorType")) == 2 assert expected == set(field) - def test_get_logID_timestamp(self) -> None: + def test_get_logID_timestamp_single(self) -> None: + expected_logID = ["logID1", "logID2", "logID3"] + expected_times = [-1, 0, 1] + + dummy_schema2 = dummy_schema.copy() + dummy_schema2["logIDs"] = ["logID1", "logID2", "logID3", "logID1"] + dummy_schema2["extractedTimestamps"] = [-1, 0, 1, 3] + input_ = schemas.DetectorSchema(dummy_schema2) + + logids, times = _extract_list_fields( + input_, field1="logIDs", field2="extractedTimestamps" + ) + + assert expected_logID == logids + assert times == expected_times + + def test_get_logID_timestamp_multiple(self) -> None: expected_logID = ["logID1", "logID2", "logID3"] expected_times = [-1, 0, 1] From dbc6ad9ced4e02fb53b3bd2213c67a3e782d4bc9 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Sat, 17 Oct 2026 19:21:51 +0200 Subject: [PATCH 15/19] update README and add logo --- README.md | 99 ++++------------------------------------------ docs/img/logo.png | Bin 0 -> 20302 bytes docs/index.md | 2 + 3 files changed, 10 insertions(+), 91 deletions(-) create mode 100644 docs/img/logo.png diff --git a/README.md b/README.md index 72a284da..22f98718 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,22 @@ -# DetectMateLibrary - -Main library to run the different components in DetectMate. - -## Main structure - -The library contains the next components: - -* **Parsers**: parse the logs received from the reader. -* **Detectors**: return alerts if anomalies are detected. -* **Alert Aggregation**: aggregate the alerts produced by the detectors. -* **Schemas**: standard data classes used in DetectMate. -``` - +--------+ +-----------+ +-------------------+ - | Parser | --> | Detector | -> | Alert Aggregation | - +--------+ +-----------+ +-------------------+ -``` -## Developer setup: - -### Step 1: Install python dependencies -Set up the dev environment and install pre-commit hooks: - -```bash -uv sync --dev -uv run prek install -``` - -### Step 2: Install Protobuf dependencies - -To install in Linux do: - -```bash -sudo apt install -y protobuf-compiler -protoc --version -``` - -This dependency is only needed if a proto file is modified. To compile the proto file do: -```bash -protoc --proto_path=src/detectmatelibrary/schemas/ --python_out=src/detectmatelibrary/schemas/ src/detectmatelibrary/schemas/schemas.proto -``` -### Step 3: Run unit tests -Run the tests: - -```bash -uv run --dev pytest -q -``` - -Run the tests with coverage (add --cov-report=html to generate an HTML report): - -```bash -uv run --dev pytest --cov=. --cov-report=term-missing -``` - -## Workspace generator (`mate create`) - -DetectMateLibrary includes a small CLI helper to bootstrap standalone workspaces -for custom parsers and detectors. This is useful if you want to develop and test -components in isolation while still using the same library and schemas. - -### Usage - -The CLI entry point is `mate` with a `create` command: - -```bash -mate create --type --name --dir -``` +# DetectMateLibrary -| Option | Description | -|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--type` | Component type to generate:
- `parser`: CoreParser-based template
- `detector`: CoreDetector-based template | -| `--name` | Name of the component and package:
- Creates package dir: `//`
- Creates main file: `.py`
- Derives class names: `` and `Config` | -| `--dir` | Directory where the workspace will be created | +![UML](docs/img/logo.png) +DetectMate is a flexible, component-based library for parsing and log anomaly detection. -### What gets generated +## Installation -For example: +Download us from `Pip`!! ```bash -mate create --type parser --name custom_parser --dir ./workspaces/custom_parser -``` - -will create: - -```text -workspaces/custom_parser/ # workspace root -├── custom_parser/ # Python package -│ ├── __init__.py -│ └── custom_parser.py # CoreParser-based template -├── tests/ -│ └── test_custom_parser.py # generated from template to test custom_parser -├── data.json # example data to run the code -├── LICENSE.md # copied from main project -├── .gitignore # copied from main project -├── .pre-commit-config.yaml # copied from main project -├── pyproject.toml # minimal project + dev extras -└── README.md # setup instructions +pip install detectmatelibrary ``` ## Documentation +Main documentations: + - [Project Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) ## Contribution diff --git a/docs/img/logo.png b/docs/img/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..a1e6491d93737517758f7381ef9f98ae94ec646c GIT binary patch literal 20302 zcmeFZbx_>Rwl0iAaF<}i;O_1Y!CeCkFj$bm2@b(sf@>g1a00>I2_BrFA-D(Ed_&%M z@3VKEQ}xxUy0_~7u~W$~^lz=zYd!t+>h3i?iBwmW$3T6F3IhX!p{O9E2?GP00DXUf zf&_j2QVHRK{^<4A)_2ncc>*9VPFA+|V1V0O2p9nNvbBPN@mg|Fcfjlgpgf*1dLZ1E z_BbCd(Y1ERiQ_BpH>AO+P0}ZfUmv`R$3#$V(_u7x#aHv$oEx|}6&60)i?wT6vpwB^ zhQcaKI2^Aj(la{vj^qQ$cSosTF+ClGJYo{JxKda%!ut;>R2WoJ>=GzEl*~ja-jpEm zb5v)gmD&o2-`V93Ckuy^QX(%HvgnysVceM`4cXC6kAMj;2NS6>3zwvxj%CA zFm&f=zCg<)PQoXW@BXsBK4?DM-EW|WtS5>60CR?gO)pEtc-H=L5;!0ncYn5gFS6W@ zj579ex+0e*f3G1!rKHanB#Yh-jFHTw>H}Mm)Fr(SI=N|)4?OXFk-KGjy|b$qO$ZMM zbWS2ER%ufeX}2ZKzK$%c0G9Xd^lj3;PPj=th#GXt9$`Zh;kMiU?AoVcUc21?il{E? z6OJ0rpteehFV#uy*SFDiy5)6OqPJ+&vVLzP59;L-2jB>)BtwFQ%1XQCf<1EZ(F}yt zZg1p`J7w5HjdeI&MlIl9TFzrVA$R(`Nui@}`>24r>*t&u&k3kUtA81H?sq6qi3hkr zH{K)vEr*wWOVd6zWiH><#RqX-nR+j}?t)D0&t<9QQ3PQ9pbfV=v73e3HOY!?a(Oo= zHn<0$P-p^n416aIFeyId&GWo=zdl;a^!V)$6U66I@r@Ya0LnWrTPVNuRg{G+oE+Fd zmQLniHZKPVlz1>OBH~^Uki{FY8^9cFZR;osJZbF&0&FcsfqJ|u94ZiLu#K&Pw+mRy zTUFb_`;CR5B~V-pRm4jOYQO>P1_F3F*gLukd5Hr5*cF1le{N<60{)1&y%7cKtEdB{ zom{{G9yT5}4pvz&TMsUv7%D)-#nMVhQ%3G@6VTs8fi`Y#5FvJUPft%aPi{6R7i)G- zK|w)w4lZ^sE>@@ntLs}wH;@;rqbv2ZiN9>ffL$$IY$0y8PL6&1cQ;WW5c(YO z5C0q>Dk}dp@96qB3Q#=Qy+9ClPBsp92M6|l)o^u_^?(}r+oAta!&MtPP1!ZUu1@YQ z7GPNqu%jFGznZYL_@_R^-NpV-IF=UdV0*9wRMZvPE9bv;DX*xa{!fi(2&`=#Ab+%= zVE>z>o2}Jx$!cy1=4BP+<`m$u666vP;Nklh6KXEDP*#HM z|JAE!Q#s)^HXui9Ff@DoP1k?4+x`z!ump2*fH}>k48!<>)Pg42@c4+{UZ?&@Ua<_U5EOIky5gklAy=O3&941WfS@!zFAZNSf? z00oSdlY{lY1IGS$!0gXC;~x==u>UtqME(f;OOk=={nZ9dUeH{~{?BCiH)c?O|8M^J zdo2EMb^!qVr<4B`zW;&iKXCo85cppS|A$@wf$M*T!2e44KkWKH1{dmo9jCyK&|Q!x z^swYo*tG;bXd#&^$;-gp!Q8^!{HpK%1#Ni&Q7~|YfkDH2{s#+_nN19BM0QhDkwxA| z#lgX+R^XlufPn$PD9T7`do3Ncd%79vo_k4QUImQK%Yu6KdIEOaUtwz{) z-6xIEXA%$+Wp_(a36WLu8a7t04S9{8mRgK`@7Gq)-X*rMNG1eglYnREgxb!=35;FC zc@3V>9wqHpreZHW=d>(vWg4>4)d^TIU0lYzHnzS@?gtfh1vZ+r22R?fzf1-wNJJ>t zdAO1vn`q@P{xUtp`mxZh1hcY)gYW_+(IAFS)Wp}BS{ZKoQ260TJD=BG`%F55B%G9G z=03Q{%S|uEis&q;CI+68=N)rB1{De}Hn5OGqpQ%=^c`-NN!`h5Rv~8n7lBgF7r_W2 zLp=RDpwc*Ct7%6=OoARjN+j;B!yy^%puBX$^-|)6}WeRZPXCQf^#m z3|CF($4JYyGsTO)@>;}#lLzZbs{A1kY-vaSyZif3)PH1#YBNrI&<;Q|rLU{u%qChZ ziAlbt8aUD5N>+3GKg_Mohbz!Xjoep(WSZ%9SsZ( zp}5OI2Vs(ez`fsV?_g={jwn??%CG$>|l*y`AS#wlO1^ zfUHTEYrHRCv1z~WgnT$MSJO?K+C1wL7s@>K^zWpUmX@9&d4Y>9<)e?ra`TurCoxqM zey3*gMrtT8XU~aNF}SeA`qdiNxS)5AYM(j2Lpn4x9)5Uu(nj{)xRl#WztA^vFsgYr zBVR8XA0Ho4LZzm~@K8h7fpsR&-U5;8VAaGMF+YcK|8DQNMh!#uHP9)B2*f;fbkx7V zcxJ!B@Mj%@n}PO}%-G~v{+GSTCniIguZM1?-W$fdDke>Gk7*G7k&7UxOqaR7}>u6VX9qJIqF%T<5A`%QUU5}K;4Vlp@)GC zO?*UAQBiKL3EID$V`-%@3?+D}8sJlEi$b(5_*gl(nNj<0eRr1u2^p#MuMw{y&_Dea z{f3ImVPN$$mm&ZFbwFqRT}wltOzh!C0OoD$UL%R`oeKx7I7 z_$-4N=rrHPLvE?M0{?WnZ8Py|-&;9D$HMB}xzx0#QIJ460cNg1Ys1;{W@d9v1A#-}zRA070hpJ(W{8M^06Biuo%KJPs$>k+lU@2%y`%NAth zol#hh_3K_y0R_K5Ob=hQ>Ty-84%W5E9wNKBTqTV0+NOJS;CUJ9%A#^)yx}I~WW_}T zO)gSe6>G-v;rxrxJb|f49FAqIdVdY=b8St0C)tPRrL3tbrMt`@mKR9&;jJx1kiGM0 zokYH`w&>bWJn*#QLiY(UrOdc|o{M~+3=M+^Ps-=)Z&#>DC_7dV7*li6Y@tl`_U7{F z9uv&t<0SY)oUuZ+D(P62tUV`^z_sjM+3&RSh8+_-=iS00*;AgU-tDLAcSe8MdfVUZ zwO2Izk^&apXU#^=E!C?1GfQDftMFzk?6_dID*fMGFV;Jd%+1)%S6nL(qKuPw@-dfx z4hjl-+ZvC8^o|(P{5N+v3}SA!l@6%vZpi&_?qp-o$3`poaU(~4@_ScTR}GNupFuG< z(eG$@8GpzyZ-472NfIvJf-<^}@^ud~+NvhULMRPQ7_Uq5paCNZCzu}NBis+bw^IML?Gj-FyNW)scUR0td zRk-Lx!+=P0#zBalY%?u>H|XOcD& zs?P8}WoyDxfB=dy;9Jz{$H6*wrw}=QDl_zjdk<#cd_$2RhwXz{xNR?*&y!dQIF(q z!Flbs&`cPdUK6wCVM_h->)10aY*?E=h{$1J;s@dLCQ3g2ijRp=um6+U=o);{O#J=7 zSZ!J=q7I6mG^4ELV_}9%C!b8NtmIsJ@`bt9BdOZ z@}~gkM3cGt;Tmr~kvozZ8xEf0JY3L4k+eiBdo8d79fk8;$>$Uh!ASpUt-})*vy;yc zOCYe7&rn0%-Z0?^FIH}9eqP@B^YhN)9^!_+zM|&MWAr3d9x;3H3$_9By|l9$tJ^F= zi22zUEEna0dnZ0B+-b*$qWgyJo&x8(*SxJV@@iN|QqLxQnohUe$sApvn+EqHZ=R}A zBYX#0tM=y&0B?v~@51>wt<9{0Hg@2VCxHY9OD*27Nn2aePFpVjou|{E7%+N%;s^Z0c4l(VJI@$;pdI zpkr|RN&OS#^qA{Nes(uz~r5izO=1u*X7Kv`z!LVM-yK^BQO$@-e)fb zF-a#8s(LhSYt1`%=(HB+V-R@HuzZ^iY~+dk4L-BL_+b3NM^4c6(_6(Z(!T1oCKTDiR;)rOIV-Wf;SK8IH{AsOP}AiA z(~>Uv3jihA19Q`pTFb!0xvH;!yP=R%Hts}dARc)Ej`Q-?D*$Z?xhIqLcNl>nL-p_n zgLT554GyHC3g}4Q+G(m?b_8+qDM}1@;zW)|EFPJ2dtC7P8`F+_3n{D(t+^+Lc%803 zk!F7JZXiaqUpB4*ECG*Nu}knaAn8oO`pa;@_}idSn;l?H^=Oke%J>nU#>3F7S6(0c z5ghbdfs6~kv#ih&X;o75U>7>NgM)$8V=$3Ro`+I7KP+*gbtNnnGiz;+Zho?t;$ur< z&qhnZ)lR*fH}pPTbuO{0G>LYVwO`PeC=7TR78b_A#U(``$-)Lp(s_|viD$N)sGg%( z$f8sxD>->{O#Lznbg~NFa#sWiH*QH=&rR*BamBT?wm?JG7lGq)-0S0B-fJ!>T|I)x z_KE?>Y{V9Wk}Ji=t`fsg;Zyk%7OP%D_ZCQukF2#eP)0~&BcK!zi|c+}*GM$fST~N0 z&PI~8?AsiS_n$>)wSK$ur4G9wchqC{2Xs7C2@`YRdI)GkBh#cce%dz9C}RU6gdz$s zW2oNH@15l&rJ6f9kDm<(DG`&+R>gOv&I&THa8H-21=;XKJw5rCp_m^d(#%qSY@+wn z?w1jw7_nO}i=-!@Gf0IU07MDxULZmLMwnk^`^1~htQ~VFRW=E&ESzt+k9k}eQ}^Z= zp;~X~w|J32nz2z3N`VnGh%%`~jMR&C*}Ey#*?YxzqbeUr?%P>Z$URmF=U ztbNtqkfqrxSEsVrki$1XBK`iM2A`$8FRL}9Im^uv)q;@HmN6-2epMviOgoFHEF1pX zek`=IOM==TY3MV0c(0^YyUacukYl4E!M;LrPD znNCvAB_-{Fvo+^&oviOnQDj&N>3nBq&akapd4MQvcW)GPkGx^i_Q`k2FIBaSbO8m< z?qac52G1+(osl{2@PwQG@&`v>_iB`{Av$j#H1(gD1V9hJKs|ARXkW*c$)?GjRvt)| zBh^+LUm%k*4_3C~$wmgfVClXdkgi)}6Vf`ObNqemTq~W4TOVcq;4k+k$N!hSUN}Pqgy+#h67=vs7wP z%zeoyVYLtbNt=&KU1aJ7uy^h#6kDc6K#fQ>9A1O#fL0R+n<{jc8(zy{4|k2Wbhd^Q zR^#3mHMk3bu+uWHH;UpqVLmE?;d3xGrLjyrd)eZ6J6H!ZYS^B?QXvVUWLl9x@?!(M zMqOs7;Ba|%l;BQ+&Ye_WeD_TC53R`1{y~;OE}RC*R9cmrfD2>Ft^>-|N&B{P-{inb zBQ(iwkLii^bwK|xP!k*bOS!!-_^RA7g}DU(R9)ledyx*Uj8bd!f}w>O9G^ka4MfR^ zEcgSr$-PVG@^Y3=tVCLp&bJ3}<}Y)61p~=Fx4Z49r&02vS3VVV?LpMDS%A%f%WW`v z+})~)>=qwfd{CE-LS<+604P8N4NoN-0+8Xkrs z6X6>hpfVKP1fED_PS7>%M;rA7+7LOZn+^5}=@FHuFRf4j@GjguweE~?6Bzp}MsX6- zo9fa}PpsAlB4SE{*!^yg)S%B9DW0K}`PL4<%EDc*_IR zVglJA^JtWgyf;o5t=rAbBZcwq?oQpZvDcDI1Dp_OX0MssW1MOcJiXpGte?&?SjVZ6 zfM=W(UBFg*>j<}ji@^CInbHDX6K+sO0~>FZ6C!2+891S%i%jVHz+m6=IN-LiF~~kb zKzr<5E!$MFzH*(($$>k4INN>YPPi}v4Lun;_RtMODld;Zw8Or5(>~icGfUggBusb+ zyKa%JCir4gS`Qu*SKs^1g9DW}uQ~mx{0<(`@j`4X3x0n8%9-Nn)fNMX&!+ZpyvL^# zjKY^1FB9$d`e3jdZ%IWf%feE>G4k8YI5PC?%)rJdX<|a5GXSLCT4b9-5X>>yKkCeK zOzIRhtV-znEz`tGH{rIS;3#g`VvAGsLv+xScz9oS(k3R~h9Rom>9vGR;aL@NApJ4x zRQ+OAYJ4DcS9NibV8H$&BC4AS1yA>QxUH8yQYwi$*C-(GaM4IsLZ$S|Xe5IL?znQT zgikd!e`>S*yu@j}Pd464_^jBFnIgcS6kV>6Zp4WvW3tJ&xmdREJ5VU%GvBsa?I&^9UZRVLaMgg1H!R8WeBFo^)Dp- zbC%MIikY9n+S=LxdV1tpUfxXJuJ!ERJ;)zVmtLb}(a=q+Ei2C=p5Oee@}Fg2Hdje` zXf28QmUY^ik#Aeh#QW}PL7vry67|g$IYo0Fe(akv7#NRMa^3OZcJQv=Z## zs~3;l>800d)q3jc zlS~L7!>O&SZRs~BB8jc8qmu&pplx9AqTQsG5Zp_FIOO~7ZG-C51Ae6Cr7e8FN-AQ;iOm) zntGg1yJ+1o?f93D@r1ulHB}kT)|8)Fz;c#N1{*@)f||yb+XM&sbcH^x+a%GjtIwV*TE(&<3q88O;BB2rRI)6z5R=VJ z)Z)AwX|!*3Nlr^k%Q2B`P%&jAV_>jY(b4PJ*lSm3+ki1#kp@)( zP^4j1w?o&vR;*}-^;@TkaL&lZs!W{+?c6n2m7^rux(*Z~1lZZx+05f;=qd=^g(A_u zK>6@OVDQ6QgPECjD^&w!TPvA+`M1p!QA(@!J)Zutv9VA;7ME(mL@@%07qA^?Z^s|2T-)eIVJkzhXzIW5AY46VGnU=azB^xMHj*X;GQyMR` zp{G4QYFV1Y#qTjVv6)9)Fzn(T=v%XUu@!Gd#$=rp73Eex)Ysz^M$=ZV+45y+2#wso zzg@u}+_aox3xQUM&8zs_H@uQPv|{E`(`(LybX z1OU4Wy0~I!LeF%hFYcsnwhoB$r)|2JG1s%$D?%AP9qAO`bqAISb|kQ6^_s&iNQm}o`zC!93VuQD zMSCOei*V?+7kWr^xG|>odR%!k=`!9=pKE}ese!&o>reMoT<{qs-B<{N+$`)w^rw-`YfsS8JDT= zQ>{dz^Pv>mLVw3&g$Au@{-vg-8sLqQXZkX%=fiLS^T^nO|Jtv&pIW1$q700b_b$EP zPiS=wBqiahlyqX;JVKHUs_8)4m;fRK>2XkB&r+;MX0o+ZmETh^Vv8}`a+AN`KwhCY(4szydFs!tts?t5Bg;l=cJ_Ft_~)WQzjB#XEffXKUMg9XKeeHV zMSD3}+0ZiUonHI2XFXsvBO|LyX6LdEPZ{CM=aNT+Y_LhS+J3g4jt*kGiCT&*H2WcN zCKePFFtQIi@R3IGkiCog(c0i` z-yHsL(T7+~(%yXlGTJyChl!IZjZlJwU<)J>;NdMK5;fu>q#4c2iY&acRq|T*1I8V-Wf-( zL{K_a4w@5B8fySX$TkwO^C9}t4$*oN+nM>JIOqxz62T+n-6alUy9i;KaPYZ)c8Giq zMp4B%5iHVkV)K^|s^0X#5jl$gsh67_+`UcN4P@|{Xw{$(6vvH4J8%IqvIV+ZH_W^Nd&e)yBn zmh<&rH|eq6e!a1}X;{7U;m$v@(WF5n>}y)FCH}u1aTAn*jAZlSz#De)=D@IP!?RzW zd}h7HLMSbvo+@~#vK#xveP7~WFIwSk#Q6iy;i)+I-c}B1iHrlj_0|l=cV9`?yR~vI zAIMKP!@41TRCJ(mHS5SyFza%{@-*wc0^2UWSb4>R--}H7rI>{REmi^$6NJfM1wB%& zOTg$&VB~`t+I9yk*htZ4w=p-XpT4KvnC6lXRF&{aO-7@Yeam9MTEG8&ixvUbx_E2z zF7frlp`PiF(=)M%MjmCp#;L_C9An4h?g599Z^DGt0%!e2pxwUy70FJ4#h}i|ydB%o z)ef)Wx!`{MxZlm!FXMh+0h>P8;OB5kcq+-$e1@2z*av%AKE6^nTet=>7ua(7!on~z zZFD`AN1KS)jM}M7zyJmWp9V?Ore>GqS!J;m1>9$&$9dvXo=6?Z#KmO@lx(!1GIXwo zx0ko|RTU`al9*OV*#gT*vYJ+puZ=eZo`!iLhAw!9gxD}Q4^c#MbN+U$!3sT2YRw$$ zr=xd1VRvmeW6sDr44kaC%`&EqKrGpQE5NW0;!YPNo{}}EFhNs5%@Y0Q;U~sSWhDaM z(A!)oy(-C%PwjGt^o|LD+?eBuI=JbsKU^+aQ}A5hWOnvj_<_= zt*=)Fksbecd9j23^XgX|nHV#62%H%Aal)sQg(mN{2}kX(r%rnB^W3y#aTOIuT*L_K ztP96N%35%Judh$Kld)z?pR(D@y5_mFR~sqoHq6|#=*c)96a9zK-JJpc(+tk#9akNY%!K&J7?l1Cr1aYf~J6}K+O!S{(M z3SHlk`@?5Ctk4lujm*4vnNS4uW3a7k%fzbVLgvg#byL!vKaW&9YGMv(9={Jt1dRKy z2KiAod>Fx#6evEPtXLsIU>i^&zJ!ymQtaxz*4*j|JmNjbA#Rg)Sw4~ z>4M4}+?ysU@I)D3=HJ1w7Wn(jQ%%xZd9^-K9iP=&*!e<&RMj&MfUA=~uvLskl|PF{ zwjVqMldRDGk!M0KC%~j;P<5ltPgr+kGclHT zK=Q6M`gi&G)JEK8(NOh-Jh+O4xp59!z$AO9z;J4xPHjSG@{1>i_xCSFtESWl_<6qD zfZBG4d_8LW9xXi|DUct|cTyWT1?RB_kbJY(+`u0;b%p&%sJWEwwvoDowlS zqh-Ea$Z#N%d$s1%DMliS1+zi!2`331>Ajea>a~sT1Mx@PK|?ERMf^>&T4H4LuGxf` za`~1Z>XoLtrKl~>8A)syM3JJFoX@ddb`CVwJ)EhqCzShLdYGojRJ}uEpyK?-&X7vP(+DPw@vUJ{E{G zB*hfhF=MiR``d$soXNb1x+&Vn<(JdzkWIdBZS2L-Gkf+4(fb_2$-T30w7C zW&j5IanYvn2+qSQrF~DsQ7j{3P@Df3tbkkoyq=@z@x^@I6{jx{Iv7wzH*_yNUVrcxRb9IeKKSAjXukJUwRx=xl^p)UiPYe zo4Qq)VP{0Y*PPuN-VBP%CKC8IU8DP7=)PI%-_#AGMf`n9Iq_p!ZPxUs_g(u8ccbqbhC%0 z23$w8E8uR(-s`JgJsMK5@iN%Hna|_-aBb)kltPQ>-=8!S#I&f+(=5KB7R5-+Vf{hhFVOgbZ!?_4nv zXxgq#&g&m(%MoZ^Lqa#ZI-(P-ek~e36|n@uh%L+qK>$g!m#%xhL{%-l+~r22z7^!c z2XbmV>-DDoPula@rYs|V2@`$dtZ`zG<3C zPF>p$O4A(@7I3{Q!f&_@{RYMNLJAlRR{C~2cDvk9Mm@K<$u;i9j^2%Y8sAY3M+^?2 ziBOBbF7r3p=!*JEhOcL&0h;M9Bcfs?8;775Qfu(Mz5*7zd|9PTo%%Kt%PLGt&y!2T zh>e?Gj{?+k$mxIkAfRw5ld*0fyK!^cSjp&K`qfy}j`SSFzo%|F83ezJHMkzF&@{t^ zZYrb88;=6)%}C}~pHIIzmQ1zuy(r?|x=OVadmB>I1g5k|K4=^Q6u23jeQZf7)?bZ; z_7MA#;Y>(Zv}U>26yD=74Hi^S~fw6yg%No(A0gs=$qsN8Xli*WF)!`xPE z&-{U`r29snKd)?n0(g{(OkI<>?p1IHS@bJ`JLhn_%55@zP3RH%v+?_w+w!eFnTyau z;uj+}wpV7_MXX#FU3*&mr?{HS#v*TorILBJ)#1%d+QxkxHxUep9KK@lxHvD5v^b30V$?G!jVQap9s*hBIVE~Rpe@lWZ zzoo1TrvS)uq&&rWetnBg&jfN#@6EdYul&d@UC{=6Q$B0D4jqi5nq6XDodtFEG_*p+ zKGar>+pV`X@Aau7;8=uk=-yEX6=DXPgyiGFA`NX2`>q1w+U8?S$&Yns(YqsJswCGC zU=3>9;Kc7*S6(3BBtE8{jc-usocy{-)fBn;fi!!<76m6W-S=tX!gEBMdqfSv zDjVnw#BTbnwl6veQqG71d^BhG5BpN6+P(_*)iUx9+DVP$m-G4rc~5Tz8&j9PIjj@% z-h-f1Ve5JpH}k9o2An&cWC-z=b*XFU#W?v{Gqe5fr$R4Hf5<|XMnNikf!Mc))&G;nxRxk*P^|*4)_PmtEh_7 zss`(}30IgD=v^E)BSEo;Huk-^qrOC4k?Z}d7tz6ZdG)}gzUEYTukAB%&Gr@mHz%te z6{(IQ;5U|9m-(c$(_G2>V>O;V4gJ*c>dw!IPGp{dvK|BdZcA&8@-c)rz&w(14gG}V zU+Fm*^VL|Tn3A6>WUm=zC;z9R=lWck2 z#)FkcaFbnFme=`aC+QP?z7g~W5-L)A3saN~K}Z;!fq}SW0KT1?{_(Qmhp%$0CR!ms zUPZS=nwp)u)0^D(nMdsx^5(>WM#klI96Zmttwil_34bn)>PMgy1g>CJtMe|Bi5v&6 zJvCF9(5CW3`AahS+N)oH$1X^eVbP4vOmNUdqc)9mZbsl4%B9 z*IHQIzW?FOEc->4!(jG^L}2GsDCwk&C?t>pbx@(v8+K}|)aZ2-QqI04wGwJ@?;<0h z5cAU6chsY9pJUxFo*Mz_q8YDIt*_+C{C6B2o|_QrPJnRAC9N04;uk^n8=f%k6369- z-_Vat@+XLtTJF7mb{qaASp3{UK};MYcNxwUm7QaypY3{cKP#+jcqmvlSk4&glkI5w z>((B0D1na|PcKppP5^gNepOYffS@K{7+d;T!7SJI$-mare|(geOnj>=$VSKnog=0` z`I0G&x2?$EabCWl;(0l6Pw7eHC$bw3xf^F#v#v{9GyB(B3uri$Zu-MYEFb)fA$%62*O8YaE>pgKcR612 zPAEGQ9#Z(kRaXc}cW%qAm1JO@&~yI8ID`!}S*j7#o>h*I>gc{v&c#Eh*~gQ?Hfpq~ zJ;ntdVp@T2Mk5D=n`SyrBlRbRIn_pT{{{YxMyt1yhk5R2I zCe(|@i5L48Q5+x;z31zSh}3rfq_ZnS^~o#!67Dzvf|#JVMl#Mhv(x{@ieA;!9hBc2 zZe@-A?dBM$s38*ss%7Q$_xXvGS&czgNrs7;#@{|gL0A-M+$(OtSaR5RTx~vxyBHz+;$zz93hvxObeXe=TWGeDiKh%Eb#h-O^(j&dY zMG*Qlg8;3z)dnv3>WSJ#SyAfQeUE~hsje7`@bbOY;C{Nxo9Nz5D}_~*fEG3zUn^9R zOdOgBmHWU1!N!YbWUR>hykUQOx*S0nem|LwU|IFSfmM=bE!5sL%y)A%{A!z)kHx@1 zaAyns7J7k5O_drutN9Wrzg%v-A))+(q!Hct&}JW3lmP4JUMcThf|EkEN0d-l8(j3Z zu*?Tli4=o06CZC(?bNMrHY3=&tpxGn+To_YlZoD1&g&b4y2=?>ygBu}!jRhOYN5y! z*sgZcXi)V8vc*~3HJ?VHEPD?CwRA~6Ry5!@iCFWok@iszJr_&u$F)&_e7~>o30#4~ z{v$v7_J#I0{@?4E6{dPQ+haKIs0Q)zA(W!BLyZeCZyMjFN%5*S%F&Q>!jK z=w<$^x}d^H2OMg7f-;0Jt=Jk}kJr_KVxe_IVYqmFs|IO_zd~tAcC_!G9=H{VNQXOulm;$8I_Qel7zG@FEz3UIxUm=zSL1b&nX^MHxb#`wRH zsq4Ous9q}^$Jg8FvWV}KZm`ZH=Cb+9NkeX-Tk_<$^)`k{#0d@;L$>y0td1C?%;&9! zSt+YB8IHCYx@_ovZG-*{>GIo8j=d=D__wkpS3HKvGaa;60^2W?j_lp(G%6gS}DNV~!r-uCfgemoN?} zkkRzfcW#>ReJ*cI56)T9_EheaaDqNZSYkJh+ZE>K`jc*U%MYDL0}V@tsyemW9;+Ii z#XwB=Iih&NE>0YmX`tbHKEf~a+-J(T{xH$sM!zDL$Q6?Zm>tVPGB0yYMke@kT4P&h1h#fH*L=F?Xd6}^`bSDJ~hdGz+E7e%|;b8JMm-snei}JpxTLi<` zCLjm`C9?;(SPs!PmSSOzcQXpHWf@VXNp?_7-ly1l&Kqx>UU`2=cVqbc-7$-TxMFJ# zf1m={w39O;>h8yTqOMhcN^jA^UEAZ{Jyk3@J?e|%e?jAUO zFOJ_Xd#X{#@Ns`s$A?y*$#OLiq&P23+%}@)sJuiq4vj4a%LD4)BrWLM%b`SkAE9@k z&e&xCy^YZExXe_cZ-^Sbf=Y*~PD7%pFU5i_IY}vJ$`Df{j_O%XYVGL7fvbakD#!UV zEx+c*Ym=044b%s$%8N({`Xdiir0D>+ChVEA#f0(a1li|X#*o^rv8(RWPKL@n^S3?Db?Axsk=8l=F z-0z(CqxtOHeFPDx{FmwEY0%x8r?x2iizZqi9d8i)q!XJ3l`YZb%I`S8@1brM`l&Qo zf9KkDjY;UKy-6phkOdHjZc2%(x?}bV~%v$1tKy~MF!Pkk(iQ5DuLEOa-zQe0m%thV#H z3MT}8VBo-4DH7Eb#@Og8KKe3U)uX5TlX@5kfg}1Z54m+x>#@Y{Fl2;a%EC5{QVsil zRXm#e6%E(>X`4d3nn{n`Zuz~`E2B_&@f9BoAF_(AV** z?cOZxBjL32GTLC`XbI(RpTTUe-|2XAiIUjF0`ZM{F={Jo#DPi<@0wiG3cltkr1fO@ z$Y`GG{eI+8rh|=vjlxyqAWWp2p9)oCTW&PU{D$w&W@D@sdeYGbXDzOj?FTv9$HmvP z;BxGQLb*<@- zhYI`@0d~|eOnMkagm~ycX>IV%4h$R)i9(VE9j+GSC<>3Xkze_y`Ac8en0vE;H(38t z*cqlf9j?zxkKZ~ooP%Rm`mK&e&stVAODXifb6ChTwDH0Ou$+5Qm;D@>@IL#&jHB<8 zjCQjvBz#4Aq|**a6bKi8-*!SS&K;o_4qubD4Yq&SjS&*Lnj!Z({3QI@O4&#m$=*OK zLX8dclkh;2pimfampC&|Mo9_P;mlndHF`BL3Ng!?ChY3UzhaSR>Tb-?vOo3CID22+ z$?_u%iX4Gi>Rc4EYV&Cnf1nh>puPd+*FGRe+s%)=kI8kjc0JTs>={Hp>7>*e7w+h# z{v9s1FHl+c<{dHf_o9y#(~g6?AQQb9=imG}2PR#LeKJ=VdV5<*xy*A`C^gpY_2yN# zm|uZ1;YTQC&6<2WNTbQ8N_)$AX)|AlYBtm($PQcxGkztrdGh^S9k-hQtja;jM-4Sv ztN5C7%nnvV;IzjTL%&%v)ViVzvi8WVo}G3(3Y!g2l$l*tS>z^1PSN7z*ya~+;zIp1 z$6m44aT<5IUFf{wA_jJ6{F;g|SLpg`pwhjRnZxQM3+udb+0v)aW!PBa>b&&eCP|k>(2IQTWEPUJ?)kgZGAZnbBL)R#IiDw z7%^<^QI49z=MHn0v*vK?_GZ(`DMTghc77?KjZ{-cQ!tX_(A$i*%=MeNmmD0f;rDy` zY8z#5--GW)8aY|F3BWwlUtPjHmpCbc^D$2TP8M=sT^=ArAN79)QN46q_~O8*Y<}lS zHJG_fe2G}Qz{oZ-$PE3yK_6Sl3-qjG{a_bY=u_yo-CA1cjfP2B<)DVMG|a~y3T|#_ zu^yyhk6xg_*Ni&$nLapuKf9^RqADhqaK4NK-z1+@%I7lHaXPp8zYt|4vmi*_+VFgO zB;5TbBFRW5aK9JZ_x+UC<6)1|X)vi^G)xp*CM-2NZcnc;>)=#=%_NIwY~Oy5Bo@?U zltZchScWQc?&Y^Mf(;KT>aX2JL_mjLRGRg&($iUb;3tNOmP}8ko_a_!ZoBg#oj#ss z;LhK(wC-7(<$1sH`v%MN1B-Yan)rZVX7v@FxXN&tf1jYb4epe_MSoTGtVQpfce(?2 zU5a%KX!?=)>@icS?e?5oU~yZz1qXmJXVtlf{}(n7`*K zT_1My#tQ;?_{vo$2A{NaEJJ`CkWl*39rd~v+<>bctRL_A$A3(wvE9&uRhOoVEf|>i zKqH^wD%NzP!@@q(K$qeU`MHA9z9`XZ{>{(a2@UrwLyk_sV%+=bnYv;FVH92 zYY24PBTU=pDLV7B!GbYx(z2gkOhw>3)yq<3$~!@C)|@y>t3JJLyZ(M~Ix%U}+ed6y z=6d?G7sICX%VxN^SY)wdz|u&opFH)GC+5S2Y1X7T)7YyU0hoaxyH*@d!p0}dMs^00 z8aE4!r#11$jds(#{;4{66zDe#?R}4H_7L$2NNX+L;^;b6c7NX5;wHkbf+%R-U8nLT zeh|^aQVPNm^uV)oRnc$$fUA>o%TUFntTyB^Q1h`CJ>bC_tFW>^sBlmia{HCT(F z_pWOu)+lsv)7=@dbad*&$5C{p+7}03&(-Uy$chjN@l$pt7HE>83~XTQH=SR2zNIbj z@Ck1v7X`ujE+G*W89!8dFfe6Hutw*nnKYPU|F~4n!9a{fk*wzs{Pt!%U?CKl!hK^Q z@lBH#7)8$;PF?>2j&-($aT2YRjkj0RuIC#NP1ms^WW`Zd(N+vy*W%S>t)r6Y%e1sj zO)Y**e-C)kc38K|FhI^Fyg}7HLjVN zG%0>7who6kh^nuha;_r4bW}dX7ENzbYvyA4p=$J#s6h&cz=&*VSQ>(lM1I}_X571=A|u`0v;# zm)Nfggt6n0v9tw#mcr#IhL~Xk;A~iG3fgKBwtcw|b-9&q1?vol!+fcl+WozJ+IE{Z z!kBTR*`W*cd;MiwO^yg>BQL6sef`X|cS)I2`XjC&e8isyqa0iRUjVTIPX3zG8?#8^ z@cM!n>i1!=eGr!iMiAp1N=$(ilEFEL!Kw!v8@%ryy|{=pF_ zDke6wUI%+=o@(Etv^9ys;dFXbO}>I4pej}W5`|;X<-KlPQa?VVieUEp}4>bV}jF)pg#=v@bo*7f;Mu?7l5~G7#*EM7>JII*`NeP;$Mwf zQ+z(tfuSKk2gS98u$0+g76jy4&GEXM#h6VfAd3bb7fa4L7-R7xpKJ=+#(BJq%GlO6 z!;=M_!!NmJ~+*oRphb$APN~j zqP%s?W@0{D+yX)n1VPLgS~3v?K@cP$1VIo42?#+D1VI8q5ClOiME-wW@q^J4Jopp< O000016Z( literal 0 HcmV?d00001 diff --git a/docs/index.md b/docs/index.md index 97977898..ad490e77 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,7 @@ # DetectMate Library +![UML](img/logo.png) + Welcome to the documentation for the DetectMate Library. DetectMate is a flexible, component-based library for parsing and log anomaly detection. It can be used in research and production, and together with the From 5b33c364e95830d3e62b666e877e0884f7429909 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Sat, 17 Oct 2026 19:23:47 +0200 Subject: [PATCH 16/19] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 22f98718..20ae0647 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ pip install detectmatelibrary Main documentations: -- [Project Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) +- [Library Documentation](https://ait-detectmate.github.io/DetectMateLibrary/latest/) (this repo) +- [Service Documentation](https://github.com/ait-detectmate/DetectMateService/tree/main) ## Contribution From 57cf75af8e20ff45e009cecd420616ea645572fb Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 07:46:22 +0200 Subject: [PATCH 17/19] solve unittests issues after merge --- tests/test_alert_aggregators/test_basic_concat.py | 2 +- tests/test_alert_aggregators/test_logid_count.py | 6 +++--- tests/test_common/test_core_alert_aggregator.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_alert_aggregators/test_basic_concat.py b/tests/test_alert_aggregators/test_basic_concat.py index 707afd08..46a7c52f 100644 --- a/tests/test_alert_aggregators/test_basic_concat.py +++ b/tests/test_alert_aggregators/test_basic_concat.py @@ -26,7 +26,7 @@ "detectorID": "1", "detectorType": "dummy", "alertID": "2", - "detectionTimestamp": 0, + "detectedTimestamp": 0, "logIDs": ["logID1"], "score": 0.2, "logTimestamps": [-1], diff --git a/tests/test_alert_aggregators/test_logid_count.py b/tests/test_alert_aggregators/test_logid_count.py index 4bd259ff..7fbd6b93 100644 --- a/tests/test_alert_aggregators/test_logid_count.py +++ b/tests/test_alert_aggregators/test_logid_count.py @@ -39,13 +39,13 @@ def generate_schema(log_id: str | list[str]) -> schemas.DetectorSchema: "detectorID": "1", "detectorType": "dummy", "alertID": "2", - "detectionTimestamp": 0, + "detectedTimestamp": 0, "logIDs": log_id, "score": 0.2, - "extractedTimestamps": [-1], + "logTimestamps": [-1], "description": "hello there", "receivedTimestamp": 1, - "alertsObtain": {"99 problems": "but logs aint one"} + "alertDetails": {"99 problems": "but logs aint one"} }) diff --git a/tests/test_common/test_core_alert_aggregator.py b/tests/test_common/test_core_alert_aggregator.py index befb73de..a9756273 100644 --- a/tests/test_common/test_core_alert_aggregator.py +++ b/tests/test_common/test_core_alert_aggregator.py @@ -99,11 +99,11 @@ def test_get_logID_timestamp_single(self) -> None: dummy_schema2 = dummy_schema.copy() dummy_schema2["logIDs"] = ["logID1", "logID2", "logID3", "logID1"] - dummy_schema2["extractedTimestamps"] = [-1, 0, 1, 3] + dummy_schema2["logTimestamps"] = [-1, 0, 1, 3] input_ = schemas.DetectorSchema(dummy_schema2) logids, times = _extract_list_fields( - input_, field1="logIDs", field2="extractedTimestamps" + input_, field1="logIDs", field2="logTimestamps" ) assert expected_logID == logids @@ -115,12 +115,12 @@ def test_get_logID_timestamp_multiple(self) -> None: dummy_schema2 = dummy_schema.copy() dummy_schema2["logIDs"] = ["logID2", "logID3", "logID1"] - dummy_schema2["extractedTimestamps"] = [0, 1, 3] + dummy_schema2["logTimestamps"] = [0, 1, 3] input_ = [dummy_schema, dummy_schema2] input_ = [schemas.DetectorSchema(in_) for in_ in input_] logids, times = _extract_list_fields( - input_, field1="logIDs", field2="extractedTimestamps" + input_, field1="logIDs", field2="logTimestamps" ) assert expected_logID == logids From 8574a6b08df07e399769e87ba02eba3d00840e94 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 10:09:19 +0200 Subject: [PATCH 18/19] correct schema field name --- docs/examples/alert_aggregators/logid_count.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/alert_aggregators/logid_count.py b/docs/examples/alert_aggregators/logid_count.py index f3e2f230..de92cda4 100644 --- a/docs/examples/alert_aggregators/logid_count.py +++ b/docs/examples/alert_aggregators/logid_count.py @@ -16,7 +16,7 @@ def generate_schema(log_id: str | list[str]) -> schemas.DetectorSchema: "detectorID": "1", "detectorType": "dummy", "alertID": "2", - "detectionTimestamp": 0, + "detectedTimestamp": 0, "logIDs": log_id, "score": 0.2, "extractedTimestamps": [-1], From c06659313eb1f7a232fcae3457af78063e71fb9d Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 10:36:58 +0200 Subject: [PATCH 19/19] correct schemas --- docs/examples/alert_aggregators/logid_count.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples/alert_aggregators/logid_count.py b/docs/examples/alert_aggregators/logid_count.py index de92cda4..cb9d012a 100644 --- a/docs/examples/alert_aggregators/logid_count.py +++ b/docs/examples/alert_aggregators/logid_count.py @@ -19,10 +19,10 @@ def generate_schema(log_id: str | list[str]) -> schemas.DetectorSchema: "detectedTimestamp": 0, "logIDs": log_id, "score": 0.2, - "extractedTimestamps": [-1], + "logTimestamps": [-1], "description": "hello there", "receivedTimestamp": 1, - "alertsObtain": {"99 problems": "but logs aint one"} + "alertDetails": {"99 problems": "but logs aint one"} })