From 0405205f87f6effb080d1e299b0673f62c44f820 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 26 May 2021 14:50:05 +0300 Subject: [PATCH 01/17] Add new bot: time filter --- intelmq/bots/experts/time_filter/__init__.py | 0 intelmq/bots/experts/time_filter/expert.py | 55 +++++++++ .../bots/experts/time_filter/__init__.py | 0 .../bots/experts/time_filter/test_expert.py | 106 ++++++++++++++++++ 4 files changed, 161 insertions(+) create mode 100644 intelmq/bots/experts/time_filter/__init__.py create mode 100644 intelmq/bots/experts/time_filter/expert.py create mode 100644 intelmq/tests/bots/experts/time_filter/__init__.py create mode 100644 intelmq/tests/bots/experts/time_filter/test_expert.py diff --git a/intelmq/bots/experts/time_filter/__init__.py b/intelmq/bots/experts/time_filter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py new file mode 100644 index 0000000000..bd6d6e8bb4 --- /dev/null +++ b/intelmq/bots/experts/time_filter/expert.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +from datetime import datetime, timedelta +from dateutil import parser +from intelmq.lib.utils import get_timedelta +from intelmq.lib.bot import Bot +from datetime import timezone + + +class TimeFilterExpertBot(Bot): + search_field: str = 'time.source' + search_from: str = '1d' + + not_after = None + + def init(self): + self.search_field = self.search_field + if self.search_from: + timedelta_params = get_timedelta(self.search_from) + self.not_after = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) + + def process(self): + event = self.receive_message() + # time based filtering + if self.search_field in event: + try: + event_time = parser.parse(str(event.get(self.search_field))) + except ValueError: + event_time = self.not_after + self.process_message(event_time, event) + return + else: + self.process_message(event_time, event) + return + else: + # not found field + event_time = self.not_after + self.process_message(event_time, event) + return + + def process_message(self, event_time, event): + event_time = event_time.replace(tzinfo=None) + self.not_after = self.not_after.replace(tzinfo=None) + + if event_time < self.not_after: + self.acknowledge_message() + self.logger.debug( + f"Filtered out event with search field {self.search_field} and event time {event_time} .") + return + else: + self.send_message(event) + self.acknowledge_message() + return + + +BOT = TimeFilterExpertBot diff --git a/intelmq/tests/bots/experts/time_filter/__init__.py b/intelmq/tests/bots/experts/time_filter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py new file mode 100644 index 0000000000..e8ff0ede00 --- /dev/null +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import unittest +from freezegun import freeze_time + +import intelmq.lib.test as test +from intelmq.bots.experts.time_filter.expert import TimeFilterExpertBot + +EXAMPLE_INPUT_DROP = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "2020-08-13T00:00:00+00:00", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline1": "2020-09-13T00:00:00+00:00", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS_2 = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} +EXAMPLE_INPUT_PASS_3 = { + "__type": "Event", + "feed.accuracy": 90.0, + "feed.name": "Feodo Tracker IPs", + "feed.provider": "abuse.ch", + "feed.url": "https://feodotracker.abuse.ch/downloads/ipblocklist.csv", + "time.observation": "2020-10-13T06:14:49+00:00", + "raw": "dGVzdA==", + "extra.firstseen": "2020-10-11T02:10:59+00:00", + "source.port": 447, + "extra.lastonline": "2020-09-13", + "malware.name": "trickbot", + "time.source": "2020-10-13T00:00:00+00:00" +} + + +class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for TimeFilterExpertBot handling Reports. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = TimeFilterExpertBot + cls.input_message = EXAMPLE_INPUT_DROP + cls.sysconfig = { + 'search_field': 'extra.lastonline', + 'search_from': "1d" + } + + @freeze_time("2021-05-05") + def test_expert_drop(self): + self.run_bot() + self.assertOutputQueueLen(0) + + @freeze_time("2020-09-09") + def test_expert_pass(self): + self.input_message = EXAMPLE_INPUT_PASS + self.run_bot() + self.assertOutputQueueLen(1) + + @freeze_time("2020-09-09") + def test_expert_pass_2(self): + self.input_message = EXAMPLE_INPUT_PASS_2 + self.run_bot() + self.assertOutputQueueLen(1) + + @freeze_time("2020-09-09") + def test_expert_pass_3(self): + self.input_message = EXAMPLE_INPUT_PASS_3 + self.run_bot() + self.assertOutputQueueLen(1) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() From ebd9e99f18214f2dde355637e42da2a6dc63c0c8 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Mon, 14 Jun 2021 16:30:18 +0300 Subject: [PATCH 02/17] Small changes --- intelmq/bots/experts/time_filter/expert.py | 40 ++++++++++------------ 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index bd6d6e8bb4..14e2305a73 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -7,25 +7,25 @@ class TimeFilterExpertBot(Bot): - search_field: str = 'time.source' - search_from: str = '1d' + """ Time based filtering """ + field: str = 'time.source' + timespan: str = '1d' - not_after = None + __delta = None def init(self): - self.search_field = self.search_field - if self.search_from: - timedelta_params = get_timedelta(self.search_from) - self.not_after = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) + if self.field: + timedelta_params = get_timedelta(self.timespan) + self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) def process(self): event = self.receive_message() - # time based filtering - if self.search_field in event: + event_time = self.__delta + + if self.field in event: try: - event_time = parser.parse(str(event.get(self.search_field))) + event_time = parser.parse(str(event.get(self.field))) except ValueError: - event_time = self.not_after self.process_message(event_time, event) return else: @@ -33,23 +33,19 @@ def process(self): return else: # not found field - event_time = self.not_after self.process_message(event_time, event) return def process_message(self, event_time, event): event_time = event_time.replace(tzinfo=None) - self.not_after = self.not_after.replace(tzinfo=None) + self.__delta = self.__delta.replace(tzinfo=None) - if event_time < self.not_after: - self.acknowledge_message() - self.logger.debug( - f"Filtered out event with search field {self.search_field} and event time {event_time} .") - return - else: + if event_time > self.__delta: self.send_message(event) - self.acknowledge_message() - return - + else: + self.logger.debug(f"Filtered out event with search field {self.field} and event time {event_time} .") + + self.acknowledge_message() + BOT = TimeFilterExpertBot From 86056a60d8e91f3c6c3f445d8dc9fb7266fb7648 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 4 Aug 2021 14:43:53 +0300 Subject: [PATCH 03/17] Update bot and add documentation --- docs/user/bots.rst | 18 ++++++++++++++++++ intelmq/bots/experts/time_filter/expert.py | 3 +++ 2 files changed, 21 insertions(+) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 4e0a269c0e..519a74cdb5 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3097,6 +3097,24 @@ Please note: Even if a message is sent, any further identical messages are dropp .. _intelmq.bots.experts.tor_nodes.expert: +Time Filter +^^^^^^^^^ + +**Information** + +* `name:` `intelmq.bots.experts.time_filter.expert` +* `lookup:` no +* `public:` yes +* `cache (redis db):` none +* `description:` Time based filtering + +**Configuration Parameters** + +* `field`: `time.source` +* `timespan`: `1d` + +.. _intelmq.bots.experts.time_filter.expert: + Tor Nodes ^^^^^^^^^ diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 14e2305a73..21df1af70c 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +""" +Time based filtering +""" from datetime import datetime, timedelta from dateutil import parser from intelmq.lib.utils import get_timedelta From 167fb370fb1b0d22b268282d5acfc1b36322f39d Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 6 Aug 2021 09:33:23 +0300 Subject: [PATCH 04/17] Add license --- intelmq/bots/experts/time_filter/expert.py | 4 ++++ intelmq/tests/bots/experts/time_filter/test_expert.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 21df1af70c..0f02017ce9 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- """ Time based filtering + +SPDX-FileCopyrightText: 2021 Marius Karotkis +SPDX-License-Identifier: AGPL-3.0-or-later """ + from datetime import datetime, timedelta from dateutil import parser from intelmq.lib.utils import get_timedelta diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index e8ff0ede00..0929fd385c 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -1,4 +1,10 @@ # -*- coding: utf-8 -*- +""" +Time based filtering + +SPDX-FileCopyrightText: 2021 Marius Karotkis +SPDX-License-Identifier: AGPL-3.0-or-later +""" import unittest from freezegun import freeze_time From 760ec91fe70b66e6e4fa7fc7c85b08e2bea3746d Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 09:51:27 +0300 Subject: [PATCH 05/17] Add requirements, small fix --- docs/user/bots.rst | 5 +++-- .../bots/experts/time_filter/REQUIREMENTS.txt | 1 + intelmq/bots/experts/time_filter/expert.py | 16 ++++++++++++---- .../bots/experts/time_filter/REQUIREMENTS.txt | 1 + .../bots/experts/time_filter/test_expert.py | 13 ++++++++++++- 5 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 intelmq/bots/experts/time_filter/REQUIREMENTS.txt create mode 100644 intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 519a74cdb5..31f2ff43dc 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3095,7 +3095,7 @@ This bot has certain limitations and is not a true threshold filter (yet). It wo Please note: Even if a message is sent, any further identical messages are dropped, if the time difference to the last message is less than the timeout! The counter is not reset if the threshold is reached. -.. _intelmq.bots.experts.tor_nodes.expert: +.. _intelmq.bots.experts.time_filter.expert: Time Filter ^^^^^^^^^ @@ -3113,7 +3113,8 @@ Time Filter * `field`: `time.source` * `timespan`: `1d` -.. _intelmq.bots.experts.time_filter.expert: + +.. _intelmq.bots.experts.tor_nodes.expert: Tor Nodes ^^^^^^^^^ diff --git a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt new file mode 100644 index 0000000000..39ca0b49b8 --- /dev/null +++ b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt @@ -0,0 +1 @@ +get_timedelta diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 0f02017ce9..eecb6335a4 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -8,20 +8,28 @@ from datetime import datetime, timedelta from dateutil import parser -from intelmq.lib.utils import get_timedelta from intelmq.lib.bot import Bot from datetime import timezone +from intelmq.lib.exceptions import MissingDependencyError + +try: + from intelmq.lib.utils import get_timedelta +except ImportError: + get_timedelta = None class TimeFilterExpertBot(Bot): """ Time based filtering """ field: str = 'time.source' - timespan: str = '1d' + timespan: str = '1d' __delta = None def init(self): if self.field: + if get_timedelta is None: + raise MissingDependencyError("get_timedelta") + timedelta_params = get_timedelta(self.timespan) self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) @@ -51,8 +59,8 @@ def process_message(self, event_time, event): self.send_message(event) else: self.logger.debug(f"Filtered out event with search field {self.field} and event time {event_time} .") - + self.acknowledge_message() - + BOT = TimeFilterExpertBot diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt new file mode 100644 index 0000000000..fa5ae1fefe --- /dev/null +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -0,0 +1 @@ +freeze_time \ No newline at end of file diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 0929fd385c..97a964cd1e 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -7,10 +7,15 @@ """ import unittest -from freezegun import freeze_time import intelmq.lib.test as test from intelmq.bots.experts.time_filter.expert import TimeFilterExpertBot +from intelmq.lib.exceptions import MissingDependencyError + +try: + from freezegun import freeze_time +except ImportError: + freeze_time = None EXAMPLE_INPUT_DROP = { "__type": "Event", @@ -70,11 +75,17 @@ } +@test.skip_exotic() class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): """ A TestCase for TimeFilterExpertBot handling Reports. """ + @classmethod + def init(cls): + if freeze_time is None: + raise MissingDependencyError("freeze_time") + @classmethod def set_bot(cls): cls.bot_reference = TimeFilterExpertBot From b6cdad4cca13eb8cc1386f7b28ad8fe9be79735e Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 09:57:32 +0300 Subject: [PATCH 06/17] Add license to requirements --- intelmq/bots/experts/time_filter/REQUIREMENTS.txt | 3 +++ intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt index 39ca0b49b8..71d1b21b11 100644 --- a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt @@ -1 +1,4 @@ +# SPDX-FileCopyrightText: 2021 Marius Karotkis +# SPDX-License-Identifier: AGPL-3.0-or-later + get_timedelta diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index fa5ae1fefe..7ae9c7038a 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1 +1,4 @@ -freeze_time \ No newline at end of file +# SPDX-FileCopyrightText: 2021 Marius Karotkis +# SPDX-License-Identifier: AGPL-3.0-or-later + +freeze_time From 0f615a8b60d341eed23ea4652680596643a1e82a Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 10:06:47 +0300 Subject: [PATCH 07/17] Add to utils get_timedelta --- intelmq/bots/experts/time_filter/REQUIREMENTS.txt | 4 ---- intelmq/bots/experts/time_filter/expert.py | 10 +--------- intelmq/lib/utils.py | 10 ++++++++++ 3 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 intelmq/bots/experts/time_filter/REQUIREMENTS.txt diff --git a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/bots/experts/time_filter/REQUIREMENTS.txt deleted file mode 100644 index 71d1b21b11..0000000000 --- a/intelmq/bots/experts/time_filter/REQUIREMENTS.txt +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Marius Karotkis -# SPDX-License-Identifier: AGPL-3.0-or-later - -get_timedelta diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index eecb6335a4..2b431ec518 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -10,12 +10,7 @@ from dateutil import parser from intelmq.lib.bot import Bot from datetime import timezone -from intelmq.lib.exceptions import MissingDependencyError - -try: - from intelmq.lib.utils import get_timedelta -except ImportError: - get_timedelta = None +from intelmq.lib.utils import get_timedelta class TimeFilterExpertBot(Bot): @@ -27,9 +22,6 @@ class TimeFilterExpertBot(Bot): def init(self): if self.field: - if get_timedelta is None: - raise MissingDependencyError("get_timedelta") - timedelta_params = get_timedelta(self.timespan) self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index d9699ff671..8242351915 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -910,3 +910,13 @@ def get_bots_settings(bot_id: str = None) -> dict: if 'global' in runtime_conf: del runtime_conf['global'] return runtime_conf + +def get_timedelta(search_last): + m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) + if not m: raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") + date_mapping = { + "d": "days", + "h": "hours", + "m": "minutes" + } + return {date_mapping[m.group('map')]: int(m.group("number"))} \ No newline at end of file From 2e35b650ba0777f6e372ebfa087dec6360efe722 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 10:09:10 +0300 Subject: [PATCH 08/17] Fix codestyle --- intelmq/lib/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 8242351915..98fb31bb9a 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -850,7 +850,8 @@ def list_all_bots() -> dict: base_path = resource_filename('intelmq', 'bots') - botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if botfile.is_file() and botfile.name != '__init__.py'] + botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if + botfile.is_file() and botfile.name != '__init__.py'] for file in botfiles: file = Path(file.as_posix().replace(base_path, 'intelmq/bots')) mod = importlib.import_module('.'.join(file.with_suffix('').parts)) @@ -870,7 +871,8 @@ def list_all_bots() -> dict: bots[file.parts[2].capitalize()[:-1]][name] = { "module": mod.__name__, - "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent(mod.BOT.__doc__), + "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent( + mod.BOT.__doc__), "parameters": keys, } return bots @@ -911,6 +913,7 @@ def get_bots_settings(bot_id: str = None) -> dict: del runtime_conf['global'] return runtime_conf + def get_timedelta(search_last): m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) if not m: raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") @@ -919,4 +922,4 @@ def get_timedelta(search_last): "h": "hours", "m": "minutes" } - return {date_mapping[m.group('map')]: int(m.group("number"))} \ No newline at end of file + return {date_mapping[m.group('map')]: int(m.group("number"))} From dd4e1387cc37b52496c53e850027de791d207e1d Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 20 Aug 2021 16:10:55 +0300 Subject: [PATCH 09/17] Fix --- intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index 7ae9c7038a..f81fab6dad 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2021 Marius Karotkis # SPDX-License-Identifier: AGPL-3.0-or-later -freeze_time +freezegun From 9d60d5b2923abc822e7649f292a73a5d6f3d71e4 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:00:41 +0300 Subject: [PATCH 10/17] Fix --- intelmq/lib/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 0b0b739c41..79b5dd836c 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -922,7 +922,8 @@ def get_bots_settings(bot_id: str = None) -> dict: def get_timedelta(search_last): m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) - if not m: raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") + if not m: + raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") date_mapping = { "d": "days", "h": "hours", From f111f9e2c6d3053f2223acb28955e2ac37922353 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:19:19 +0300 Subject: [PATCH 11/17] Fix --- intelmq/tests/bots/experts/time_filter/test_expert.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 97a964cd1e..c9566745ab 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -95,23 +95,27 @@ def set_bot(cls): 'search_from': "1d" } + @test.skip_exotic() @freeze_time("2021-05-05") def test_expert_drop(self): self.run_bot() self.assertOutputQueueLen(0) + @test.skip_exotic() @freeze_time("2020-09-09") def test_expert_pass(self): self.input_message = EXAMPLE_INPUT_PASS self.run_bot() self.assertOutputQueueLen(1) + @test.skip_exotic() @freeze_time("2020-09-09") def test_expert_pass_2(self): self.input_message = EXAMPLE_INPUT_PASS_2 self.run_bot() self.assertOutputQueueLen(1) + @test.skip_exotic() @freeze_time("2020-09-09") def test_expert_pass_3(self): self.input_message = EXAMPLE_INPUT_PASS_3 From 811871b478cad0af52dc0e3133fb00febc7feb27 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:48:53 +0300 Subject: [PATCH 12/17] Change freezegun to time_machine --- .../bots/experts/time_filter/REQUIREMENTS.txt | 2 +- .../bots/experts/time_filter/test_expert.py | 24 +++++++------------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index f81fab6dad..3e3e4172c5 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2021 Marius Karotkis # SPDX-License-Identifier: AGPL-3.0-or-later -freezegun +time_machine diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index c9566745ab..6d9a471ddc 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -13,9 +13,9 @@ from intelmq.lib.exceptions import MissingDependencyError try: - from freezegun import freeze_time + import time_machine except ImportError: - freeze_time = None + time_machine = None EXAMPLE_INPUT_DROP = { "__type": "Event", @@ -81,13 +81,11 @@ class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): A TestCase for TimeFilterExpertBot handling Reports. """ - @classmethod - def init(cls): - if freeze_time is None: - raise MissingDependencyError("freeze_time") - @classmethod def set_bot(cls): + if time_machine is None: + raise MissingDependencyError("time_machine") + cls.bot_reference = TimeFilterExpertBot cls.input_message = EXAMPLE_INPUT_DROP cls.sysconfig = { @@ -95,28 +93,24 @@ def set_bot(cls): 'search_from': "1d" } - @test.skip_exotic() - @freeze_time("2021-05-05") + @time_machine.travel("2021-05-05") def test_expert_drop(self): self.run_bot() self.assertOutputQueueLen(0) - @test.skip_exotic() - @freeze_time("2020-09-09") + @time_machine.travel("2020-09-09") def test_expert_pass(self): self.input_message = EXAMPLE_INPUT_PASS self.run_bot() self.assertOutputQueueLen(1) - @test.skip_exotic() - @freeze_time("2020-09-09") + @time_machine.travel("2020-09-09") def test_expert_pass_2(self): self.input_message = EXAMPLE_INPUT_PASS_2 self.run_bot() self.assertOutputQueueLen(1) - @test.skip_exotic() - @freeze_time("2020-09-09") + @time_machine.travel("2020-09-09") def test_expert_pass_3(self): self.input_message = EXAMPLE_INPUT_PASS_3 self.run_bot() From 9d41b4a2d4b8031ec0eec77b655eb5450ff26cc7 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Tue, 24 Aug 2021 14:58:01 +0300 Subject: [PATCH 13/17] fix time-machine --- intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt index 3e3e4172c5..b00113cdef 100644 --- a/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt +++ b/intelmq/tests/bots/experts/time_filter/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2021 Marius Karotkis # SPDX-License-Identifier: AGPL-3.0-or-later -time_machine +time-machine From 2d8f636ed080242e78d5e2cc6ee765ed5f7ce9d4 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 25 Aug 2021 15:20:17 +0300 Subject: [PATCH 14/17] Fix tests --- .../bots/experts/time_filter/test_expert.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 6d9a471ddc..5c0538148d 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -93,28 +93,29 @@ def set_bot(cls): 'search_from': "1d" } - @time_machine.travel("2021-05-05") - def test_expert_drop(self): - self.run_bot() - self.assertOutputQueueLen(0) + if time_machine: + @time_machine.travel("2021-05-05") + def test_expert_drop(self): + self.run_bot() + self.assertOutputQueueLen(0) - @time_machine.travel("2020-09-09") - def test_expert_pass(self): - self.input_message = EXAMPLE_INPUT_PASS - self.run_bot() - self.assertOutputQueueLen(1) + @time_machine.travel("2020-09-09") + def test_expert_pass(self): + self.input_message = EXAMPLE_INPUT_PASS + self.run_bot() + self.assertOutputQueueLen(1) - @time_machine.travel("2020-09-09") - def test_expert_pass_2(self): - self.input_message = EXAMPLE_INPUT_PASS_2 - self.run_bot() - self.assertOutputQueueLen(1) + @time_machine.travel("2020-09-09") + def test_expert_pass_2(self): + self.input_message = EXAMPLE_INPUT_PASS_2 + self.run_bot() + self.assertOutputQueueLen(1) - @time_machine.travel("2020-09-09") - def test_expert_pass_3(self): - self.input_message = EXAMPLE_INPUT_PASS_3 - self.run_bot() - self.assertOutputQueueLen(1) + @time_machine.travel("2020-09-09") + def test_expert_pass_3(self): + self.input_message = EXAMPLE_INPUT_PASS_3 + self.run_bot() + self.assertOutputQueueLen(1) if __name__ == '__main__': # pragma: no cover From a20be810d4704bfe13c9d375735ce60a9164b2dc Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Wed, 25 Aug 2021 15:34:54 +0300 Subject: [PATCH 15/17] Fix tests --- intelmq/tests/bots/experts/time_filter/test_expert.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 5c0538148d..8239ae34d0 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -83,9 +83,6 @@ class TestFilterExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(cls): - if time_machine is None: - raise MissingDependencyError("time_machine") - cls.bot_reference = TimeFilterExpertBot cls.input_message = EXAMPLE_INPUT_DROP cls.sysconfig = { From adb68ee2e93df31ff7b06305ee77f139ebbf2d41 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 27 Aug 2021 13:04:24 +0300 Subject: [PATCH 16/17] Add description --- docs/user/bots.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index dcacf17644..a2f09d1d95 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3205,7 +3205,6 @@ Threshold **Information** - * **Cache parameters** (see section :ref:`common-parameters`) * `name`: `intelmq.bots.experts.threshold.expert` * `lookup`: redis cache @@ -3244,6 +3243,8 @@ Please note: Even if a message is sent, any further identical messages are dropp Time Filter ^^^^^^^^^ +Filtering records that do not exceed the specified deadline. + **Information** * `name:` `intelmq.bots.experts.time_filter.expert` @@ -3254,8 +3255,12 @@ Time Filter **Configuration Parameters** -* `field`: `time.source` -* `timespan`: `1d` +* `field`: the event field which should be filtered on. Needs to be in ISO 8601 formatted. Default: `time.source` +* `timespan`: time interval for filtering records. Default: `1d` + +**Description** + +The bot filters new records, for example when only records from the last 2 days are needed. Also allows entries without a date. .. _intelmq.bots.experts.tor_nodes.expert: From cd41494affd907a20032021772709c807349c1d0 Mon Sep 17 00:00:00 2001 From: "marius.karotkis" Date: Fri, 27 Aug 2021 13:23:39 +0300 Subject: [PATCH 17/17] Remove similar function --- docs/user/bots.rst | 4 ++-- intelmq/bots/experts/time_filter/expert.py | 7 +++---- intelmq/lib/utils.py | 12 ------------ .../tests/bots/experts/time_filter/test_expert.py | 2 +- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index a2f09d1d95..bf4c528a57 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -3256,11 +3256,11 @@ Filtering records that do not exceed the specified deadline. **Configuration Parameters** * `field`: the event field which should be filtered on. Needs to be in ISO 8601 formatted. Default: `time.source` -* `timespan`: time interval for filtering records. Default: `1d` +* `timespan`: time interval for filtering records. Default: `24 hours` **Description** -The bot filters new records, for example when only records from the last 2 days are needed. Also allows entries without a date. +The bot filters new records, for example when only records from the last 2 days (48 hours) are needed. Also allows entries without a date. .. _intelmq.bots.experts.tor_nodes.expert: diff --git a/intelmq/bots/experts/time_filter/expert.py b/intelmq/bots/experts/time_filter/expert.py index 2b431ec518..ce07395665 100644 --- a/intelmq/bots/experts/time_filter/expert.py +++ b/intelmq/bots/experts/time_filter/expert.py @@ -10,20 +10,19 @@ from dateutil import parser from intelmq.lib.bot import Bot from datetime import timezone -from intelmq.lib.utils import get_timedelta +from intelmq.lib.utils import parse_relative class TimeFilterExpertBot(Bot): """ Time based filtering """ field: str = 'time.source' - timespan: str = '1d' + timespan: str = '24 hours' __delta = None def init(self): if self.field: - timedelta_params = get_timedelta(self.timespan) - self.__delta = datetime.now(tz=timezone.utc) - timedelta(**timedelta_params) + self.__delta = datetime.now(tz=timezone.utc) - timedelta(minutes=parse_relative(self.timespan)) def process(self): event = self.receive_message() diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 79b5dd836c..8f0eb814ce 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -918,15 +918,3 @@ def get_bots_settings(bot_id: str = None) -> dict: if 'global' in runtime_conf: del runtime_conf['global'] return runtime_conf - - -def get_timedelta(search_last): - m = re.search('(?P^[1-9]+[0-9]*)(?P[dmh])$', search_last) - if not m: - raise ValueError(f"ERROR: Incorrect search_last {search_last} parameter. Example 14h, 166d, 15m") - date_mapping = { - "d": "days", - "h": "hours", - "m": "minutes" - } - return {date_mapping[m.group('map')]: int(m.group("number"))} diff --git a/intelmq/tests/bots/experts/time_filter/test_expert.py b/intelmq/tests/bots/experts/time_filter/test_expert.py index 8239ae34d0..5f47b96b07 100644 --- a/intelmq/tests/bots/experts/time_filter/test_expert.py +++ b/intelmq/tests/bots/experts/time_filter/test_expert.py @@ -87,7 +87,7 @@ def set_bot(cls): cls.input_message = EXAMPLE_INPUT_DROP cls.sysconfig = { 'search_field': 'extra.lastonline', - 'search_from': "1d" + 'search_from': "72 hours" } if time_machine: