Merge pull request #676 from Intevation/dev-filterexpert

ENH: Filterexpert can now use RegEx

Signed-off-by: Sebastian Wagner <[email protected]>

Author: sebix

intelmq/bots/BOTS

@@ -702,6 +702,7 @@
             "parameters": {
                 "filter_action": "<keep/drop>",
                 "filter_key": "<source.geolocation.cc>",
+                "filter_regex": "",
                 "filter_value": "<PT>"
             }
         },

intelmq/bots/experts/filter/README.md

@@ -1,20 +1,22 @@
-### Filter Bot
+# Filter Bot
 
-Bot allowing out specific events
+The filter bot is capable of filtering specific events.
 
-Parameters for filtering with key/value attributes:
-* filter_key - key from data harmonization
-* filter_value - value for the key
-* filter_action - action when a message match to the criteria (possible actions: keep/drop)
+## Parameters for filtering with key/value attributes:
+* `filter_key` - key from data harmonization
+* `filter_value` - value for the key
+* `filter_action` - action when a message match to the criteria (possible actions: keep/drop)
+* `filter_regex` - attribute determines if the `filter_value` shall be treated as regular expression or not.
+   If this attribute is not empty, the bot uses python's "search" function to evaluate the filter.
 
-Parameters for time based filtering:
-* not_before - events before this time will be dropped
-* not_after - events after this time will be dropped
+## Parameters for time based filtering:
+* `not_before` - events before this time will be dropped
+* `not_after` - events after this time will be dropped
 
 Both parameters accept string values describing absolute or relative time:
 * absolute
  * basically anything parsable by datetime parser, eg. "2015-09-012T06:22:11+00:00"
- * time.source taken from the event will be compared to this value to decide the filter behavior
+ * `time.source` taken from the event will be compared to this value to decide the filter behavior
 * relative
  * accepted string formatted like this "<integer> <epoch>", where epoch could be any of following strings (could optionally end with trailing 's'): hour, day, week, month, year
  * time.source taken from the event will be compared to the value (now - relative) to decide the filter behavior

intelmq/bots/experts/filter/expert.py

@@ -67,6 +67,10 @@ def init(self):
             self.logger.info("Filter_action parameter definition unknown.")
             self.filter = False
 
+        self.use_regex = False
+        if hasattr(self.parameters, 'filter_regex') and self.parameters.filter_regex:
+            self.use_regex = True
+
         if not (self.filter or self.not_after is not None or self.not_before is not None):
             self.logger.error("No relevant filter configuration found, stopping...")
             self.stop()
@@ -102,9 +106,8 @@ def process(self):
 
         # key/value based filtering
         if self.filter and self.parameters.filter_action == "drop":
-            if (event.contains(self.parameters.filter_key) and
-                    event.get(self.parameters.filter_key) ==
-                    self.parameters.filter_value):
+            if self.doFilter(event, self.parameters.filter_key,
+                        self.parameters.filter_value):
                 self.acknowledge_message()
                 return
             else:
@@ -113,9 +116,8 @@ def process(self):
                 return
 
         if self.filter and self.parameters.filter_action == "keep":
-            if (event.contains(self.parameters.filter_key) and
-                    event.get(self.parameters.filter_key) ==
-                    self.parameters.filter_value):
+            if self.doFilter(event, self.parameters.filter_key,
+                        self.parameters.filter_value):
                 self.send_message(event)
                 self.acknowledge_message()
                 return
@@ -126,6 +128,23 @@ def process(self):
         self.send_message(event)
         self.acknowledge_message()
 
+    def doFilter(self, event, key, condition):
+        if self.use_regex:
+            return self.regexSearchFilter(event, key, condition)
+        else:
+            return self.equalsFilter(event, key, condition)
+
+    def equalsFilter(self, event, key, value):
+        return (event.contains(key) and
+                event.get(key) == value)
+
+    def regexSearchFilter(self, event, key, regex):
+        if event.contains(key):
+            exp = re.compile(regex)
+            return exp.search(str(event.get(key)))
+        else:
+            return False
+
 if __name__ == "__main__":
     bot = FilterExpertBot(sys.argv[1])
     bot.start()

intelmq/tests/bots/experts/filter/test_expert_regex_search.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+import intelmq.lib.test as test
+from intelmq.bots.experts.filter.expert import FilterExpertBot
+
+EXAMPLE_INPUT = {"__type": "Event",
+                 "classification.type": "defacement",
+                 "time.source": "2005-01-01T00:00:00+00:00",
+                 "time.observation": "2015-09-12T00:00:00+00:00",
+                 "feed.name": "test-feed",
+                 "raw": "fds56gf4jh4jhgh4j6"
+                 }
+
+
+class TestFilterExpertBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for FilterExpertBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = FilterExpertBot
+        cls.input_message = EXAMPLE_INPUT
+        cls.sysconfig = {'filter_regex': 'search',
+                         'filter_key': 'feed.name',
+                         'filter_value': 'feed'}
+
+    def test_searchRegex(self):
+        self.run_bot()
+        self.assertMessageEqual(0, EXAMPLE_INPUT)
+
+if __name__ == '__main__':
+    unittest.main()