improved performance of rule matching

README.md

@@ -0,0 +1,5 @@
+### Approach
+Rule engine supports only simple comparison operators. The trade-off of this approach is that you can't encode complex rules. To be able to support more robust rules we could embed a scripting languages like _lua_, possibly at the cost of performance bottlenecks.
+
+### Performance
+for 200*4=800 sig and 3 rules this python code takes about 0.04 secs on a AMD6410

quartic_rule.py

@@ -1,5 +1,7 @@
 import json
 import datetime
+# import ijson
+# import ijson.backends.yajl2_cffi as ijson
 
 class Rule(object):
     """docstring for Rule."""
@@ -22,15 +24,14 @@ class Rule(object):
         return not x if self.neg else x
 
     def apply(self,signal):
-        if signal.signal == self.pattern:
+        if signal.value_type != self.value_type:
-            if signal.value_type != self.value_type:
+            return True
-                return True
+        if self.op == ">":
-            if self.op == ">":
+            return self.apply_neg(signal.value > self.const)
-                return self.apply_neg(signal.value > self.const)
+        if self.op == "<":
-            if self.op == "<":
+            return self.apply_neg(signal.value < self.const)
-                return self.apply_neg(signal.value < self.const)
+        if self.op == "=":
-            if self.op == "=":
+            return self.apply_neg(signal.value == self.const)
-                return self.apply_neg(signal.value == self.const)
         return True
 
     @classmethod
@@ -39,10 +40,17 @@ class Rule(object):
 
     @staticmethod
     def rule_engine_gen(rule_list):
+        pattern_dict = {}
+        for r in rule_list:
+            if not pattern_dict.has_key(r.pattern):
+                pattern_dict[r.pattern] = [r]
+            else:
+                pattern_dict[r.pattern].append(r)
         def check_rules(signal):
-            for rule in rule_list:
+            if pattern_dict.has_key(signal.signal):
-                if not rule.apply(signal):
+                for rule in pattern_dict[signal.signal]:
-                    return (False,rule)
+                    if not rule.apply(signal):
+                        return (False,rule)
             return (True,None)
         return check_rules
 
@@ -87,6 +95,10 @@ def load_sigs(filename):
     with open(filename,"r") as sig_file:
         return json.load(sig_file)
 
+# def load_isigs(filename):
+#     with open(filename, 'r') as fd:
+#         return [o for o in ijson.items(fd,'item')]
+
 def validate_sigs(signals,rule_engine):
     invalids = []
     index = 0

quartic_rule_test.py

@@ -6,7 +6,6 @@ class QuarticTest(unittest.TestCase):
 
     def setUp(self):
         self.signals = quartic_rule.load_sigs("./raw_data.json")
-        pass
 
     def test_rules(self):
         check_rules = quartic_rule.Rule.read_rules("./rules.txt")