improved performance of rule matching

README.md

@@ -0,0 +1,5 @@
+### Approach
+Rule engine supports only simple comparison operators. The trade-off of this approach is that you can't encode complex rules. To be able to support more robust rules we could embed a scripting languages like _lua_, possibly at the cost of performance bottlenecks.
+
+### Performance
+for 200*4=800 sig and 3 rules this python code takes about 0.04 secs on a AMD6410

quartic_rule.py

@@ -1,5 +1,7 @@
 import json
 import datetime
+# import ijson
+# import ijson.backends.yajl2_cffi as ijson
 
 class Rule(object):
     """docstring for Rule."""
@@ -22,7 +24,6 @@ class Rule(object):
         return not x if self.neg else x
 
     def apply(self,signal):
-        if signal.signal == self.pattern:
         if signal.value_type != self.value_type:
             return True
         if self.op == ">":
@@ -39,8 +40,15 @@ class Rule(object):
 
     @staticmethod
     def rule_engine_gen(rule_list):
+        pattern_dict = {}
+        for r in rule_list:
+            if not pattern_dict.has_key(r.pattern):
+                pattern_dict[r.pattern] = [r]
+            else:
+                pattern_dict[r.pattern].append(r)
         def check_rules(signal):
-            for rule in rule_list:
+            if pattern_dict.has_key(signal.signal):
+                for rule in pattern_dict[signal.signal]:
                     if not rule.apply(signal):
                         return (False,rule)
             return (True,None)
@@ -87,6 +95,10 @@ def load_sigs(filename):
     with open(filename,"r") as sig_file:
         return json.load(sig_file)
 
+# def load_isigs(filename):
+#     with open(filename, 'r') as fd:
+#         return [o for o in ijson.items(fd,'item')]
+
 def validate_sigs(signals,rule_engine):
     invalids = []
     index = 0

quartic_rule_test.py

@@ -6,7 +6,6 @@ class QuarticTest(unittest.TestCase):
 
     def setUp(self):
         self.signals = quartic_rule.load_sigs("./raw_data.json")
-        pass
 
     def test_rules(self):
         check_rules = quartic_rule.Rule.read_rules("./rules.txt")