Merge pull request #10 from ProjetPP/add-recursion

I can haz recursion

ppp_core/config.py

@@ -43,12 +43,13 @@ class Config:
                                 cls.config_path_variable)
         return path
 class CoreConfig(Config):
-    __slots__ = ('modules')
+    __slots__ = ('modules', 'nb_passes')
     config_path_variable = 'PPP_CORE_CONFIG'
 
     def parse_config(self, data):
         self.modules = self._parse_modules(data.get('modules', {}))
         self.debug = data.get('debug', False)
+        self.nb_passes = data.get('recursion', {}).get('max_passes', 10)
 
     def _parse_modules(self, data):
         modules = []

ppp_core/http.py

@@ -122,5 +122,4 @@ class HttpRequestHandler:
 def app(environ, start_response):
     """Function called by the WSGI server."""
     r = HttpRequestHandler(environ, start_response, Router).dispatch()
-    print(repr(r))
     return r

ppp_core/router.py

@@ -13,6 +13,17 @@ from .exceptions import ClientError, BadGateway
 
 s = lambda x:x if isinstance(x, str) else x.decode()
 
+DEFAULT_ACCURACY = 0
+DEFAULT_RELEVANCE = 0
+
+def answer_id(answer):
+    return (answer.language, answer.tree,
+            frozenset(answer.measures.items()),
+            answer.tree)
+def remove_duplicates(reference, new):
+    return filter(lambda x:answer_id(x) not in reference, new)
+
+
 class Router:
     def __init__(self, request):
         self.id = request.id
@@ -24,23 +35,43 @@ class Router:
         self.config = CoreConfig()
 
     def answer(self):
+        answer_ids = set()
+        answers = []
+        new_answers = [Response(self.language, self.tree,
+                                self.measures, self.trace)]
+        for i in range(0, self.config.nb_passes):
+            # Perform the pass
+            requests = list(map(self.request_from_answer, new_answers))
+            new_answers = []
+            for request in requests:
+                new_answers.extend(self.one_pass(request))
+            # Remove duplicates, and update the answer list
+            new_answers = list(remove_duplicates(answers, new_answers))
+            answers.extend(new_answers)
+            answer_ids.update(map(answer_id, new_answers))
+        # TODO: should sort according to accuracy too
+        return sorted(answers,
+                      key=lambda x:x.measures.get('relevance', DEFAULT_RELEVANCE),
+                      reverse=True)
+
+    def request_from_answer(self, answer):
+        return Request(self.id, answer.language, answer.tree,
+                       answer.measures, answer.trace)
+
+    def one_pass(self, request):
         # First make all requests so modules can prepare their answer
         # while we send requests to other modules
-        streams = self._get_streams()
+        streams = self._get_streams(request)
         answers = map(self._stream_reader, streams)
         answers = map(self._process_answers, answers)
         answers = itertools.chain(*list(answers)) # Flatten answers lists
         answers = filter(bool, answers) # Eliminate None values
-        # TODO: should sort according to accuracy too
-        return sorted(answers, key=lambda x:x.measures['relevance'],
-                      reverse=True)
+        return answers
 
-    def _get_streams(self):
+    def _get_streams(self, request):
         headers = {'Content-type': 'application/json',
                    'Accept': 'application/json'}
-        payload = Request(self.id, self.language,
-                          self.tree,
-                          self.measures, self.trace).as_json()
+        payload = request.as_json()
         getter = functools.partial(requests.post, stream=True,
                                    headers=headers, data=payload)
         streams = []
@@ -75,8 +106,8 @@ class Router:
         if missing:
             logging.warning('Missing mandatory measures from module %s: %r' %
                              (module, missing))
-        accuracy = answer.measures.get('accuracy', 0)
-        relevance = answer.measures.get('relevance', 0)
+        accuracy = answer.measures.get('accuracy', DEFAULT_ACCURACY)
+        relevance = answer.measures.get('relevance', DEFAULT_RELEVANCE)
         if accuracy < 0 or accuracy > 1:
             logging.warning('Module %s answered with invalid accuracy: %r' %
                     (module, accuracy))

setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='ppp_core',
-    version='0.5.1',
+    version='0.5.2',
     description='Core/router of the PPP framework. Also contains a library ' \
                 'usable by module developpers to handle the query API.',
     url='https://github.com/ProjetPP/PPP-Core',
@@ -27,7 +27,7 @@ setup(
     ],
     install_requires=[
         'requests',
-        'ppp_datamodel>=0.5.1,<0.6',
+        'ppp_datamodel>=0.5.2,<0.6',
     ],
     packages=[
         'ppp_core',

tests/test_call_module.py → tests/test_no_recursion.py

@@ -17,7 +17,10 @@ one_module_config = """
             "url": "http://test/my_module/",
             "coefficient": 1
         }
-    ]
+    ],
+    "recursion": {
+        "max_passes": 1
+    }
 }"""
 
 three_modules_config = """
@@ -39,7 +42,10 @@ three_modules_config = """
             "url": "http://test/my_module3/",
             "coefficient": 1
         }
-    ]
+    ],
+    "recursion": {
+        "max_passes": 1
+    }
 }"""
 
 one_valid_module_config = """
@@ -56,7 +62,10 @@ one_valid_module_config = """
             "url": "http://test/my_module4/",
             "coefficient": 1
         }
-    ]
+    ],
+    "recursion": {
+        "max_passes": 1
+    }
 }"""
 
 @urlmatch(netloc='test', path='/my_module/')

tests/test_recursion.py

+import json
+from httmock import urlmatch, HTTMock, with_httmock, all_requests
+
+from ppp_datamodel import Resource
+from ppp_datamodel.communication import Request, TraceItem, Response
+from ppp_core.tests import PPPTestCase
+from ppp_core import app
+
+config = """
+{
+    "debug": true,
+    "modules": [
+        {
+            "name": "my_module",
+            "url": "http://test/my_module/",
+            "coefficient": 1
+        },
+        {
+            "name": "my_module2",
+            "url": "http://test/my_module2/",
+            "coefficient": 1
+        }
+    ]
+}"""
+
+@urlmatch(netloc='test', path='/my_module/')
+def my_module_mock(url, request):
+    r = Request.from_json(request.body)
+    if r.tree == Resource('one'):
+        c = '"measures": {"accuracy": 1, "relevance": 1}, "tree": {"type": "resource", "value": "two"}'
+        return {'status_code': 200,
+                'content': '[{"language": "en", %s, '
+                             '"trace": [{"module": "module1", %s}]}]' %
+                             (c, c)}
+    else:
+        return {'status_code': 200,
+                'content': '[]'}
+
+@urlmatch(netloc='test', path='/my_module2/')
+def my_module2_mock(url, request):
+    r = Request.from_json(request.body)
+    if r.tree == Resource('two'):
+        c = '"measures": {"accuracy": 1, "relevance": 2}, "tree": {"type": "resource", "value": "three"}'
+        return {'status_code': 200,
+                'content': '[{"language": "en", %s, '
+                             '"trace": [{"module": "module1", %s}]}]' %
+                             (c, c)}
+    else:
+        return {'status_code': 200,
+                'content': '[]'}
+
+class TestRecursion(PPPTestCase(app)):
+    def testRecursion(self):
+        self.config_file.write(config)
+        self.config_file.seek(0)
+        q = Request('1', 'en', Resource('one'), {}, [])
+        with HTTMock(my_module_mock, my_module2_mock):
+            answers = self.request(q)
+            self.assertEqual(len(answers), 2, answers)
+            self.assertEqual(answers[0].tree, Resource('three'))
+            self.assertEqual(answers[1].tree, Resource('two'))