Skip to content
Snippets Groups Projects
Commit e423e4b8 authored by Thomas Tanon's avatar Thomas Tanon
Browse files

Merge pull request #10 from ProjetPP/add-recursion

I can haz recursion
parents 22a8c46d b41e254e
No related branches found
No related tags found
No related merge requests found
......@@ -43,12 +43,13 @@ class Config:
cls.config_path_variable)
return path
class CoreConfig(Config):
__slots__ = ('modules')
__slots__ = ('modules', 'nb_passes')
config_path_variable = 'PPP_CORE_CONFIG'
def parse_config(self, data):
self.modules = self._parse_modules(data.get('modules', {}))
self.debug = data.get('debug', False)
self.nb_passes = data.get('recursion', {}).get('max_passes', 10)
def _parse_modules(self, data):
modules = []
......
......@@ -122,5 +122,4 @@ class HttpRequestHandler:
def app(environ, start_response):
"""Function called by the WSGI server."""
r = HttpRequestHandler(environ, start_response, Router).dispatch()
print(repr(r))
return r
......@@ -13,6 +13,17 @@ from .exceptions import ClientError, BadGateway
s = lambda x:x if isinstance(x, str) else x.decode()
DEFAULT_ACCURACY = 0
DEFAULT_RELEVANCE = 0
def answer_id(answer):
return (answer.language, answer.tree,
frozenset(answer.measures.items()),
answer.tree)
def remove_duplicates(reference, new):
return filter(lambda x:answer_id(x) not in reference, new)
class Router:
def __init__(self, request):
self.id = request.id
......@@ -24,23 +35,43 @@ class Router:
self.config = CoreConfig()
def answer(self):
answer_ids = set()
answers = []
new_answers = [Response(self.language, self.tree,
self.measures, self.trace)]
for i in range(0, self.config.nb_passes):
# Perform the pass
requests = list(map(self.request_from_answer, new_answers))
new_answers = []
for request in requests:
new_answers.extend(self.one_pass(request))
# Remove duplicates, and update the answer list
new_answers = list(remove_duplicates(answers, new_answers))
answers.extend(new_answers)
answer_ids.update(map(answer_id, new_answers))
# TODO: should sort according to accuracy too
return sorted(answers,
key=lambda x:x.measures.get('relevance', DEFAULT_RELEVANCE),
reverse=True)
def request_from_answer(self, answer):
return Request(self.id, answer.language, answer.tree,
answer.measures, answer.trace)
def one_pass(self, request):
# First make all requests so modules can prepare their answer
# while we send requests to other modules
streams = self._get_streams()
streams = self._get_streams(request)
answers = map(self._stream_reader, streams)
answers = map(self._process_answers, answers)
answers = itertools.chain(*list(answers)) # Flatten answers lists
answers = filter(bool, answers) # Eliminate None values
# TODO: should sort according to accuracy too
return sorted(answers, key=lambda x:x.measures['relevance'],
reverse=True)
return answers
def _get_streams(self):
def _get_streams(self, request):
headers = {'Content-type': 'application/json',
'Accept': 'application/json'}
payload = Request(self.id, self.language,
self.tree,
self.measures, self.trace).as_json()
payload = request.as_json()
getter = functools.partial(requests.post, stream=True,
headers=headers, data=payload)
streams = []
......@@ -75,8 +106,8 @@ class Router:
if missing:
logging.warning('Missing mandatory measures from module %s: %r' %
(module, missing))
accuracy = answer.measures.get('accuracy', 0)
relevance = answer.measures.get('relevance', 0)
accuracy = answer.measures.get('accuracy', DEFAULT_ACCURACY)
relevance = answer.measures.get('relevance', DEFAULT_RELEVANCE)
if accuracy < 0 or accuracy > 1:
logging.warning('Module %s answered with invalid accuracy: %r' %
(module, accuracy))
......
......@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
setup(
name='ppp_core',
version='0.5.1',
version='0.5.2',
description='Core/router of the PPP framework. Also contains a library ' \
'usable by module developpers to handle the query API.',
url='https://github.com/ProjetPP/PPP-Core',
......@@ -27,7 +27,7 @@ setup(
],
install_requires=[
'requests',
'ppp_datamodel>=0.5.1,<0.6',
'ppp_datamodel>=0.5.2,<0.6',
],
packages=[
'ppp_core',
......
......@@ -17,7 +17,10 @@ one_module_config = """
"url": "http://test/my_module/",
"coefficient": 1
}
]
],
"recursion": {
"max_passes": 1
}
}"""
three_modules_config = """
......@@ -39,7 +42,10 @@ three_modules_config = """
"url": "http://test/my_module3/",
"coefficient": 1
}
]
],
"recursion": {
"max_passes": 1
}
}"""
one_valid_module_config = """
......@@ -56,7 +62,10 @@ one_valid_module_config = """
"url": "http://test/my_module4/",
"coefficient": 1
}
]
],
"recursion": {
"max_passes": 1
}
}"""
@urlmatch(netloc='test', path='/my_module/')
......
import json
from httmock import urlmatch, HTTMock, with_httmock, all_requests
from ppp_datamodel import Resource
from ppp_datamodel.communication import Request, TraceItem, Response
from ppp_core.tests import PPPTestCase
from ppp_core import app
config = """
{
"debug": true,
"modules": [
{
"name": "my_module",
"url": "http://test/my_module/",
"coefficient": 1
},
{
"name": "my_module2",
"url": "http://test/my_module2/",
"coefficient": 1
}
]
}"""
@urlmatch(netloc='test', path='/my_module/')
def my_module_mock(url, request):
r = Request.from_json(request.body)
if r.tree == Resource('one'):
c = '"measures": {"accuracy": 1, "relevance": 1}, "tree": {"type": "resource", "value": "two"}'
return {'status_code': 200,
'content': '[{"language": "en", %s, '
'"trace": [{"module": "module1", %s}]}]' %
(c, c)}
else:
return {'status_code': 200,
'content': '[]'}
@urlmatch(netloc='test', path='/my_module2/')
def my_module2_mock(url, request):
r = Request.from_json(request.body)
if r.tree == Resource('two'):
c = '"measures": {"accuracy": 1, "relevance": 2}, "tree": {"type": "resource", "value": "three"}'
return {'status_code': 200,
'content': '[{"language": "en", %s, '
'"trace": [{"module": "module1", %s}]}]' %
(c, c)}
else:
return {'status_code': 200,
'content': '[]'}
class TestRecursion(PPPTestCase(app)):
def testRecursion(self):
self.config_file.write(config)
self.config_file.seek(0)
q = Request('1', 'en', Resource('one'), {}, [])
with HTTMock(my_module_mock, my_module2_mock):
answers = self.request(q)
self.assertEqual(len(answers), 2, answers)
self.assertEqual(answers[0].tree, Resource('three'))
self.assertEqual(answers[1].tree, Resource('two'))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment