├── README.md
└── lambda
├── README.md
├── helper.py
├── paperwithcode_lambdas_script.py
├── scholarly_lambda_script.py
└── test_lambda_script.py
/README.md:
--------------------------------------------------------------------------------
1 | # ML SEARCH
2 |
3 |
--------------------------------------------------------------------------------
/lambda/README.md:
--------------------------------------------------------------------------------
1 | # Input Test
2 |
3 | ```
4 | {
5 | "category": "code",
6 | "query": "tensorflow",
7 | "page": 0
8 | }
9 | ```
10 |
11 | At the test_lambda_script, only `query` is taken into account
12 |
13 |
14 | ## APIs for ML Search
15 |
16 | 1. For Paper With Code
17 | ```
18 | curl --header 'x-api-key:YOURAPIKEY' \
19 | -H "Content-Type: application/json" -X POST \
20 | -d '{
21 | "query": "LSTM",
22 | "init_idx": 0,
23 | "count": 30
24 | }' \
25 | https://hjmlipdub8.execute-api.us-east-2.amazonaws.com/development/paperwithcode
26 | ```
27 |
28 | 2. For Scholarly
29 |
30 | ```
31 | curl --header 'x-api-key:YOURAPIKEY' \
32 | -H "Content-Type: application/json" -X POST \
33 | -d '{
34 | "query": "LSTM",
35 | "init_idx": 0,
36 | "count": 30
37 | }' \
38 | https://hjmlipdub8.execute-api.us-east-2.amazonaws.com/development/scholarly
39 | ```
40 | Expected Headers:
41 | - x-api-key - The API authentication key for accessing the APIs.
42 | - Content-Type - applicatin/json is expected.
43 |
44 | Expected Body:
45 | - query - The key word for searching.
46 | - init_idx - The initial result index. Use for pagination.
47 | - count - Total number of query result to be fetched.
48 |
49 | Note: For Scholarly, due to the performance issue, only 30 max request could be done per single keyword.
--------------------------------------------------------------------------------
/lambda/helper.py:
--------------------------------------------------------------------------------
1 | def is_valid_parameters(event, param_names):
2 | """
3 | Check whether the item in param_names exist in event dictionary.
4 |
5 | :param event: Lambda event object.
6 | :param param_names: The list of the param names to be checked.
7 |
8 | :retrun: True if exist else False
9 | """
10 | for param in param_names:
11 | if not param in event:
12 | return False
13 | return True
14 |
15 | def response(message, status_code):
16 | """
17 | Response message for the request.
18 |
19 | :param message: The response message.
20 | :param status_code: The response status.
21 |
22 | :return: The dic('statusCode', 'body')
23 | """
24 | return {
25 | 'statusCode': status_code,
26 | 'body': message
27 | }
28 |
29 | def parse_parameters(event):
30 | """
31 | Parse the parameters from event dictionary.
32 |
33 | :param event: The event dictionary.
34 | :return: dict('query', 'init_idx', 'count')
35 | """
36 | try:
37 | param = dict()
38 | param['query'] = event['query']
39 | param['init_idx'] = int(event['init_idx'])
40 | param['count'] = int(event['count'])
41 |
42 | if param['init_idx'] >= 0 and param['count'] > 0:
43 | return param
44 | else:
45 | return dict()
46 |
47 | except:
48 | return dict()
--------------------------------------------------------------------------------
/lambda/paperwithcode_lambdas_script.py:
--------------------------------------------------------------------------------
1 | import json
2 | from botocore.vendored import requests
3 | from botocore.vendored.requests.auth import HTTPBasicAuth
4 | import helper as hp
5 |
6 | def lambda_handler(event, context):
7 | try:
8 | param_names = ['query', 'init_idx', 'count']
9 | response_msg = hp.response('success', 200)
10 |
11 | if hp.is_valid_parameters(event, param_names):
12 | params = hp.parse_parameters(event)
13 | if params.values():
14 | auth_info = {
15 | 'user': 'xxxx',
16 | 'password': 'xxxx'
17 | }
18 | url = f"https://paperswithcode.com/api/v0/search/?q={params['query']}"
19 | query_result = requests.get(
20 | url,
21 | auth=HTTPBasicAuth(auth_info['user'], auth_info['password'])
22 | )
23 |
24 | if query_result.status_code == 200:
25 | content = json.loads(query_result.content)
26 | content = content[params['init_idx']:params['init_idx'] + params['count']]
27 | response_msg = hp.response(content, query_result.status_code)
28 | return response_msg
29 |
30 | response_msg = hp.response('Invalid parameters.', 400)
31 | return response_msg
32 | except Exception as ex:
33 | response_msg = hp.response(str(ex), 500)
34 | return response_msg
--------------------------------------------------------------------------------
/lambda/scholarly_lambda_script.py:
--------------------------------------------------------------------------------
1 | import json
2 | import scholarly
3 | import helper as hp
4 |
5 | def lambda_handler(event, context):
6 | try:
7 | param_names = ['query', 'init_idx', 'count']
8 | response_msg = hp.response('success', 200)
9 |
10 | if hp.is_valid_parameters(event, param_names):
11 | params = hp.parse_parameters(event)
12 | if params.values():
13 | result = []
14 | iters = scholarly.search_pubs_query(params['query'])
15 | for _ in range(params['init_idx'] + params['count']):
16 | result.append(next(iters).__dict__)
17 | result = result[params['init_idx']:]
18 | response_msg = hp.response(result, 200)
19 | return response_msg
20 |
21 | response_msg = hp.response('Invalid parameters.', 400)
22 | return response_msg
23 | except Exception as ex:
24 | response_msg = hp.response(str(ex), 500)
25 | return response_msg
--------------------------------------------------------------------------------
/lambda/test_lambda_script.py:
--------------------------------------------------------------------------------
1 | import json
2 | from botocore.vendored import requests
3 |
4 | def lambda_handler(event, context):
5 | query = event['query']
6 | query+='+language:python'
7 | params = (
8 | ('q', query),
9 | ('sort', 'stars'),
10 | ('order', 'desc'),
11 | )
12 |
13 | response = requests.get('https://api.github.com/search/repositories', params=params)
14 |
15 | return response.json()
--------------------------------------------------------------------------------