├── README.md └── lambda ├── README.md ├── helper.py ├── paperwithcode_lambdas_script.py ├── scholarly_lambda_script.py └── test_lambda_script.py /README.md: -------------------------------------------------------------------------------- 1 | # ML SEARCH 2 | 3 | -------------------------------------------------------------------------------- /lambda/README.md: -------------------------------------------------------------------------------- 1 | # Input Test 2 | 3 | ``` 4 | { 5 | "category": "code", 6 | "query": "tensorflow", 7 | "page": 0 8 | } 9 | ``` 10 | 11 | At the test_lambda_script, only `query` is taken into account 12 | 13 |
14 | ## APIs for ML Search 15 | 16 | 1. For Paper With Code 17 | ``` 18 | curl --header 'x-api-key:YOURAPIKEY' \ 19 | -H "Content-Type: application/json" -X POST \ 20 | -d '{ 21 | "query": "LSTM", 22 | "init_idx": 0, 23 | "count": 30 24 | }' \ 25 | https://hjmlipdub8.execute-api.us-east-2.amazonaws.com/development/paperwithcode 26 | ``` 27 | 28 | 2. For Scholarly 29 | 30 | ``` 31 | curl --header 'x-api-key:YOURAPIKEY' \ 32 | -H "Content-Type: application/json" -X POST \ 33 | -d '{ 34 | "query": "LSTM", 35 | "init_idx": 0, 36 | "count": 30 37 | }' \ 38 | https://hjmlipdub8.execute-api.us-east-2.amazonaws.com/development/scholarly 39 | ``` 40 | Expected Headers: 41 | - x-api-key - The API authentication key for accessing the APIs. 42 | - Content-Type - applicatin/json is expected. 43 | 44 | Expected Body: 45 | - query - The key word for searching. 46 | - init_idx - The initial result index. Use for pagination. 47 | - count - Total number of query result to be fetched. 48 | 49 | Note: For Scholarly, due to the performance issue, only 30 max request could be done per single keyword. -------------------------------------------------------------------------------- /lambda/helper.py: -------------------------------------------------------------------------------- 1 | def is_valid_parameters(event, param_names): 2 | """ 3 | Check whether the item in param_names exist in event dictionary. 4 | 5 | :param event: Lambda event object. 6 | :param param_names: The list of the param names to be checked. 7 | 8 | :retrun: True if exist else False 9 | """ 10 | for param in param_names: 11 | if not param in event: 12 | return False 13 | return True 14 | 15 | def response(message, status_code): 16 | """ 17 | Response message for the request. 18 | 19 | :param message: The response message. 20 | :param status_code: The response status. 21 | 22 | :return: The dic('statusCode', 'body') 23 | """ 24 | return { 25 | 'statusCode': status_code, 26 | 'body': message 27 | } 28 | 29 | def parse_parameters(event): 30 | """ 31 | Parse the parameters from event dictionary. 32 | 33 | :param event: The event dictionary. 34 | :return: dict('query', 'init_idx', 'count') 35 | """ 36 | try: 37 | param = dict() 38 | param['query'] = event['query'] 39 | param['init_idx'] = int(event['init_idx']) 40 | param['count'] = int(event['count']) 41 | 42 | if param['init_idx'] >= 0 and param['count'] > 0: 43 | return param 44 | else: 45 | return dict() 46 | 47 | except: 48 | return dict() -------------------------------------------------------------------------------- /lambda/paperwithcode_lambdas_script.py: -------------------------------------------------------------------------------- 1 | import json 2 | from botocore.vendored import requests 3 | from botocore.vendored.requests.auth import HTTPBasicAuth 4 | import helper as hp 5 | 6 | def lambda_handler(event, context): 7 | try: 8 | param_names = ['query', 'init_idx', 'count'] 9 | response_msg = hp.response('success', 200) 10 | 11 | if hp.is_valid_parameters(event, param_names): 12 | params = hp.parse_parameters(event) 13 | if params.values(): 14 | auth_info = { 15 | 'user': 'xxxx', 16 | 'password': 'xxxx' 17 | } 18 | url = f"https://paperswithcode.com/api/v0/search/?q={params['query']}" 19 | query_result = requests.get( 20 | url, 21 | auth=HTTPBasicAuth(auth_info['user'], auth_info['password']) 22 | ) 23 | 24 | if query_result.status_code == 200: 25 | content = json.loads(query_result.content) 26 | content = content[params['init_idx']:params['init_idx'] + params['count']] 27 | response_msg = hp.response(content, query_result.status_code) 28 | return response_msg 29 | 30 | response_msg = hp.response('Invalid parameters.', 400) 31 | return response_msg 32 | except Exception as ex: 33 | response_msg = hp.response(str(ex), 500) 34 | return response_msg -------------------------------------------------------------------------------- /lambda/scholarly_lambda_script.py: -------------------------------------------------------------------------------- 1 | import json 2 | import scholarly 3 | import helper as hp 4 | 5 | def lambda_handler(event, context): 6 | try: 7 | param_names = ['query', 'init_idx', 'count'] 8 | response_msg = hp.response('success', 200) 9 | 10 | if hp.is_valid_parameters(event, param_names): 11 | params = hp.parse_parameters(event) 12 | if params.values(): 13 | result = [] 14 | iters = scholarly.search_pubs_query(params['query']) 15 | for _ in range(params['init_idx'] + params['count']): 16 | result.append(next(iters).__dict__) 17 | result = result[params['init_idx']:] 18 | response_msg = hp.response(result, 200) 19 | return response_msg 20 | 21 | response_msg = hp.response('Invalid parameters.', 400) 22 | return response_msg 23 | except Exception as ex: 24 | response_msg = hp.response(str(ex), 500) 25 | return response_msg -------------------------------------------------------------------------------- /lambda/test_lambda_script.py: -------------------------------------------------------------------------------- 1 | import json 2 | from botocore.vendored import requests 3 | 4 | def lambda_handler(event, context): 5 | query = event['query'] 6 | query+='+language:python' 7 | params = ( 8 | ('q', query), 9 | ('sort', 'stars'), 10 | ('order', 'desc'), 11 | ) 12 | 13 | response = requests.get('https://api.github.com/search/repositories', params=params) 14 | 15 | return response.json() --------------------------------------------------------------------------------