├── api ├── __init__.py ├── v1 │ ├── models │ │ ├── __init__.py │ │ ├── quarters.py │ │ ├── geographies.py │ │ ├── title_count.py │ │ ├── skills_related.py │ │ ├── jobs_skills.py │ │ ├── jobs_importance.py │ │ ├── geo_title_count.py │ │ ├── jobs_alternate_titles.py │ │ ├── skills_importance.py │ │ ├── jobs_unusual_titles.py │ │ ├── skills_master.py │ │ └── jobs_master.py │ ├── __init__.py │ └── endpoints.py └── router │ ├── __init__.py │ └── endpoints.py ├── app ├── __init__.py └── app.py ├── common ├── __init__.py └── utils.py ├── tests ├── __init__.py ├── es_nohit_output.json ├── api_test_case.py ├── factories.py ├── es_normalize_output.json ├── job_normalize_test.py └── job_get_test.py ├── tox.ini ├── .gitignore ├── setup.py ├── openskills.wsgi ├── server.py ├── bin ├── upgrade_pip.sh ├── clean.sh ├── switch_aws_config.sh ├── make_config.sh └── make_zappa_config.sh ├── requirements.txt ├── LICENSE.txt ├── README.md └── skills-api.json /api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /api/v1/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27 3 | [testenv] 4 | deps= 5 | pytest 6 | -rrequirements.txt 7 | commands=py.test -vvv -s tests/ 8 | -------------------------------------------------------------------------------- /tests/es_nohit_output.json: -------------------------------------------------------------------------------- 1 | {"took":86,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":0,"max_score":null,"hits":[]}} 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .python-version 3 | venv/ 4 | config/ 5 | zappa_settings.json 6 | create-login-profile.json 7 | tmp/ 8 | config.* 9 | *.zip 10 | *.sql 11 | *.tsv 12 | *.csv 13 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | 5 | setup( 6 | name='Skills API', 7 | version='1.0', 8 | description='Skills API', 9 | packages=find_packages(), 10 | ) 11 | -------------------------------------------------------------------------------- /openskills.wsgi: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('/var/www/skills-api') 3 | activate_this = '/var/www/skills-api/venv/bin/activate_this.py' 4 | execfile(activate_this, dict(__file__=activate_this)) 5 | 6 | from app.app import app as application 7 | -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | server 5 | ~~~~~~ 6 | 7 | Local web server for the Open Data API. 8 | 9 | """ 10 | 11 | import os 12 | 13 | from app.app import app 14 | from flask_script import Manager 15 | 16 | manager = Manager(app) 17 | 18 | if __name__ == '__main__': 19 | manager.run() 20 | -------------------------------------------------------------------------------- /bin/upgrade_pip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # --------------------------------------------------------------------------- 4 | # upgrade_pip - Utility for upgrading the Python pip package manager. 5 | # 6 | # Usage: ./upgrade_pip.sh 7 | # --------------------------------------------------------------------------- 8 | 9 | pip freeze --local | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 pip install -U 10 | -------------------------------------------------------------------------------- /api/v1/models/quarters.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from app.app import db 4 | 5 | 6 | class Quarter(db.Model): 7 | __tablename__ = 'quarters' 8 | 9 | quarter_id = db.Column(db.SmallInteger, primary_key=True) 10 | year = db.Column(db.Integer, nullable=False) 11 | quarter = db.Column(db.Integer, nullable=False) 12 | 13 | def __repr__(self): 14 | return ''.format( 15 | self.year, self.quarter 16 | ) 17 | -------------------------------------------------------------------------------- /api/v1/models/geographies.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from app.app import db 4 | 5 | 6 | class Geography(db.Model): 7 | __tablename__ = 'geographies' 8 | 9 | geography_id = db.Column(db.SmallInteger, primary_key=True) 10 | geography_type = db.Column(db.String, nullable=False) 11 | geography_name = db.Column(db.String, nullable=False) 12 | 13 | def __repr__(self): 14 | return ''.format( 15 | self.geography_type, self.geography_name 16 | ) 17 | -------------------------------------------------------------------------------- /bin/clean.sh: -------------------------------------------------------------------------------- 1 | # ---------------------------------------------------------------------------- 2 | # clean - General purpose cleanup utility for removing development crud. 3 | # 4 | # Usage: ./clean.sh 5 | # ---------------------------------------------------------------------------- 6 | 7 | SCRIPT_NAME=`basename "$0"` 8 | HERE=`pwd` 9 | 10 | if [ -f $SCRIPT_NAME ]; then 11 | cd .. 12 | fi 13 | 14 | echo -ne "Cleaning Work Directories........." 15 | find . -name "*.pyc" -exec rm -rf {} \; 16 | cd $HERE 17 | echo "Done" 18 | -------------------------------------------------------------------------------- /tests/api_test_case.py: -------------------------------------------------------------------------------- 1 | from app import app 2 | import unittest 3 | from flask_migrate import upgrade 4 | 5 | 6 | class ApiTestCase(unittest.TestCase): 7 | def setUp(self): 8 | app.app.config.from_object('config.test_config.Config') 9 | self.app = app.app.test_client() 10 | self.app.testing = True 11 | with app.app.app_context(): 12 | upgrade() 13 | app.db.session.begin(subtransactions=True) 14 | 15 | def tearDown(self): 16 | app.db.session.rollback() 17 | app.db.session.close() 18 | -------------------------------------------------------------------------------- /api/v1/models/title_count.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from app.app import db 4 | 5 | 6 | class TitleCount(db.Model): 7 | __tablename__ = 'title_counts' 8 | 9 | job_uuid = db.Column(db.String, primary_key=True) 10 | quarter_id = db.Column(db.SmallInteger, db.ForeignKey('quarters.quarter_id'), primary_key=True) 11 | job_title = db.Column(db.String) 12 | count = db.Column(db.Integer) 13 | 14 | def __repr__(self): 15 | return ''.format( 16 | self.quarter_id, self.job_uuid, self.count 17 | ) 18 | -------------------------------------------------------------------------------- /api/v1/models/skills_related.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Skills Related ORM""" 4 | 5 | from app.app import db 6 | from sqlalchemy.dialects.postgresql import JSONB 7 | 8 | class SkillRelated(db.Model): 9 | __tablename__ = 'skills_related' 10 | 11 | uuid = db.Column(db.String, primary_key=True) 12 | related_skills = db.Column(JSONB) 13 | 14 | def __init__(self, uuid, related_skills): 15 | self.uuid = uuid 16 | self.related_skills = related_skills 17 | 18 | def __repr__(self): 19 | return ''.format(self.uuid) 20 | -------------------------------------------------------------------------------- /api/v1/models/jobs_skills.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Jobs Skills ORM""" 4 | 5 | from app.app import db 6 | 7 | class JobSkill(db.Model): 8 | __tablename__ = 'jobs_skills' 9 | 10 | job_uuid = db.Column(db.String, db.ForeignKey('jobs_master.uuid'), primary_key=True) 11 | skill_uuid = db.Column(db.String, db.ForeignKey('skills_master.uuid'), primary_key=True) 12 | 13 | def __init__(self, job_uuid, skill_uuid): 14 | self.job_uuid = job_uuid 15 | self.skill_uuid = skill_uuid 16 | 17 | def __repr__(self): 18 | return ''.format(self.job_uuid) 19 | -------------------------------------------------------------------------------- /api/v1/models/jobs_importance.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from app.app import db 4 | 5 | 6 | class JobImportance(db.Model): 7 | __tablename__ = 'jobs_importance' 8 | 9 | quarter_id = db.Column(db.SmallInteger, db.ForeignKey('quarters.quarter_id'), primary_key=True) 10 | geography_id = db.Column(db.SmallInteger, db.ForeignKey('geographies.geography_id'), primary_key=True) 11 | job_uuid = db.Column(db.String, primary_key=True) 12 | importance = db.Column(db.Float) 13 | 14 | def __repr__(self): 15 | return ''.format( 16 | self.geography_id, self.quarter_id, self.job_uuid 17 | ) 18 | -------------------------------------------------------------------------------- /api/v1/models/geo_title_count.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from app.app import db 4 | 5 | 6 | class GeoTitleCount(db.Model): 7 | __tablename__ = 'geo_title_counts' 8 | 9 | quarter_id = db.Column(db.SmallInteger, db.ForeignKey('quarters.quarter_id'), primary_key=True) 10 | geography_id = db.Column(db.SmallInteger, db.ForeignKey('geographies.geography_id'), primary_key=True) 11 | job_uuid = db.Column(db.String, primary_key=True) 12 | job_title = db.Column(db.String) 13 | count = db.Column(db.Integer) 14 | 15 | def __repr__(self): 16 | return ''.format( 17 | self.geography_id, self.quarter_id, self.job_uuid, self.count 18 | ) 19 | -------------------------------------------------------------------------------- /api/v1/models/jobs_alternate_titles.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Alternate Titles ORM""" 4 | 5 | from app.app import db 6 | 7 | class JobAlternateTitle(db.Model): 8 | __tablename__ = 'jobs_alternate_titles' 9 | 10 | uuid = db.Column(db.String, primary_key=True) 11 | title = db.Column(db.String) 12 | nlp_a = db.Column(db.String) 13 | job_uuid = db.Column(db.String, db.ForeignKey('jobs_master.uuid')) 14 | 15 | def __init__(self, uuid, title, nlp_a, job_uuid): 16 | self.uuid = uuid 17 | self.title = title 18 | self.nlp_a = nlp_a 19 | self.job_uuid = job_uuid 20 | 21 | def __repr__(self): 22 | return ''.format(self.uuid) 23 | -------------------------------------------------------------------------------- /api/v1/models/skills_importance.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Skills Importance ORM""" 4 | 5 | from app.app import db 6 | 7 | class SkillImportance(db.Model): 8 | __tablename__ = 'skills_importance' 9 | 10 | job_uuid = db.Column(db.String, db.ForeignKey('jobs_master.uuid'), primary_key=True) 11 | skill_uuid = db.Column(db.String, db.ForeignKey('skills_master.uuid'), primary_key=True) 12 | level = db.Column(db.Float) 13 | importance = db.Column(db.Float) 14 | 15 | 16 | def __init__(self, job_uuid, skill_uuid, level, importance): 17 | self.job_uuid = job_uuid 18 | self.skill_uuid = skill_uuid 19 | self.level = level 20 | self.importance = importance 21 | 22 | def __repr__(self): 23 | return ''.format(self.job_uuid, self.skill_uuid) 24 | -------------------------------------------------------------------------------- /api/v1/models/jobs_unusual_titles.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Jobs Unusual Titles ORM""" 4 | 5 | from app.app import db 6 | 7 | class JobUnusualTitle(db.Model): 8 | __tablename__ = 'jobs_unusual_titles' 9 | 10 | uuid = db.Column(db.String, primary_key=True) 11 | title = db.Column(db.String) 12 | description = db.Column(db.String) 13 | job_uuid = db.Column(db.String, db.ForeignKey('jobs_master.uuid')) 14 | 15 | def __init__(self, uuid, title, description, job_uuid): 16 | self.uuid = uuid 17 | self.title = title 18 | self.description = description 19 | self.job_uuid = job_uuid 20 | 21 | def __repr__(self): 22 | return ''.format(self.uuid) 23 | 24 | def find_all(self): 25 | return self.query.order_by(self.title.asc()).all() 26 | -------------------------------------------------------------------------------- /api/v1/models/skills_master.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Skills Master ORM""" 4 | 5 | from app.app import db 6 | 7 | class SkillMaster(db.Model): 8 | __tablename__ = 'skills_master' 9 | 10 | uuid = db.Column(db.String, primary_key=True) 11 | skill_name = db.Column(db.String) 12 | ksa_type = db.Column(db.String) 13 | onet_element_id = db.Column(db.String) 14 | description = db.Column(db.String) 15 | nlp_a = db.Column(db.String) 16 | 17 | def __init__(self, uuid, skill_name, ksa_type, onet_element_id, description, nlp_a): 18 | self.uuid = uuid 19 | self.skill_name = skill_name 20 | self.ksa_type = ksa_type 21 | self.onet_element_id = onet_element_id 22 | self.description = description 23 | self.nlp_a = nlp_a 24 | 25 | def __repr__(self): 26 | return ''.format(self.uuid) 27 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | alembic==0.8.6 2 | aniso8601==1.1.0 3 | awscli==1.10.47 4 | base58==0.2.3 5 | boto3==1.3.1 6 | botocore==1.4.36 7 | cffi==1.7.0 8 | click==6.6 9 | colorama==0.3.7 10 | cryptography==1.4 11 | docutils==0.12 12 | enum34==1.1.6 13 | Flask==0.11.1 14 | Flask-Cors==2.1.2 15 | Flask-Migrate==1.8.1 16 | Flask-RESTful==0.3.5 17 | Flask-Script==2.0.5 18 | Flask-SQLAlchemy==2.1 19 | Flask-Elasticsearch 20 | factory_boy 21 | futures==3.0.5 22 | HTTPretty 23 | hjson==1.5.6 24 | idna==2.1 25 | ipaddress==1.0.16 26 | itsdangerous==0.24 27 | Jinja2==2.8 28 | jmespath==0.9.0 29 | lambda-packages==0.5.0 30 | Mako==1.0.4 31 | MarkupSafe==0.23 32 | mock 33 | ndg-httpsclient==0.4.1 34 | psycopg2==2.6.1 35 | pyasn1==0.1.9 36 | pycparser==2.14 37 | pyOpenSSL==16.0.0 38 | python-dateutil==2.5.3 39 | python-editor==1.0.1 40 | python-slugify==1.2.0 41 | pytz==2016.6.1 42 | requests==2.10.0 43 | rsa==3.4.2 44 | s3transfer==0.0.1 45 | six==1.10.0 46 | SQLAlchemy==1.0.14 47 | tqdm==4.7.6 48 | Unidecode==0.4.19 49 | Werkzeug==0.11.10 50 | wsgi-request-logger==0.4.5 51 | zappa==0.19.4 52 | -------------------------------------------------------------------------------- /api/v1/models/jobs_master.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Jobs Master ORM""" 4 | 5 | from app.app import db 6 | 7 | class JobMaster(db.Model): 8 | __tablename__ = 'jobs_master' 9 | 10 | uuid = db.Column(db.String, primary_key=True) 11 | onet_soc_code = db.Column(db.String) 12 | title = db.Column(db.String) 13 | original_title = db.Column(db.String) 14 | description = db.Column(db.String) 15 | nlp_a = db.Column(db.String) 16 | alternate_titles = db.relationship('JobAlternateTitle', backref='job', lazy='dynamic') 17 | unusual_titles = db.relationship('JobUnusualTitle', backref='job', lazy='dynamic') 18 | 19 | def __init__(self, uuid, onet_soc_code, title, original_title, description, nlp_a): 20 | self.uuid = uuid 21 | self.onet_soc_code = onet_soc_code 22 | self.title = title 23 | self.original_title = original_title 24 | self.description = description 25 | self.nlp_a = nlp_a 26 | 27 | def __repr__(self): 28 | return ''.format(self.uuid) 29 | 30 | @classmethod 31 | def all_uuids(cls): 32 | return Session.query.order_by(JobMaster.title.asc()).all() -------------------------------------------------------------------------------- /bin/switch_aws_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # --------------------------------------------------------------------------- 4 | # switch-aws-config - Because sometimes you just get tired of changing your 5 | # AWS configs manually... 6 | # 7 | # Usage: ./switch-aws-config.sh [ --development | --testing | --production ] 8 | # --------------------------------------------------------------------------- 9 | 10 | AWS_CONFIG_HOME=$HOME/.aws/ 11 | 12 | if [ -d $AWS_CONFIG_HOME ]; then 13 | HERE=`pwd` 14 | if [ ! -z $1 ]; then 15 | if [ $1 == "--development" ]; then 16 | echo "Switched to Development AWS Credentials" 17 | CONFIG_TYPE="development" 18 | elif [ $1 == "--testing" ]; then 19 | echo "Switched to Testing AWS Credentials" 20 | CONFIG_TYPE="testing" 21 | elif [ $1 == "--production" ]; then 22 | echo "Switched to Production AWS Credentials" 23 | CONFIG_TYPE="production" 24 | else 25 | echo "Invalid Configuration ${1}" 26 | exit 1 27 | fi 28 | else 29 | echo "No Configuration Specified" 30 | exit 1 31 | fi 32 | cd $AWS_CONFIG_HOME 33 | cp credentials.$CONFIG_TYPE credentials 34 | cp config.$CONFIG_TYPE config 35 | fi 36 | -------------------------------------------------------------------------------- /tests/factories.py: -------------------------------------------------------------------------------- 1 | from app import app 2 | from factory import alchemy, LazyAttribute, Sequence 3 | from api.v1 import Geography, Quarter, JobImportance, JobMaster 4 | from faker import Faker 5 | 6 | fake = Faker() 7 | 8 | 9 | class JobMasterFactory(alchemy.SQLAlchemyModelFactory): 10 | class Meta(object): 11 | model = JobMaster 12 | sqlalchemy_session = app.db.session 13 | 14 | uuid = Sequence(lambda n: str(n)) 15 | onet_soc_code = LazyAttribute(lambda x: fake.phone_number()) 16 | title = LazyAttribute(lambda x: fake.job()) 17 | original_title = LazyAttribute(lambda x: fake.job()) 18 | description = LazyAttribute(lambda x: fake.job()) 19 | nlp_a = LazyAttribute(lambda x: fake.job()) 20 | 21 | 22 | class GeographyFactory(alchemy.SQLAlchemyModelFactory): 23 | class Meta(object): 24 | model = Geography 25 | sqlalchemy_session = app.db.session 26 | 27 | geography_id = Sequence(lambda x: x) 28 | geography_type = 'CBSA' 29 | geography_name = LazyAttribute(lambda x: fake.phone_number()) 30 | 31 | 32 | class QuarterFactory(alchemy.SQLAlchemyModelFactory): 33 | class Meta(object): 34 | model = Quarter 35 | sqlalchemy_session = app.db.session 36 | 37 | quarter_id = Sequence(lambda x: x) 38 | quarter = '3' 39 | year = '2016' 40 | 41 | 42 | class JobImportanceFactory(alchemy.SQLAlchemyModelFactory): 43 | class Meta(object): 44 | model = JobImportance 45 | sqlalchemy_session = app.db.session 46 | 47 | importance = 0.2 48 | -------------------------------------------------------------------------------- /api/router/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """API Router Package. 4 | 5 | The router package provides a simple façade for handing versions of the API. 6 | When a version is not specified via the endpoint (e.g. /v1/foo/bar) the router 7 | is responsible for determining what version of the API should be used. 8 | 9 | """ 10 | 11 | from flask import Blueprint 12 | from flask_restful import Api 13 | 14 | api_bp = Blueprint('api_router', __name__) 15 | api = Api(api_bp) 16 | 17 | from . endpoints import * 18 | 19 | # ------------------------------------------------------------------------ 20 | # API Version 1 Endpoints 21 | # ------------------------------------------------------------------------ 22 | api.add_resource(AllJobsEndpoint, '/jobs') 23 | api.add_resource(AllSkillsEndpoint, '/skills') 24 | api.add_resource(JobTitleAutocompleteEndpoint, '/jobs/autocomplete') 25 | api.add_resource(SkillNameAutocompleteEndpoint, '/skills/autocomplete') 26 | api.add_resource(JobTitleNormalizeEndpoint, '/jobs/normalize') 27 | api.add_resource(AllUnusualJobsEndpoint, '/jobs/unusual_titles') 28 | api.add_resource(NormalizeSkillNameEndpoint, '/skills/normalize') 29 | api.add_resource(JobTitleFromONetCodeEndpoint, '/jobs/') 30 | api.add_resource(SkillNameAndFrequencyEndpoint, '/skills/') 31 | api.add_resource(AssociatedSkillsForJobEndpoint, '/jobs//related_skills') 32 | api.add_resource(AssociatedJobsForSkillEndpoint, '/skills//related_jobs') 33 | api.add_resource(AssociatedJobsForJobEndpoint, '/jobs//related_jobs') 34 | api.add_resource(AssociatedSkillForSkillEndpoint, '/skills//related_skills') -------------------------------------------------------------------------------- /app/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ OpenSkills API application. 4 | 5 | This package is the main entryway to the API. It provides all the configuration 6 | elements needed to run the application. 7 | 8 | """ 9 | 10 | from flask import Flask, Blueprint 11 | from flask_sqlalchemy import SQLAlchemy 12 | from flask_migrate import Migrate, MigrateCommand 13 | from flask_restful import Api 14 | from flask_cors import CORS, cross_origin 15 | from flask.ext.elasticsearch import FlaskElasticsearch 16 | 17 | # ---------------------------------------------------------------------------- 18 | # Flask Application Configuration 19 | # ---------------------------------------------------------------------------- 20 | app = Flask(__name__) 21 | app.config.from_object('config.config.Config') 22 | CORS(app) 23 | 24 | # ---------------------------------------------------------------------------- 25 | # Flask-RESTFul API Object 26 | # ---------------------------------------------------------------------------- 27 | api = Api(app, catch_all_404s=True) 28 | 29 | # ---------------------------------------------------------------------------- 30 | # Database 31 | # ---------------------------------------------------------------------------- 32 | db = SQLAlchemy(app) 33 | migrate = Migrate(app, db) 34 | es = FlaskElasticsearch(app) 35 | 36 | from api.router import api_bp as api_router_blueprint 37 | from api.v1 import api_bp as api_1_blueprint 38 | 39 | # ---------------------------------------------------------------------------- 40 | # API Blueprints 41 | # ---------------------------------------------------------------------------- 42 | app.register_blueprint(api_router_blueprint) 43 | app.register_blueprint(api_1_blueprint, url_prefix='/v1') 44 | 45 | -------------------------------------------------------------------------------- /tests/es_normalize_output.json: -------------------------------------------------------------------------------- 1 | { 2 | "hits":{ 3 | "hits":[ 4 | { 5 | "_score":4.096966, 6 | "_type":"titles", 7 | "_id":"-12345", 8 | "fields":{ 9 | "canonicaltitle":[ 10 | "Sales Managers" 11 | ] 12 | }, 13 | "_index":"normalizer" 14 | }, 15 | { 16 | "_score":4.0939603, 17 | "_type":"titles", 18 | "_id":"23456", 19 | "fields":{ 20 | "canonicaltitle":[ 21 | "Sales Managers" 22 | ] 23 | }, 24 | "_index":"normalizer" 25 | }, 26 | { 27 | "_score":4.0939603, 28 | "_type":"titles", 29 | "_id":"-34567", 30 | "fields":{ 31 | "canonicaltitle":[ 32 | "Sales Managers" 33 | ] 34 | }, 35 | "_index":"normalizer" 36 | }, 37 | { 38 | "_score":4.0794497, 39 | "_type":"titles", 40 | "_id":"-45678", 41 | "fields":{ 42 | "canonicaltitle":[ 43 | "Sales Managers" 44 | ] 45 | }, 46 | "_index":"normalizer" 47 | }, 48 | { 49 | "_score":2.1334965, 50 | "_type":"titles", 51 | "_id":"56789", 52 | "fields":{ 53 | "canonicaltitle":[ 54 | "First-Line Supervisors/Managers of Non-Retail Sales Workers" 55 | ] 56 | }, 57 | "_index":"normalizer" 58 | } 59 | ], 60 | "total":922009, 61 | "max_score":4.096966 62 | }, 63 | "_shards":{ 64 | "successful":1, 65 | "failed":0, 66 | "total":1 67 | }, 68 | "took":443, 69 | "timed_out":false 70 | } 71 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | BY DOWNLOADING aequitas PROGRAM YOU AGREE TO THE FOLLOWING TERMS OF USE: 2 | 3 | Copyright ©2018. The University of Chicago (“Chicago”). All Rights Reserved. 4 | 5 | Permission to use, copy, modify, and distribute this software, including all object code and source code, and any accompanying documentation (together the “Program”) for educational and not-for-profit research purposes, without fee and without a signed licensing agreement, is hereby granted, provided that the above copyright notice, this paragraph and the following three paragraphs appear in all copies, modifications, and distributions. For the avoidance of doubt, educational and not-for-profit research purposes excludes any service or part of selling a service that uses the Program. To obtain a commercial license for the Program, contact the Technology Commercialization and Licensing, Polsky Center for Entrepreneurship and Innovation, University of Chicago, 1452 East 53rd Street, 2nd floor, Chicago, IL 60615. 6 | 7 | Created by Data Science and Public Policy, University of Chicago 8 | 9 | The Program is copyrighted by Chicago. The Program is supplied "as is", without any accompanying services from Chicago. Chicago does not warrant that the operation of the Program will be uninterrupted or error-free. The end-user understands that the Program was developed for research purposes and is advised not to rely exclusively on the Program for any reason. 10 | 11 | IN NO EVENT SHALL CHICAGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THE PROGRAM, EVEN IF CHICAGO HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CHICAGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE PROGRAM PROVIDED HEREUNDER IS PROVIDED "AS IS". CHICAGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 12 | -------------------------------------------------------------------------------- /api/v1/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """API Version 1 4 | 5 | This package contains all the endpoints and data models for the version 1 API. 6 | 7 | """ 8 | 9 | from flask import Blueprint 10 | from flask_restful import Api 11 | 12 | api_bp = Blueprint('api_v1', __name__) 13 | api = Api(api_bp) 14 | 15 | from . models.skills_master import SkillMaster 16 | from . models.skills_related import SkillRelated 17 | from . models.jobs_master import JobMaster 18 | from . models.jobs_alternate_titles import JobAlternateTitle 19 | from . models.jobs_unusual_titles import JobUnusualTitle 20 | from . models.jobs_skills import JobSkill 21 | from . models.skills_importance import SkillImportance 22 | from . models.quarters import Quarter 23 | from . models.geographies import Geography 24 | from . models.jobs_importance import JobImportance 25 | 26 | from . endpoints import * 27 | 28 | # ------------------------------------------------------------------------ 29 | # API Version 1 Endpoints 30 | # ------------------------------------------------------------------------ 31 | api.add_resource(AllJobsEndpoint, '/jobs') 32 | api.add_resource(AllSkillsEndpoint, '/skills') 33 | api.add_resource(JobTitleAutocompleteEndpoint, '/jobs/autocomplete') 34 | api.add_resource(SkillNameAutocompleteEndpoint, '/skills/autocomplete') 35 | api.add_resource(JobTitleNormalizeEndpoint, '/jobs/normalize') 36 | api.add_resource(AllUnusualJobsEndpoint, '/jobs/unusual_titles') 37 | api.add_resource(NormalizeSkillNameEndpoint, '/skills/normalize') 38 | api.add_resource(JobTitleFromONetCodeEndpoint, '/jobs/') 39 | api.add_resource(SkillNameAndFrequencyEndpoint, '/skills/') 40 | api.add_resource(AssociatedSkillsForJobEndpoint, '/jobs//related_skills') 41 | api.add_resource(AssociatedJobsForSkillEndpoint, '/skills//related_jobs') 42 | api.add_resource(AssociatedJobsForJobEndpoint, '/jobs//related_jobs') 43 | api.add_resource(AssociatedSkillForSkillEndpoint, '/skills//related_skills') 44 | api.add_resource(TitleCountsEndpoint, '/title_counts') 45 | -------------------------------------------------------------------------------- /bin/make_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # ----------------------------------------------------------------------------- 4 | # make_config.sh - Create a configuration file for the application. 5 | # 6 | # Usage: ./make_config.sh [--development | --testing | --production] 7 | # ----------------------------------------------------------------------------- 8 | 9 | SCRIPT_NAME=`basename "$0"` 10 | CURRENT_DIR=`pwd` 11 | 12 | if [ -f $SCRIPT_NAME ]; then 13 | SOURCE_DIR=.. 14 | else 15 | SOURCE_DIR=. 16 | fi 17 | 18 | CONFIG_DIR=$SOURCE_DIR/config 19 | CONFIG_MODE=$1 20 | 21 | if [ -z $CONFIG_MODE ]; then 22 | echo "Error. Please specify a configuration mode (--development, --testing, --production)" 23 | exit 1 24 | fi 25 | 26 | # Purge the original config module 27 | if [ -d $CONFIG_DIR ]; then 28 | rm -Rf $CONFIG_DIR 29 | fi 30 | 31 | case $CONFIG_MODE in 32 | --development ) 33 | read -p "Please specify the URI for the development database and hit Enter => " DATABASE_URI 34 | DEBUG=True 35 | TESTING=False 36 | ;; 37 | --testing ) 38 | read -p "Please specify the URI for the testing database and hit Enter => " DATABASE_URI 39 | DEBUG=False 40 | TESTING=True 41 | ;; 42 | --production ) 43 | read -p "Please specify the URI for the production database and hit Enter => " DATABASE_URI 44 | DEBUG=False 45 | TESTING=False 46 | ;; 47 | * ) echo "Invalid configuration mode (${CONFIG_MODE}), select one of: --development, --testing, --production" 48 | exit 1 49 | ;; 50 | esac 51 | 52 | # Build the config module 53 | mkdir $CONFIG_DIR 54 | cd $CONFIG_DIR 55 | touch __init__.py 56 | touch config.py 57 | 58 | cat <> ./config.py 59 | # -*- coding: utf-8 -*- 60 | 61 | """Configuration object for Open Skills API. 62 | 63 | Generated by ${SCRIPT_NAME} with mode ${CONFIG_MODE} 64 | 65 | """ 66 | 67 | class Config(object): 68 | DEBUG = ${DEBUG} 69 | TESTING = ${TESTING} 70 | SQLALCHEMY_TRACK_MODIFICATIONS = False 71 | CSRF_ENABLED = True 72 | SQLALCHEMY_DATABASE_URI = '${DATABASE_URI}' 73 | EOF 74 | 75 | # All done, return to the home directory. 76 | cd $CURRENT_DIR 77 | echo "Application configuration completed. WARNING: Please DO NOT add the api_config module to the code repo as it may expose credentials!" 78 | exit 0 79 | -------------------------------------------------------------------------------- /tests/job_normalize_test.py: -------------------------------------------------------------------------------- 1 | from app import app 2 | import httpretty 3 | import json 4 | import re 5 | import unittest 6 | 7 | 8 | class JobNormalizeTestCase(unittest.TestCase): 9 | 10 | def setUp(self): 11 | self.app = app.app.test_client() 12 | self.app.testing = True 13 | 14 | def mock_endpoint_with_response(self, source_filename): 15 | httpretty.HTTPretty.allow_net_connect = False 16 | url_regex = 'http://{}/.*'.format(app.app.config['ELASTICSEARCH_HOST']) 17 | with open(source_filename) as f: 18 | output = f.read() 19 | httpretty.register_uri( 20 | httpretty.GET, 21 | re.compile(url_regex), 22 | body=output, 23 | content_type='application/json' 24 | ) 25 | 26 | @httpretty.activate 27 | def test_with_hits(self): 28 | self.mock_endpoint_with_response('tests/es_normalize_output.json') 29 | response = self.app.get('v1/jobs/normalize?job_title=cupcake+ninja') 30 | self.assertEqual(response.status_code, 200) 31 | response_data = json.loads(response.get_data()) 32 | self.assertEqual(len(response_data), 1) 33 | self.assertEqual(response_data[0]['title'], 'Sales Managers') 34 | 35 | @httpretty.activate 36 | def test_with_defined_limit(self): 37 | self.mock_endpoint_with_response('tests/es_normalize_output.json') 38 | response = self.app.get('v1/jobs/normalize?job_title=cupcake+ninja&limit=2') 39 | self.assertEqual(response.status_code, 200) 40 | response_data = json.loads(response.get_data()) 41 | self.assertEqual(len(response_data), 2) 42 | self.assertEqual( 43 | response_data[1]['title'], 44 | 'First-Line Supervisors/Managers of Non-Retail Sales Workers' 45 | ) 46 | 47 | @httpretty.activate 48 | def test_no_hits(self): 49 | self.mock_endpoint_with_response('tests/es_nohit_output.json') 50 | response = self.app.get('v1/jobs/normalize?job_title=baker') 51 | self.assertEqual(response.status_code, 404) 52 | 53 | def test_no_title(self): 54 | response = self.app.get('v1/jobs/normalize') 55 | self.assertEqual(response.status_code, 400) 56 | 57 | def test_bad_limit(self): 58 | response = self.app.get('v1/jobs/normalize?job_title=test&limit=999999') 59 | self.assertEqual(response.status_code, 400) 60 | 61 | if __name__ == '__main__': 62 | unittest.main() 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Open Skills API - Sharing the DNA of America's Jobs 2 | Provides a complete and standard data store for canonical and emerging skills, 3 | knowledge, abilities, tools, technolgies, and how they relate to jobs. 4 | 5 | ## Overview 6 | A web application to serve the [Open Skills API](http://api.dataatwork.org/v1/spec/). 7 | 8 | An overview of the API is maintained in this repository's Wiki: [API Overview](https://github.com/workforce-data-initiative/skills-api/wiki/API-Overview) 9 | 10 | 11 | ## Loading Data 12 | The data necessary to drive the Open Skills API is loaded through the tasks present in the [skills-airflow](https://github.com/workforce-data-initiative/skills-airflow/) project. Follow the instructions in that repo to run the workflow and load data into a database, along with an Elasticsearch endpoint. You will use the database credentials and Elasticsearch endpoint when configuring this application. 13 | 14 | ## Dependencies 15 | - Python 2.7.11 16 | - Postgres database with skills and jobs data loaded. (see skills-airflow note above) 17 | - Elasticsearch 5.x instance with job normalization data loaded (see skills-airflow note above) 18 | 19 | ## Installation 20 | To run the API locally, please perform the following steps: 21 | 1. Clone the repository from [https://www.github.com/workforce-data-initiative/skills-api](https://www.github.com/workforce-data-initiative/skills-api) 22 | ``` 23 | $ git clone https://www.github.com/workforce-data-initiative/skills-api 24 | ``` 25 | 2. Navigate to the checked out project 26 | ``` 27 | $ cd skills-api 28 | ``` 29 | 3. Ensure that pip package manager is installed. See installation instructions [here](https://pip.pypa.io/en/stable/installing/). 30 | ``` 31 | $ pip --version 32 | ``` 33 | 4. Install the `virtualenv` package. Please review the [documentation](https://virtualenv.pypa.io/en/stable/) if you are unfamiliar with how `virtualenv` works. 34 | ``` 35 | $ pip install virtualenv 36 | ``` 37 | 5. Create a Python 2.7.11 virtual environment called `venv` in the project root directory 38 | ``` 39 | $ virtualenv -p /path/to/python/2.7.11 venv 40 | ``` 41 | 6. Activate the virtual environment. Note that the name of the virtual environment (`venv`) will be appended to the front of the command prompt. 42 | ``` 43 | $ source venv/bin/activate 44 | (venv) $ 45 | ``` 46 | 7. Install dependencies from `requirements.txt` 47 | ``` 48 | $ pip install -r requirements.txt 49 | ``` 50 | 51 | 8. Make regular (development) config. Run bin/make_config.sh and fill in connection string to the database used in skills-airflow. 52 | ``` 53 | $ bin/make_config.sh 54 | ``` 55 | 56 | 9. Add an ELASTICSEARCH_HOST variable to config/config.py to point to the Elasticsearch instance that holds the job normalization data from skills-airflow 57 | 58 | 10. Clone development config for test config. Copy the resultant config/config.py to config/test_config.py and modify the SQL connection string to match your test database (you can leave this the same as your development database, if you wish, but we recommend keeping separate ones. 59 | ``` 60 | $ cp config/config.py config/test_config.py 61 | ``` 62 | 63 | Now you can run the Flask server. 64 | ``` 65 | (venv) $ python server.py runserver 66 | ``` 67 | 68 | Navigate to `http://127.0.0.1:5000/v1/jobs` and you should see a listing of jobs. You can check out more endpoints at the [API Specification](http://127.0.0.1:5000/v1/jobs) 69 | -------------------------------------------------------------------------------- /tests/job_get_test.py: -------------------------------------------------------------------------------- 1 | from app import app 2 | import json 3 | import unittest 4 | from tests.api_test_case import ApiTestCase 5 | 6 | from tests.factories import GeographyFactory, \ 7 | JobImportanceFactory, \ 8 | JobMasterFactory, \ 9 | QuarterFactory 10 | 11 | 12 | class JobGetTestCase(ApiTestCase): 13 | 14 | def test_with_uuid(self): 15 | job = JobMasterFactory() 16 | app.db.session.add(job) 17 | response = self.app.get('v1/jobs/{}'.format(job.uuid)) 18 | self.assertEqual(response.status_code, 200) 19 | response_data = json.loads(response.get_data()) 20 | self.assertEqual(response_data['title'], job.title) 21 | 22 | def test_with_soc_code(self): 23 | job = JobMasterFactory() 24 | app.db.session.add(job) 25 | response = self.app.get('v1/jobs/{}'.format(job.onet_soc_code)) 26 | self.assertEqual(response.status_code, 200) 27 | response_data = json.loads(response.get_data()) 28 | self.assertEqual(response_data['title'], job.title) 29 | 30 | def test_missing_geo(self): 31 | job = JobMasterFactory() 32 | app.db.session.add(job) 33 | response = self.app.get('v1/jobs/{}?fips=40500'.format(job.uuid)) 34 | self.assertEqual(response.status_code, 404) 35 | 36 | def test_geo_mismatch(self): 37 | job = JobMasterFactory() 38 | right_geography = GeographyFactory() 39 | wrong_geography = GeographyFactory() 40 | quarter = QuarterFactory() 41 | app.db.session.begin(subtransactions=True) 42 | app.db.session.add(job) 43 | app.db.session.add(right_geography) 44 | app.db.session.add(wrong_geography) 45 | app.db.session.add(quarter) 46 | app.db.session.commit() 47 | importance = JobImportanceFactory( 48 | geography_id=right_geography.geography_id, 49 | quarter_id=quarter.quarter_id, 50 | job_uuid=job.uuid 51 | ) 52 | app.db.session.add(importance) 53 | response = self.app.get( 54 | 'v1/jobs/{}?fips={}'.format( 55 | job.uuid, 56 | wrong_geography.geography_name 57 | ) 58 | ) 59 | self.assertEqual(response.status_code, 404) 60 | 61 | def test_geo_match(self): 62 | job = JobMasterFactory() 63 | right_geography = GeographyFactory() 64 | wrong_geography = GeographyFactory() 65 | quarter = QuarterFactory() 66 | app.db.session.begin(subtransactions=True) 67 | app.db.session.add(job) 68 | app.db.session.add(right_geography) 69 | app.db.session.add(wrong_geography) 70 | app.db.session.add(quarter) 71 | app.db.session.commit() 72 | 73 | importance = JobImportanceFactory( 74 | geography_id=right_geography.geography_id, 75 | quarter_id=quarter.quarter_id, 76 | job_uuid=job.uuid 77 | ) 78 | app.db.session.add(importance) 79 | response = self.app.get( 80 | 'v1/jobs/{}?fips={}'.format( 81 | job.uuid, 82 | right_geography.geography_name 83 | ) 84 | ) 85 | self.assertEqual(response.status_code, 200) 86 | response_data = json.loads(response.get_data()) 87 | self.assertEqual(response_data['title'], job.title) 88 | 89 | if __name__ == '__main__': 90 | unittest.main() 91 | -------------------------------------------------------------------------------- /common/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """General Purpose Utilites""" 4 | 5 | import re 6 | import json 7 | import functools 8 | from flask import request, make_response, redirect, url_for 9 | from collections import OrderedDict 10 | 11 | def get_api_version_custom(): 12 | """Retrieve custom version header from HTTP request. 13 | 14 | Returns: 15 | Content of the api-version custom header. 16 | 17 | """ 18 | return request.headers.get('api-version') 19 | 20 | 21 | def get_api_version_accept(): 22 | """Retrieve accept header from HTTP request. 23 | 24 | Returns: 25 | Content of the accept header. 26 | """ 27 | return request.headers.get('accept') 28 | 29 | def parse_version_number(header): 30 | """Parses the version number from an accept header 31 | Args: 32 | header (str): Header content to parse for version number. 33 | 34 | Returns: 35 | Version number. 36 | 37 | """ 38 | version_regex = '\w+/[VvNnDd]{3}\.\w+\.v([0-9\.]+)\+' 39 | match = re.search(version_regex, header) 40 | if match: 41 | found = match.group(1) 42 | else: 43 | found = None 44 | 45 | return found 46 | 47 | def normalize_version_number(version_number): 48 | """Clean up the version number extracted from the header 49 | Args: 50 | version_number (str): Version number to normalize. 51 | 52 | Returns: 53 | The normalized version number. 54 | 55 | """ 56 | return version_number.replace('.', '_') 57 | 58 | def create_response(data, status, custom_headers=None): 59 | """Create a custom JSON response. 60 | Args: 61 | data (dict): Data to place in to the custom response body. 62 | status (str): HTTP status code to return with the response. 63 | custom_headers (list): Any optional custom headers to return with the response. 64 | 65 | Returns: 66 | Custom JSON response. 67 | 68 | """ 69 | response = make_response(json.dumps(data), status) 70 | response.headers['Content-Type'] = "application/json" 71 | response.headers['Access-Control-Allow-Headers'] = "Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token" 72 | response.headers['Access-Control-Allow-Methods'] = "*" 73 | response.headers['Access-Control-Allow-Origin'] = "*" 74 | 75 | if custom_headers is not None: 76 | for custom_header in custom_headers: 77 | header = custom_header.strip().split('=') 78 | response.headers[header[0].strip()] = header[1].strip() 79 | 80 | return response 81 | 82 | def create_error(data, status, custom_headers=None): 83 | """Create a custom JSON error response. 84 | Args: 85 | data (dict): Data to place in the custom response body. 86 | status (str): HTTP status code to return with the response. 87 | 88 | Returns: 89 | Custom JSON error response. 90 | 91 | """ 92 | response_obj = {} 93 | response_obj['error'] = OrderedDict() 94 | response_obj['error']['code'] = status 95 | response_obj['error']['message'] = data 96 | 97 | return create_response(response_obj, status, custom_headers) 98 | 99 | def route_api(endpoint_class, id=None): 100 | """Routes an API call based on its endpoint class and version. 101 | 102 | Args: 103 | endpoint_class (str): Name of endpoint class to route the API to. 104 | id (str): An optional ID parameter to pass to the appropriate endpoint. 105 | 106 | Returns: 107 | A redirect to the appropriate API endpoint based on its version number. 108 | 109 | """ 110 | api_version = get_api_version_custom() 111 | if api_version is not None: 112 | endpoint = 'api_v' + normalize_version_number(api_version) + '.' + endpoint_class 113 | return redirect(url_for(endpoint, id=id, **request.args)) 114 | else: 115 | api_version = get_api_version_accept() 116 | if api_version is not None: 117 | endpoint = 'api_v' + normalize_version_number(parse_version_number(api_version)) + '.' + endpoint_class 118 | return redirect(url_for(endpoint, id=id, **request.args)) 119 | else: 120 | return create_error('A version header is missing from the request', 400) 121 | -------------------------------------------------------------------------------- /bin/make_zappa_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # ----------------------------------------------------------------------------- 4 | # make_zappa_config.sh - Create a Zappa configuration file for the application. 5 | # 6 | # Usage: ./make_zappa_config.sh [--development | --testing | --production] 7 | # ----------------------------------------------------------------------------- 8 | 9 | SCRIPT_NAME=`basename "$0"` 10 | CURRENT_DIR=`pwd` 11 | 12 | if [ -f $SCRIPT_NAME ]; then 13 | SOURCE_DIR=.. 14 | else 15 | SOURCE_DIR=. 16 | fi 17 | 18 | CONFIG_DIR=$SOURCE_DIR 19 | CONFIG_MODE=$1 20 | ZAPPA_FILE=$CONFIG_DIR/zappa_settings.json 21 | 22 | # Default Zappa configs 23 | ZAPPA_AWS_REGION=us-east-1 24 | ZAPPA_CACHE_CLUSTER_ENABLED=false 25 | ZAPPA_CACHE_CLUSTER_SIZE=.5 26 | ZAPPA_DEBUG=true 27 | ZAPPA_DELETE_ZIP=true 28 | ZAPPA_DOMAIN= 29 | ZAPPA_EVENTS= 30 | ZAPPA_EXCLUDE=(\"*.pem\", \"*.gz\", \"*.csv\", \"*.tsv\") 31 | ZAPPA_HTTP_METHODS=(\"GET\", \"POST\", \"PUT\", \"DELETE\") 32 | ZAPPA_INTEGRATION_RESPONSE_CODES=(200, 301, 404, 500) 33 | ZAPPA_KEEP_WARM=true 34 | ZAPPA_LOG_LEVEL=DEBUG 35 | ZAPPA_MEMORY_SIZE=512 36 | ZAPPA_METHOD_RESPONSE_CODES=(200, 301, 404, 500) 37 | ZAPPA_PARAMETER_DEPTH=8 38 | ZAPPA_PREBUILD_SCRIPT= 39 | ZAPPA_PROFILE_NAME="default" 40 | ZAPPA_PROJECT_NAME= 41 | ZAPPA_ROLE_NAME= 42 | ZAPPA_S3_BUCKET= 43 | ZAPPA_SETTINGS_FILE= 44 | ZAPPA_TIMEOUT_SECONDS=30 45 | ZAPPA_TOUCH=true 46 | ZAPPA_USE_PRECOMPILED_PACKAGES=true 47 | ZAPPA_VPC_CONFIG= 48 | 49 | # Purge the old Zappa settings file 50 | if [ -f $ZAPPA_FILE ]; then 51 | rm -f $ZAPPA_FILE 52 | fi 53 | 54 | # Build the config module 55 | cat <> $ZAPPA_FILE 56 | { 57 | "development": { 58 | "aws_region": "${ZAPPA_AWS_REGION}", 59 | "cache_cluster_enabled": ${ZAPPA_CACHE_CLUSTER_ENABLED}, 60 | "cache_cluster_size": ${ZAPPA_CACHE_CLUSTER_SIZE}, 61 | "debug": ${ZAPPA_DEBUG}, 62 | "delete_zip": ${ZAPPA_DELETE_ZIP}, 63 | "domain": "${ZAPPA_DOMAIN}", 64 | "events": [{ 65 | ${ZAPPA_EVENTS} 66 | }], 67 | "exclude": [${ZAPPA_EXCLUDE[*]}], 68 | "http_methods": [${ZAPPA_HTTP_METHODS[*]}], 69 | "integration_response_codes": [${ZAPPA_INTEGRATION_RESPONSE_CODES[*]}], 70 | "keep_warm": ${ZAPPA_KEEP_WARM}, 71 | "log_level": "${ZAPPA_LOG_LEVEL}", 72 | "memory_size": $ZAPPA_MEMORY_SIZE, 73 | "method_response_codes": [${ZAPPA_METHOD_RESPONSE_CODES[*]}], 74 | "parameter_depth": $ZAPPA_PARAMETER_DEPTH, 75 | "prebuild_script": "$ZAPPA_PREBUILD_SCRIPT", 76 | "profile_name": "$ZAPPA_PROFILE_NAME", 77 | "project_name": "$ZAPPA_PROJECT_NAME", 78 | "role_name": "$ZAPPA_ROLE_NAME", 79 | "s3_bucket": "$ZAPPA_S3_BUCKET", 80 | "settings_file": "$ZAPPA_SETTINGS_FILE", 81 | "timeout_seconds": $ZAPPA_TIMEOUT_SECONDS, 82 | "touch": $ZAPPA_TOUCH, 83 | "use_precompiled_packages": $ZAPPA_USE_PRECOMPILED_PACKAGES, 84 | "vpc_config": { 85 | "SubnetIds": [], 86 | "SecurityGroupIds": [] 87 | } 88 | }, 89 | "testing": { 90 | "aws_region": "${ZAPPA_AWS_REGION}", 91 | "cache_cluster_enabled": ${ZAPPA_CACHE_CLUSTER_ENABLED}, 92 | "cache_cluster_size": ${ZAPPA_CACHE_CLUSTER_SIZE}, 93 | "debug": ${ZAPPA_DEBUG}, 94 | "delete_zip": ${ZAPPA_DELETE_ZIP}, 95 | "domain": "${ZAPPA_DOMAIN}", 96 | "events": [{ 97 | ${ZAPPA_EVENTS} 98 | }], 99 | "exclude": [${ZAPPA_EXCLUDE[*]}], 100 | "http_methods": [${ZAPPA_HTTP_METHODS[*]}], 101 | "integration_response_codes": [${ZAPPA_INTEGRATION_RESPONSE_CODES[*]}], 102 | "keep_warm": ${ZAPPA_KEEP_WARM}, 103 | "log_level": "${ZAPPA_LOG_LEVEL}", 104 | "memory_size": $ZAPPA_MEMORY_SIZE, 105 | "method_response_codes": [${ZAPPA_METHOD_RESPONSE_CODES[*]}], 106 | "parameter_depth": $ZAPPA_PARAMETER_DEPTH, 107 | "prebuild_script": "$ZAPPA_PREBUILD_SCRIPT", 108 | "profile_name": "$ZAPPA_PROFILE_NAME", 109 | "project_name": "$ZAPPA_PROJECT_NAME", 110 | "role_name": "$ZAPPA_ROLE_NAME", 111 | "s3_bucket": "$ZAPPA_S3_BUCKET", 112 | "settings_file": "$ZAPPA_SETTINGS_FILE", 113 | "timeout_seconds": $ZAPPA_TIMEOUT_SECONDS, 114 | "touch": $ZAPPA_TOUCH, 115 | "use_precompiled_packages": $ZAPPA_USE_PRECOMPILED_PACKAGES, 116 | "vpc_config": { 117 | "SubnetIds": [], 118 | "SecurityGroupIds": [] 119 | } 120 | }, 121 | "production": { 122 | "aws_region": "${ZAPPA_AWS_REGION}", 123 | "cache_cluster_enabled": ${ZAPPA_CACHE_CLUSTER_ENABLED}, 124 | "cache_cluster_size": ${ZAPPA_CACHE_CLUSTER_SIZE}, 125 | "debug": ${ZAPPA_DEBUG}, 126 | "delete_zip": ${ZAPPA_DELETE_ZIP}, 127 | "domain": "${ZAPPA_DOMAIN}", 128 | "events": [{ 129 | ${ZAPPA_EVENTS} 130 | }], 131 | "exclude": [${ZAPPA_EXCLUDE[*]}], 132 | "http_methods": [${ZAPPA_HTTP_METHODS[*]}], 133 | "integration_response_codes": [${ZAPPA_INTEGRATION_RESPONSE_CODES[*]}], 134 | "keep_warm": ${ZAPPA_KEEP_WARM}, 135 | "log_level": "${ZAPPA_LOG_LEVEL}", 136 | "memory_size": $ZAPPA_MEMORY_SIZE, 137 | "method_response_codes": [${ZAPPA_METHOD_RESPONSE_CODES[*]}], 138 | "parameter_depth": $ZAPPA_PARAMETER_DEPTH, 139 | "prebuild_script": "$ZAPPA_PREBUILD_SCRIPT", 140 | "profile_name": "$ZAPPA_PROFILE_NAME", 141 | "project_name": "$ZAPPA_PROJECT_NAME", 142 | "role_name": "$ZAPPA_ROLE_NAME", 143 | "s3_bucket": "$ZAPPA_S3_BUCKET", 144 | "settings_file": "$ZAPPA_SETTINGS_FILE", 145 | "timeout_seconds": $ZAPPA_TIMEOUT_SECONDS, 146 | "touch": $ZAPPA_TOUCH, 147 | "use_precompiled_packages": $ZAPPA_USE_PRECOMPILED_PACKAGES, 148 | "vpc_config": { 149 | "SubnetIds": [], 150 | "SecurityGroupIds": [] 151 | } 152 | } 153 | } 154 | EOF 155 | 156 | # All done, return to the home directory. 157 | cd $CURRENT_DIR 158 | echo "Application configuration completed. WARNING: Please DO NOT add the Zappa settings file to the code repo as it may expose credentials!" 159 | exit 0 160 | -------------------------------------------------------------------------------- /api/router/endpoints.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """API Façade. 4 | 5 | The API façade provides routing for the API endpoints based on the version 6 | specified by the user either via a custom or Accept header. 7 | 8 | Note: 9 | All classes in this module inherit from the Flask RESTful Resource parent 10 | class. 11 | 12 | """ 13 | 14 | from flask import abort, request, redirect, url_for 15 | from flask_restful import Resource 16 | from common.utils import route_api 17 | 18 | # ------------------------------------------------------------------------ 19 | # API Version 1 Endpoints 20 | # ------------------------------------------------------------------------ 21 | class AllJobsEndpoint(Resource): 22 | """All Jobs Endpoint Class""" 23 | 24 | def get(self): 25 | """GET operation for the endpoint class. 26 | 27 | Returns: 28 | A redirect to the appropriate API version as specified by the 29 | accept or custom header. 30 | 31 | """ 32 | return route_api('alljobsendpoint') 33 | 34 | class AllSkillsEndpoint(Resource): 35 | """All Skills Endpoint Class""" 36 | 37 | def get(self): 38 | """GET operation for the endpoint class. 39 | 40 | Returns: 41 | A redirect to the appropriate API version as specified by the 42 | accept or custom header. 43 | 44 | """ 45 | return route_api('allskillsendpoint') 46 | 47 | class JobTitleAutocompleteEndpoint(Resource): 48 | """Job Title Autocomplete Endpoint Class""" 49 | 50 | def get(self): 51 | """GET operation for the endpoint class. 52 | 53 | Returns: 54 | A redirect to the appropriate API version as specified by the 55 | accept or custom header. 56 | 57 | """ 58 | return route_api('jobtitleautocompleteendpoint') 59 | 60 | class SkillNameAutocompleteEndpoint(Resource): 61 | """Skill Name Autocomplete Endpoint Class""" 62 | 63 | def get(self): 64 | """GET operation for the endpoint class. 65 | 66 | Returns: 67 | A redirect to the appropriate API version as specified by the 68 | accept or custom header. 69 | 70 | """ 71 | return route_api('skillnameautocompleteendpoint') 72 | 73 | class JobTitleNormalizeEndpoint(Resource): 74 | """Job Title Normalize Endpoint Class""" 75 | 76 | def get(self): 77 | """GET operation for the endpoint class. 78 | 79 | Returns: 80 | A redirect to the appropriate API version as specified by the 81 | accept or custom header. 82 | 83 | """ 84 | return route_api('jobtitlenormalizeendpoint') 85 | 86 | 87 | class AllUnusualJobsEndpoint(Resource): 88 | """All Unusual Jobs Endpoint Class""" 89 | 90 | def get(self): 91 | """GET operation for the endpoint class. 92 | 93 | Returns: 94 | A redirect to the appropriate API version as specified by the 95 | accept or custom header. 96 | 97 | """ 98 | return route_api('allunusualjobsendpoint') 99 | 100 | class JobTitleFromONetCodeEndpoint(Resource): 101 | """Job Title From O*NET SOC Code Endpoint Class""" 102 | 103 | def get(self, id=None): 104 | """GET operation for the endpoint class. 105 | 106 | Args: 107 | id: Optional O*NET SOC Code to query for. All job titles if none 108 | specified. 109 | 110 | Returns: 111 | A redirect to the appropriate API version as specified by the 112 | accept or custom header. 113 | 114 | """ 115 | return route_api('jobtitlefromonetcodeendpoint', id=id) 116 | 117 | class NormalizeSkillNameEndpoint(Resource): 118 | """Normalize Skill Name Endpoint Class""" 119 | 120 | def get(self): 121 | """GET operation for the endpoint class. 122 | 123 | Returns: 124 | A redirect to the appropriate API version as specified by the 125 | accept or custom header. 126 | 127 | """ 128 | return route_api('normalizeskillnameendpoint') 129 | 130 | class AssociatedSkillsForJobEndpoint(Resource): 131 | """Associated Skills For Jobs Endpoint Class""" 132 | 133 | def get(self, id): 134 | """GET operation for the endpoint class. 135 | 136 | Args: 137 | id: Job UUID. 138 | 139 | Returns: 140 | A redirect to the appropriate API version as specified by the 141 | accept or custom header. 142 | 143 | """ 144 | return route_api('associatedskillsforjobendpoint', id=id) 145 | 146 | class AssociatedJobsForSkillEndpoint(Resource): 147 | """Associated Jobs For Skills Endpoint Class""" 148 | 149 | def get(self, id): 150 | """GET operation for the endpoint class. 151 | 152 | Args: 153 | id: Skill UUID. 154 | 155 | Returns: 156 | A redirect to the appropriate API version as specified by the 157 | accept or custom header. 158 | 159 | """ 160 | return route_api('associatedjobsforskillendpoint', id=id) 161 | 162 | class AssociatedJobsForJobEndpoint(Resource): 163 | """Associated Jobs For Job Endpoint Class""" 164 | 165 | def get(self, id=None): 166 | """GET operation for the endpoint class. 167 | 168 | Args: 169 | id: Optional job UUID. 170 | 171 | Returns: 172 | A redirect to the appropriate API version as specified by the 173 | accept or custom header. 174 | 175 | """ 176 | return route_api('associatedjobsforjobendpoint', id=id) 177 | 178 | class AssociatedSkillForSkillEndpoint(Resource): 179 | """Associated Skills For Skill Endpoint Class""" 180 | 181 | def get(self, id=None): 182 | """GET operation for the endpoint class. 183 | 184 | Args: 185 | id: Optional skill UUID. 186 | 187 | Returns: 188 | A redirect to the appropriate API version as specified by the 189 | accept or custom header. 190 | 191 | """ 192 | return route_api('associatedskillforskillendpoint', id=id) 193 | 194 | class SkillNameAndFrequencyEndpoint(Resource): 195 | """Skill Name And Frequency Endpoint Class""" 196 | 197 | def get(self, id=None): 198 | """GET operation for the endpoint class. 199 | 200 | Args: 201 | id: Optional skill UUID. 202 | 203 | Returns: 204 | A redirect to the appropriate API version as specified by the 205 | accept or custom header. 206 | 207 | """ 208 | return route_api('skillnameandfrequencyendpoint', id=id) -------------------------------------------------------------------------------- /skills-api.json: -------------------------------------------------------------------------------- 1 | { 2 | "swagger" : "2.0", 3 | "info" : { 4 | "version" : "1.0", 5 | "title" : "Open Skills API", 6 | "description" : "A complete and standard data store for canonical and emerging skills, knowledge, abilities, tools, technolgies, and how they relate to jobs.", 7 | "contact" : { 8 | "name" : "Work Data Initiative", 9 | "url" : "http://www.dataatwork.org" 10 | } 11 | }, 12 | "host" : "api.dataatwork.org", 13 | "schemes" : [ "http" ], 14 | "basePath" : "/v1", 15 | "produces" : [ "application/json" ], 16 | "consumes" : [ "application/json" ], 17 | "paths" : { 18 | "/jobs" : { 19 | "get" : { 20 | "summary" : "Job Titles and Descriptions", 21 | "description" : "Retrieves the names, descriptions, and UUIDs of all job titles.", 22 | "parameters" : [ { 23 | "name" : "offset", 24 | "in" : "query", 25 | "description" : "Pagination offset. Default is 0.", 26 | "type" : "integer" 27 | }, { 28 | "name" : "limit", 29 | "in" : "query", 30 | "description" : "Maximum number of items per page. Default is 20 and cannot exceed 500.", 31 | "type" : "integer" 32 | } ], 33 | "responses" : { 34 | "200" : { 35 | "description" : "A collection of jobs", 36 | "schema" : { 37 | "$ref" : "#/definitions/Jobs" 38 | } 39 | }, 40 | "default" : { 41 | "description" : "Unexpected error", 42 | "schema" : { 43 | "$ref" : "#/definitions/Error" 44 | } 45 | } 46 | } 47 | } 48 | }, 49 | "/jobs/{id}" : { 50 | "get" : { 51 | "summary" : "Job Title and Description", 52 | "description" : "Retrieves the name, description, and UUID of a job by specifying its O*NET SOC Code or UUID.", 53 | "parameters" : [ { 54 | "name" : "id", 55 | "in" : "path", 56 | "description" : "The O*NET SOC Code or UUID of the job title to retrieve", 57 | "required" : true, 58 | "type" : "string" 59 | }, { 60 | "name" : "fips", 61 | "in" : "query", 62 | "description" : "The FIPS Code of a Core-Based Statistical Area. Only return the job if present in this area", 63 | "type" : "string" 64 | } ], 65 | "responses" : { 66 | "200" : { 67 | "description" : "A job", 68 | "schema" : { 69 | "$ref" : "#/definitions/Job" 70 | } 71 | }, 72 | "default" : { 73 | "description" : "Unexpected error", 74 | "schema" : { 75 | "$ref" : "#/definitions/Error" 76 | } 77 | } 78 | } 79 | } 80 | }, 81 | "/jobs/{id}/related_skills" : { 82 | "get" : { 83 | "summary" : "Skills Associated with a Job", 84 | "description" : "Retrieves a collection of skills associated with a specified job.", 85 | "parameters" : [ { 86 | "name" : "id", 87 | "in" : "path", 88 | "description" : "The UUID of the job to retrieve skills for", 89 | "required" : true, 90 | "type" : "string" 91 | } ], 92 | "responses" : { 93 | "200" : { 94 | "description" : "A job and its related skills", 95 | "schema" : { 96 | "$ref" : "#/definitions/JobSkills" 97 | } 98 | }, 99 | "default" : { 100 | "description" : "Unexpected error", 101 | "schema" : { 102 | "$ref" : "#/definitions/Error" 103 | } 104 | } 105 | } 106 | } 107 | }, 108 | "/jobs/{id}/related_jobs" : { 109 | "get" : { 110 | "summary" : "Jobs Associated with a Job", 111 | "description" : "Retrieves a collection of jobs associated with a specified job.", 112 | "parameters" : [ { 113 | "name" : "id", 114 | "in" : "path", 115 | "description" : "The UUID of the job to retrieve related jobs for", 116 | "required" : true, 117 | "type" : "string" 118 | } ], 119 | "responses" : { 120 | "200" : { 121 | "description" : "A job and its related jobs", 122 | "schema" : { 123 | "$ref" : "#/definitions/JobRelatedJobs" 124 | } 125 | }, 126 | "default" : { 127 | "description" : "Unexpected error", 128 | "schema" : { 129 | "$ref" : "#/definitions/Error" 130 | } 131 | } 132 | } 133 | } 134 | }, 135 | "/jobs/autocomplete" : { 136 | "get" : { 137 | "summary" : "Job Title Autocomplete", 138 | "description" : "Retrieves the names, descriptions, and UUIDs of all job titles matching a given search criteria.", 139 | "parameters" : [ { 140 | "name" : "begins_with", 141 | "in" : "query", 142 | "description" : "Find job titles beginning with the given text fragment", 143 | "required" : false, 144 | "type" : "string" 145 | }, { 146 | "name" : "contains", 147 | "in" : "query", 148 | "description" : "Find job titles containing the given text fragment", 149 | "required" : false, 150 | "type" : "string" 151 | }, { 152 | "name" : "ends_with", 153 | "in" : "query", 154 | "description" : "Find job titles ending with the given text fragment", 155 | "required" : false, 156 | "type" : "string" 157 | } ], 158 | "responses" : { 159 | "200" : { 160 | "description" : "A collection of jobs", 161 | "schema" : { 162 | "$ref" : "#/definitions/Jobs" 163 | } 164 | }, 165 | "default" : { 166 | "description" : "Unexpected error", 167 | "schema" : { 168 | "$ref" : "#/definitions/Error" 169 | } 170 | } 171 | } 172 | } 173 | }, 174 | "/jobs/normalize" : { 175 | "get" : { 176 | "summary" : "Job Title Normalization", 177 | "description" : "Retrieves the canonical job title for a synonymous job title", 178 | "parameters" : [ { 179 | "name" : "job_title", 180 | "in" : "query", 181 | "description" : "Find the canonical job title(s) for jobs matching the given text fragment", 182 | "required" : true, 183 | "type" : "string" 184 | }, { 185 | "name" : "limit", 186 | "in" : "query", 187 | "description" : "Maximumn number of job title synonyms to return. Default is 1 and cannot exceed 10.", 188 | "required" : false, 189 | "type" : "integer" 190 | } ], 191 | "responses" : { 192 | "200" : { 193 | "description" : "A collection of normalized jobs", 194 | "schema" : { 195 | "$ref" : "#/definitions/NormalizedJobs" 196 | } 197 | } 198 | } 199 | } 200 | }, 201 | "/jobs/unusual_titles" : { 202 | "get" : { 203 | "summary" : "Unusual Job Titles", 204 | "description" : "Retrieves a list of unusual job titles and the UUIDs of their canonical jobs.", 205 | "responses" : { 206 | "200" : { 207 | "description" : "A collection of normalized jobs", 208 | "schema" : { 209 | "$ref" : "#/definitions/NormalizedJobs" 210 | } 211 | } 212 | } 213 | } 214 | }, 215 | "/skills" : { 216 | "get" : { 217 | "summary" : "Skill Names and Descriptions", 218 | "description" : "Retrieve the names, descriptions, and UUIDs of all skills.", 219 | "parameters" : [ { 220 | "name" : "offset", 221 | "in" : "query", 222 | "description" : "Pagination offset. Default is 0.", 223 | "type" : "integer" 224 | }, { 225 | "name" : "limit", 226 | "in" : "query", 227 | "description" : "Maximum number of items per page. Default is 20 and cannot exceed 500.", 228 | "type" : "integer" 229 | } ], 230 | "responses" : { 231 | "200" : { 232 | "description" : "A collection of skills", 233 | "schema" : { 234 | "$ref" : "#/definitions/Skills" 235 | } 236 | }, 237 | "default" : { 238 | "description" : "Unexpected error", 239 | "schema" : { 240 | "$ref" : "#/definitions/Error" 241 | } 242 | } 243 | } 244 | } 245 | }, 246 | "/skills/{id}" : { 247 | "get" : { 248 | "summary" : "Skill Name and Description", 249 | "description" : "Retrieves the name, description, and UUID of a job by specifying its UUID.", 250 | "parameters" : [ { 251 | "name" : "id", 252 | "in" : "path", 253 | "description" : "The UUID of the skill name to retrieve", 254 | "required" : true, 255 | "type" : "string" 256 | } ], 257 | "responses" : { 258 | "200" : { 259 | "description" : "A skill", 260 | "schema" : { 261 | "$ref" : "#/definitions/Skill" 262 | } 263 | }, 264 | "default" : { 265 | "description" : "Unexpected error", 266 | "schema" : { 267 | "$ref" : "#/definitions/Error" 268 | } 269 | } 270 | } 271 | } 272 | }, 273 | "/skills/{id}/related_jobs" : { 274 | "get" : { 275 | "summary" : "Jobs Associated with a Skill", 276 | "description" : "Retrieves a collection of jobs associated with a specified skill.", 277 | "parameters" : [ { 278 | "name" : "id", 279 | "in" : "path", 280 | "description" : "The UUID of the skill to retrieve jobs for", 281 | "required" : true, 282 | "type" : "string" 283 | } ], 284 | "responses" : { 285 | "200" : { 286 | "description" : "A skill and its related jobs", 287 | "schema" : { 288 | "$ref" : "#/definitions/SkillJobs" 289 | } 290 | }, 291 | "default" : { 292 | "description" : "Unexpected error", 293 | "schema" : { 294 | "$ref" : "#/definitions/Error" 295 | } 296 | } 297 | } 298 | } 299 | }, 300 | "/skills/{id}/related_skills" : { 301 | "get" : { 302 | "summary" : "Skills Associated with a Skill", 303 | "description" : "Retrieves a collection of skills associated with a specified skill.", 304 | "parameters" : [ { 305 | "name" : "id", 306 | "in" : "path", 307 | "description" : "The UUID of the skill to retrieve related skills for", 308 | "required" : true, 309 | "type" : "string" 310 | } ], 311 | "responses" : { 312 | "200" : { 313 | "description" : "A skill and its related skills", 314 | "schema" : { 315 | "$ref" : "#/definitions/SkillRelatedSkills" 316 | } 317 | }, 318 | "default" : { 319 | "description" : "Unexpected error", 320 | "schema" : { 321 | "$ref" : "#/definitions/Error" 322 | } 323 | } 324 | } 325 | } 326 | }, 327 | "/skills/autocomplete" : { 328 | "get" : { 329 | "summary" : "Skill Name Autocomplete", 330 | "description" : "Retrieves the names, descriptions, and UUIDs of all skills matching a given search criteria.", 331 | "parameters" : [ { 332 | "name" : "begins_with", 333 | "in" : "query", 334 | "description" : "Find skill names beginning with the given text fragment", 335 | "required" : false, 336 | "type" : "string" 337 | }, { 338 | "name" : "contains", 339 | "in" : "query", 340 | "description" : "Find skill names containing the given text fragment", 341 | "required" : false, 342 | "type" : "string" 343 | }, { 344 | "name" : "ends_with", 345 | "in" : "query", 346 | "description" : "Find skill names ending with the given text fragment", 347 | "required" : false, 348 | "type" : "string" 349 | } ], 350 | "responses" : { 351 | "200" : { 352 | "description" : "A collection of skills", 353 | "schema" : { 354 | "$ref" : "#/definitions/SkillJobs" 355 | } 356 | }, 357 | "default" : { 358 | "description" : "Unexpected error", 359 | "schema" : { 360 | "$ref" : "#/definitions/Error" 361 | } 362 | } 363 | } 364 | } 365 | }, 366 | "/skills/normalize" : { 367 | "get" : { 368 | "summary" : "Skill Name Normalization", 369 | "description" : "Retrieves the canonical skill name for a synonymous skill name", 370 | "parameters" : [ { 371 | "name" : "skill_name", 372 | "in" : "query", 373 | "description" : "Find the canonical skill name(s) for skills matching the given text fragment", 374 | "required" : true, 375 | "type" : "string" 376 | } ], 377 | "responses" : { 378 | "200" : { 379 | "description" : "A collection of normalized skills", 380 | "schema" : { 381 | "$ref" : "#/definitions/NormalizedSkills" 382 | } 383 | } 384 | } 385 | } 386 | } 387 | }, 388 | "definitions" : { 389 | "Jobs" : { 390 | "type" : "array", 391 | "items" : { 392 | "$ref" : "#/definitions/Job" 393 | }, 394 | "properties" : { 395 | "links" : { 396 | "type" : "array", 397 | "items" : { 398 | "$ref" : "#/definitions/PageLink" 399 | } 400 | } 401 | } 402 | }, 403 | "Job" : { 404 | "properties" : { 405 | "uuid" : { 406 | "type" : "string", 407 | "description" : "Universally Unique Identifier for the job" 408 | }, 409 | "title" : { 410 | "type" : "string", 411 | "description" : "Job title" 412 | }, 413 | "normalized_job_title" : { 414 | "type" : "string", 415 | "description" : "Normalized job title" 416 | }, 417 | "parent_uuid" : { 418 | "type" : "string", 419 | "description" : "UUID for the job's parent job category" 420 | } 421 | } 422 | }, 423 | "NormalizedJobs" : { 424 | "type" : "array", 425 | "items" : { 426 | "$ref" : "#/definitions/NormalizedJob" 427 | } 428 | }, 429 | "JobSkills" : { 430 | "properties" : { 431 | "job_uuid" : { 432 | "type" : "string", 433 | "description" : "Universally Unique Identifier for the job" 434 | }, 435 | "job_title" : { 436 | "type" : "string", 437 | "description" : "Title of the job associated with the UUID" 438 | }, 439 | "normalized_job_title" : { 440 | "type" : "string", 441 | "description" : "Normalized title of the job associated with the UUID" 442 | }, 443 | "skills" : { 444 | "type" : "array", 445 | "items" : { 446 | "$ref" : "#/definitions/SkillJob" 447 | } 448 | } 449 | } 450 | }, 451 | "NormalizedJob" : { 452 | "properties" : { 453 | "uuid" : { 454 | "type" : "string", 455 | "description" : "Universally Unique Identifier for the synonymous job title" 456 | }, 457 | "title" : { 458 | "type" : "string", 459 | "description" : "Job title for the synonymous job title" 460 | }, 461 | "relevance_score" : { 462 | "type" : "string", 463 | "description" : "Relevance score for job title." 464 | }, 465 | "parent_uuid" : { 466 | "type" : "string", 467 | "description" : "Universal Unique Identifier for the canonical job title" 468 | } 469 | } 470 | }, 471 | "JobSkill" : { 472 | "properties" : { 473 | "job_uuid" : { 474 | "type" : "string", 475 | "description" : "Universally Unique Identifier for the job" 476 | }, 477 | "job_title" : { 478 | "type" : "string", 479 | "description" : "Job title" 480 | }, 481 | "normalized_job_title" : { 482 | "type" : "string", 483 | "description" : "Normalized job title" 484 | }, 485 | "importance" : { 486 | "type" : "number", 487 | "description" : "O*NET importance score indicating how important skill is to job." 488 | }, 489 | "level" : { 490 | "type" : "number", 491 | "description" : "O*NET level score indicating the skill level required for the job." 492 | } 493 | } 494 | }, 495 | "JobRelatedJobs" : { 496 | "properties" : { 497 | "uuid" : { 498 | "type" : "string", 499 | "description" : "Universally Unique Identifier for the job" 500 | }, 501 | "related_job_titles" : { 502 | "type" : "array", 503 | "items" : { 504 | "$ref" : "#/definitions/JobRelatedJob" 505 | } 506 | } 507 | } 508 | }, 509 | "JobRelatedJob" : { 510 | "properties" : { 511 | "uuid" : { 512 | "type" : "string", 513 | "description" : "Universally Unique Identifier for the job" 514 | }, 515 | "title" : { 516 | "type" : "string", 517 | "description" : "Job title" 518 | }, 519 | "parent_uuid" : { 520 | "type" : "string", 521 | "description" : "Universally Unique Identifier for the job's canonical job title" 522 | } 523 | } 524 | }, 525 | "Skills" : { 526 | "type" : "array", 527 | "items" : { 528 | "$ref" : "#/definitions/Skill" 529 | }, 530 | "properties" : { 531 | "links" : { 532 | "type" : "array", 533 | "items" : { 534 | "$ref" : "#/definitions/PageLink" 535 | } 536 | } 537 | } 538 | }, 539 | "Skill" : { 540 | "properties" : { 541 | "uuid" : { 542 | "type" : "string", 543 | "description" : "Universally Unique Identifier for the skill" 544 | }, 545 | "name" : { 546 | "type" : "string", 547 | "description" : "Skill name" 548 | }, 549 | "onet_element_id" : { 550 | "type" : "string", 551 | "description" : "O*NET Element Identifier" 552 | }, 553 | "normalized_skill_name" : { 554 | "type" : "string", 555 | "description" : "Normalized skill name" 556 | } 557 | } 558 | }, 559 | "NormalizedSkills" : { 560 | "type" : "array", 561 | "items" : { 562 | "$ref" : "#/definitions/NormalizedSkill" 563 | } 564 | }, 565 | "NormalizedSkill" : { 566 | "properties" : { 567 | "uuid" : { 568 | "type" : "string", 569 | "description" : "Universally Unique Identifier for the canonical skill name" 570 | }, 571 | "skill_name" : { 572 | "type" : "string", 573 | "description" : "Canonical skill name" 574 | } 575 | } 576 | }, 577 | "SkillJobs" : { 578 | "properties" : { 579 | "skill_uuid" : { 580 | "type" : "string", 581 | "description" : "Universally Unique Identifier for the job" 582 | }, 583 | "skill_name" : { 584 | "type" : "string", 585 | "description" : "Title of the job associated with the UUID" 586 | }, 587 | "normalized_job_title" : { 588 | "type" : "string", 589 | "description" : "Normalized title of the job associated with the UUID" 590 | }, 591 | "jobs" : { 592 | "type" : "array", 593 | "items" : { 594 | "$ref" : "#/definitions/JobSkill" 595 | } 596 | } 597 | } 598 | }, 599 | "SkillJob" : { 600 | "properties" : { 601 | "skill_uuid" : { 602 | "type" : "string", 603 | "description" : "Universally Unique Identifier for the skill" 604 | }, 605 | "skill_name" : { 606 | "type" : "string", 607 | "description" : "Name of the skill" 608 | }, 609 | "description" : { 610 | "type" : "string", 611 | "description" : "Description of the skill" 612 | }, 613 | "normalized_skill_name" : { 614 | "type" : "string", 615 | "description" : "Normalized skill name" 616 | }, 617 | "importance" : { 618 | "type" : "number", 619 | "description" : "O*NET importance score" 620 | }, 621 | "level" : { 622 | "type" : "number", 623 | "description" : "O*NET level score" 624 | } 625 | } 626 | }, 627 | "SkillRelatedSkills" : { 628 | "properties" : { 629 | "uuid" : { 630 | "type" : "string", 631 | "description" : "Universally Unique Identifier for the skills" 632 | }, 633 | "related_skill_name" : { 634 | "type" : "array", 635 | "items" : { 636 | "$ref" : "#/definitions/SkillRelatedSkill" 637 | } 638 | } 639 | } 640 | }, 641 | "SkillRelatedSkill" : { 642 | "properties" : { 643 | "uuid" : { 644 | "type" : "string", 645 | "description" : "Universally Unique Identifier for the skill" 646 | }, 647 | "skill_name" : { 648 | "type" : "string", 649 | "description" : "Skill name" 650 | } 651 | } 652 | }, 653 | "PageLink" : { 654 | "properties" : { 655 | "rel" : { 656 | "type" : "string", 657 | "description" : "Link descriptor (e.g. self, first, prev, next, last)" 658 | }, 659 | "href" : { 660 | "type" : "string", 661 | "description" : "Link URI" 662 | } 663 | } 664 | }, 665 | "Error" : { 666 | "properties" : { 667 | "code" : { 668 | "type" : "integer", 669 | "format" : "int32" 670 | }, 671 | "message" : { 672 | "type" : "string" 673 | } 674 | } 675 | } 676 | } 677 | } -------------------------------------------------------------------------------- /api/v1/endpoints.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """API Version 1 Endpoints 4 | 5 | This module contains all the implementation logic for the Version 1 API. 6 | In order to maintain consistency, it mirrors the endpoints module in the 7 | router package. 8 | 9 | Note: 10 | All classes in this module inherit from the Flask RESTful Resource parent 11 | class. 12 | 13 | """ 14 | 15 | import hashlib 16 | import math 17 | import random 18 | from app.app import db, es 19 | from flask import abort, request 20 | from flask_restful import Resource 21 | from common.utils import create_response, create_error 22 | from . models.jobs_master import JobMaster 23 | from . models.skills_master import SkillMaster 24 | from . models.jobs_alternate_titles import JobAlternateTitle 25 | from . models.jobs_unusual_titles import JobUnusualTitle 26 | from . models.jobs_skills import JobSkill 27 | from . models.skills_importance import SkillImportance 28 | from . models.geographies import Geography 29 | from . models.jobs_importance import JobImportance 30 | from . models.geo_title_count import GeoTitleCount 31 | from . models.title_count import TitleCount 32 | from collections import OrderedDict 33 | 34 | # Pagination Control Parameters 35 | MAX_PAGINATION_LIMIT = 500 36 | DEFAULT_PAGINATION_LIMIT = 20 37 | 38 | # ------------------------------------------------------------------------ 39 | # Helper Methods 40 | # ------------------------------------------------------------------------ 41 | 42 | def fake_relevance_score(): 43 | """Return a fake relevance score between 0 and 1.""" 44 | return round(random.uniform(0,1),3) 45 | 46 | def compute_offset(page, items_per_page): 47 | """Calculate the offset value to use for pagination. 48 | 49 | Args: 50 | page (int): The current page to compute the offset from. 51 | items_per_page (int): Number of items per page. 52 | 53 | """ 54 | return (page - 1) * items_per_page 55 | 56 | def compute_page(offset, items_per_page): 57 | """Calculate the current page number based on offset. 58 | 59 | Args: 60 | offset (int): The offset to use for calculating the page. 61 | items_per_page (int): Number of items per page. 62 | 63 | """ 64 | return int(math.ceil(offset / items_per_page)) + 1 65 | 66 | def get_limit_and_offset(args): 67 | """Calculate the limit and offset to use for pagination 68 | 69 | Args: 70 | args (dict): All parameters passed in via the HTTP request. 71 | 72 | """ 73 | limit = 0 74 | offset = 0 75 | if args is not None: 76 | if 'offset' in args.keys(): 77 | try: 78 | offset = int(args['offset']) 79 | if offset < 0: 80 | offset = 0 81 | except: 82 | offset = 0 83 | else: 84 | offset = 0 85 | 86 | if 'limit' in args.keys(): 87 | try: 88 | limit = int(args['limit']) 89 | if limit < 0: 90 | limit = DEFAULT_PAGINATION_LIMIT 91 | except: 92 | limit = DEFAULT_PAGINATION_LIMIT 93 | else: 94 | limit = DEFAULT_PAGINATION_LIMIT 95 | else: 96 | offset = 0 97 | limit = DEFAULT_PAGINATION_LIMIT 98 | 99 | if limit > MAX_PAGINATION_LIMIT: 100 | limit = MAX_PAGINATION_LIMIT 101 | 102 | return limit, offset 103 | 104 | # ------------------------------------------------------------------------ 105 | # API Version 1 Endpoints 106 | # ------------------------------------------------------------------------ 107 | class AllJobsEndpoint(Resource): 108 | """All Jobs Endpoint Class""" 109 | 110 | def get(self): 111 | """GET operation for the endpoint class. 112 | 113 | Returns: 114 | A collection of jobs. 115 | 116 | Notes: 117 | The endpoint supports pagination. 118 | 119 | """ 120 | 121 | args = request.args 122 | limit, offset = get_limit_and_offset(args) 123 | 124 | all_jobs = [] 125 | links = OrderedDict() 126 | links['links'] = [] 127 | jobs = JobAlternateTitle.query.order_by(JobAlternateTitle.title.asc()).limit(limit).offset(offset) 128 | rows = JobAlternateTitle.query.count() 129 | 130 | # compute pages 131 | url_link = '/jobs?offset={}&limit={}' 132 | custom_headers = [] 133 | custom_headers.append('X-Total-Count = ' + str(rows)) 134 | 135 | total_pages = int(math.ceil(rows / limit)) 136 | current_page = compute_page(offset, limit) 137 | first = OrderedDict() 138 | prev = OrderedDict() 139 | next = OrderedDict() 140 | last = OrderedDict() 141 | current = OrderedDict() 142 | 143 | current['rel'] = 'self' 144 | current['href'] = url_link.format(str(offset), str(limit)) 145 | links['links'].append(current) 146 | 147 | first['rel'] = 'first' 148 | first['href'] = url_link.format(str(compute_offset(1, limit)), str(limit)) 149 | links['links'].append(first) 150 | 151 | if current_page > 1: 152 | prev['rel'] = 'prev' 153 | prev['href'] = url_link.format(str(compute_offset(current_page - 1, limit)), str(limit)) 154 | links['links'].append(prev) 155 | 156 | if current_page < total_pages: 157 | next['rel'] = 'next' 158 | next['href'] = url_link.format(str(compute_offset(current_page + 1, limit)), str(limit)) 159 | links['links'].append(next) 160 | 161 | last['rel'] = 'last' 162 | last['href'] = url_link.format(str(compute_offset(total_pages, limit)), str(limit)) 163 | links['links'].append(last) 164 | 165 | if jobs is not None: 166 | for job in jobs: 167 | job_response = OrderedDict() 168 | job_response['uuid'] = job.uuid 169 | job_response['title'] = job.title 170 | job_response['normalized_job_title'] = job.nlp_a 171 | job_response['parent_uuid'] = job.job_uuid 172 | all_jobs.append(job_response) 173 | 174 | all_jobs.append(links) 175 | 176 | return create_response(all_jobs, 200, custom_headers) 177 | else: 178 | return create_error('No jobs were found', 404) 179 | 180 | class AllSkillsEndpoint(Resource): 181 | """All Skills Endpoint Class""" 182 | 183 | def get(self): 184 | """GET operation for the endpoint class. 185 | 186 | Returns: 187 | A collection of skills. 188 | 189 | Notes: 190 | The endpoint supports pagination. 191 | 192 | """ 193 | args = request.args 194 | limit, offset = get_limit_and_offset(args) 195 | 196 | all_skills = [] 197 | links = OrderedDict() 198 | links['links'] = [] 199 | 200 | skills = SkillMaster.query.order_by(SkillMaster.skill_name.asc()).limit(limit).offset(offset) 201 | rows = SkillMaster.query.count() 202 | 203 | # compute pages 204 | url_link = '/skills?offset={}&limit={}' 205 | custom_headers = [] 206 | custom_headers.append('X-Total-Count = ' + str(rows)) 207 | 208 | total_pages = int(math.ceil(rows / limit)) 209 | current_page = compute_page(offset, limit) 210 | first = OrderedDict() 211 | prev = OrderedDict() 212 | next = OrderedDict() 213 | last = OrderedDict() 214 | current = OrderedDict() 215 | 216 | current['rel'] = 'self' 217 | current['href'] = url_link.format(str(offset), str(limit)) 218 | links['links'].append(current) 219 | 220 | first['rel'] = 'first' 221 | first['href'] = url_link.format(str(compute_offset(1, limit)), str(limit)) 222 | links['links'].append(first) 223 | 224 | if current_page > 1: 225 | prev['rel'] = 'prev' 226 | prev['href'] = url_link.format(str(compute_offset(current_page - 1, limit)), str(limit)) 227 | links['links'].append(prev) 228 | 229 | if current_page < total_pages: 230 | next['rel'] = 'next' 231 | next['href'] = url_link.format(str(compute_offset(current_page + 1, limit)), str(limit)) 232 | links['links'].append(next) 233 | 234 | last['rel'] = 'last' 235 | last['href'] = url_link.format(str(compute_offset(total_pages, limit)), str(limit)) 236 | links['links'].append(last) 237 | 238 | 239 | if skills is not None: 240 | for skill in skills: 241 | skill_response = OrderedDict() 242 | skill_response['uuid'] = skill.uuid 243 | skill_response['name'] = skill.skill_name 244 | skill_response['type'] = skill.ksa_type 245 | skill_response['description'] = skill.description 246 | skill_response['onet_element_id'] = skill.onet_element_id 247 | skill_response['normalized_skill_name'] = skill.nlp_a 248 | all_skills.append(skill_response) 249 | all_skills.append(links) 250 | 251 | return create_response(all_skills, 200, custom_headers) 252 | else: 253 | return create_error('No skills were found', 404) 254 | 255 | class JobTitleAutocompleteEndpoint(Resource): 256 | """Job Title Autocomplete Endpoint Class""" 257 | 258 | def get(self): 259 | """GET operation for the endpoint class. 260 | 261 | Returns: 262 | A collection of jobs that partially match the specified search string. 263 | 264 | """ 265 | args = request.args 266 | 267 | query_mode = '' 268 | if args is not None: 269 | if 'begins_with' in args.keys(): 270 | search_string = str(args['begins_with']) 271 | query_mode = 'begins_with' 272 | elif 'contains' in args.keys(): 273 | search_string = str(args['contains']) 274 | query_mode = 'contains' 275 | elif 'ends_with' in args.keys(): 276 | search_string = str(args['ends_with']) 277 | query_mode = 'ends_with' 278 | else: 279 | return create_error('Invalid query mode specified for job title autocomplete', 400) 280 | 281 | search_string = search_string.replace('"','').strip() 282 | all_suggestions = [] 283 | 284 | if query_mode == 'begins_with': 285 | results = JobAlternateTitle.query.filter(JobAlternateTitle.nlp_a.startswith(search_string.lower())).all() 286 | 287 | if query_mode == 'contains': 288 | results = JobAlternateTitle.query.filter(JobAlternateTitle.nlp_a.contains(search_string.lower())).all() 289 | 290 | if query_mode == 'ends_with': 291 | results = JobAlternateTitle.query.filter(JobAlternateTitle.nlp_a.endswith(search_string.lower())).all() 292 | 293 | if len(results) == 0: 294 | return create_error('No job title suggestions found', 404) 295 | 296 | for result in results: 297 | suggestion = OrderedDict() 298 | suggestion['uuid'] = result.uuid 299 | suggestion['suggestion'] = result.title 300 | suggestion['normalized_job_title'] = result.nlp_a 301 | suggestion['parent_uuid'] = result.job_uuid 302 | all_suggestions.append(suggestion) 303 | 304 | return create_response(all_suggestions, 200) 305 | else: 306 | return create_error('No job title suggestions found', 404) 307 | 308 | class SkillNameAutocompleteEndpoint(Resource): 309 | """Skill Name Autocomplete Endpoint Class""" 310 | 311 | def get(self): 312 | """GET operation for the endpoint class. 313 | 314 | Returns: 315 | A collection of skills that partially match the specified search string. 316 | 317 | """ 318 | args = request.args 319 | 320 | query_mode = '' 321 | if args is not None: 322 | if 'begins_with' in args.keys(): 323 | search_string = str(args['begins_with']) 324 | query_mode = 'begins_with' 325 | elif 'contains' in args.keys(): 326 | search_string = str(args['contains']) 327 | query_mode = 'contains' 328 | elif 'ends_with' in args.keys(): 329 | search_string = str(args['ends_with']) 330 | query_mode = 'ends_with' 331 | else: 332 | return create_error('Invalid query mode specified for skill name autocomplete', 400) 333 | 334 | search_string = search_string.replace('"','').strip() 335 | all_suggestions = [] 336 | 337 | if query_mode == 'begins_with': 338 | results = SkillMaster.query.filter(SkillMaster.nlp_a.startswith(search_string.lower())).all() 339 | 340 | if query_mode == 'contains': 341 | results = SkillMaster.query.filter(SkillMaster.nlp_a.contains(search_string.lower())).all() 342 | 343 | if query_mode == 'ends_with': 344 | results = SkillMaster.query.filter(SkillMaster.nlp_a.endswith(search_string.lower())).all() 345 | 346 | if len(results) == 0: 347 | return create_error('No skill name suggestions found', 404) 348 | 349 | for result in results: 350 | suggestion = OrderedDict() 351 | suggestion['uuid'] = result.uuid 352 | suggestion['suggestion'] = result.skill_name 353 | suggestion['normalized_skill_name'] = result.nlp_a 354 | all_suggestions.append(suggestion) 355 | 356 | return create_response(all_suggestions, 200) 357 | else: 358 | return create_error('No skill name suggestions found', 404) 359 | 360 | class JobTitleNormalizeEndpoint(Resource): 361 | """Job Title Normalize Endpoint Class""" 362 | 363 | def get(self): 364 | """GET operation for the endpoint class. 365 | 366 | Returns: 367 | A normalized version of a specified job title. 368 | 369 | """ 370 | args = request.args 371 | 372 | if args is not None: 373 | if 'limit' in args.keys(): 374 | try: 375 | limit = int(args['limit']) 376 | if limit < 0: 377 | return create_error('Limit must be a positive integer.', 400) 378 | if limit > 10: 379 | return create_error('Limit has a maximum of 10.', 400) 380 | except: 381 | return create_error('Limit must be an integer', 400) 382 | else: 383 | limit = 1 384 | 385 | if 'job_title' in args.keys(): 386 | search_string = str(args['job_title']) 387 | else: 388 | return create_error('Invalid parameter specified for job title normalization', 400) 389 | 390 | indexed_field = 'canonicaltitle' 391 | 392 | request_body = { 393 | "size": limit*10, # give us a buffer to remove duplicates 394 | "_source": [indexed_field], 395 | "query" : { 396 | "bool": { 397 | "should": [ 398 | { "multi_match": { 399 | "query": search_string, 400 | "fields": ['{}^5'.format(indexed_field), "occupation", "jobtitle"] 401 | } }, 402 | { "terms": { 403 | "jobdesc": search_string.split(' ') 404 | } } 405 | ] 406 | } 407 | } 408 | } 409 | 410 | response = es.search(index='normalize', body=request_body) 411 | results = response['hits']['hits'] 412 | 413 | if len(results) == 0: 414 | return create_error('No normalized job titles found', 404) 415 | 416 | def normalize(value): 417 | minimum = 0.0 418 | maximum = 10.0 419 | if value < minimum: 420 | return 0.0 421 | elif value > maximum: 422 | return 1.0 423 | else: 424 | return (value - minimum) / (maximum - minimum) 425 | 426 | # take unique titles until reaching the limit 427 | distinct_titles = set() 428 | titles = [] 429 | num_distinct_titles = 0 430 | for result in results: 431 | if num_distinct_titles >= limit: 432 | break 433 | if '_source' not in result or indexed_field not in result['_source']: 434 | continue 435 | title = result['_source'][indexed_field] 436 | if title in distinct_titles: 437 | continue 438 | distinct_titles.add(title) 439 | titles.append({ 440 | 'title': title, 441 | 'score': normalize(result['_score']), 442 | 'uuid': str(hashlib.md5(title).hexdigest()) 443 | }) 444 | num_distinct_titles += 1 445 | 446 | all_suggestions = [] 447 | 448 | category_fetch_stmt = JobAlternateTitle.__table__.select( 449 | JobAlternateTitle.uuid.in_([title['uuid'] for title in titles]) 450 | ) 451 | 452 | category_fetch_results = db.engine.execute(category_fetch_stmt) 453 | category_lookup = { row[0]: row[3] for row in category_fetch_results } 454 | 455 | for row in titles: 456 | suggestion = OrderedDict() 457 | suggestion['uuid'] = row['uuid'] 458 | suggestion['title'] = row['title'] 459 | suggestion['relevance_score'] = row['score'] 460 | if row['uuid'] in category_lookup: 461 | suggestion['parent_uuid'] = category_lookup[row['uuid']] 462 | else: 463 | suggestion['parent_uuid'] = '' 464 | all_suggestions.append(suggestion) 465 | 466 | return create_response(sorted(all_suggestions, key=lambda k: k['relevance_score'], reverse=True), 200) 467 | else: 468 | return create_error('No normalized job titles found', 404) 469 | 470 | class JobTitleFromONetCodeEndpoint(Resource): 471 | """Job Title From O*NET SOC Code Endpoint Class""" 472 | 473 | def get(self, id=None): 474 | """GET operation for the endpoint class. 475 | 476 | Returns: 477 | A job associated with its O*NET SOC code or UUID. 478 | 479 | Notes: 480 | This endpoint actually supports two use cases. It first checks if 481 | the identifier is a valid O*NET SOC code, if not it queries for a 482 | UUID. 483 | 484 | """ 485 | if id is not None: 486 | args = request.args 487 | 488 | if args is not None: 489 | geography = None 490 | if 'fips' in args.keys(): 491 | fips = args['fips'] 492 | geography = Geography.query.filter_by( 493 | geography_type = 'CBSA', 494 | geography_name = fips 495 | ).first() 496 | if geography is None: 497 | return create_error('Core-Based Statistical Area FIPS code not found', 404) 498 | importance = JobImportance.query.filter_by( 499 | geography_id = geography.geography_id, 500 | job_uuid = id 501 | ).first() 502 | if importance is None: 503 | return create_error('Job not found in given Core-Based statistical area', 404) 504 | 505 | result = JobMaster.query.filter_by(onet_soc_code = id).first() 506 | if result is None: 507 | result = JobMaster.query.filter_by(uuid = id).first() 508 | 509 | if result is None: 510 | # search for a related job 511 | result = JobAlternateTitle.query.filter_by(uuid = id).first() 512 | if result is not None: 513 | output = OrderedDict() 514 | output['uuid'] = result.uuid 515 | output['title'] = result.title 516 | output['normalized_job_title'] = result.nlp_a 517 | output['parent_uuid'] = result.job_uuid 518 | return create_response(output, 200) 519 | else: 520 | result = JobUnusualTitle.query.filter_by(uuid = id).first() 521 | if result is not None: 522 | output = OrderedDict() 523 | output['uuid'] = result.uuid 524 | output['title'] = result.title 525 | output['normalized_job_title'] = result.title 526 | output['parent_uuid'] = result.job_uuid 527 | return create_response(output, 200) 528 | else: 529 | return create_error('Cannot find job with id ' + id, 404) 530 | else: 531 | output = OrderedDict() 532 | output['uuid'] = result.uuid 533 | output['onet_soc_code'] = result.onet_soc_code 534 | output['title'] = result.title 535 | output['description'] = result.description 536 | output['related_job_titles'] = [] 537 | output['unusual_job_titles'] = [] 538 | 539 | # alternate job titles 540 | alt_titles = JobAlternateTitle.query.filter_by(job_uuid = result.uuid).all() 541 | for alt_title in alt_titles: 542 | title = OrderedDict() 543 | title['uuid'] = alt_title.uuid 544 | title['title'] = alt_title.title 545 | output['related_job_titles'].append(title) 546 | 547 | # unusual job titles 548 | other_titles = JobUnusualTitle.query.filter_by(job_uuid = result.uuid).all() 549 | for other_title in other_titles: 550 | title = OrderedDict() 551 | title['uuid'] = other_title.uuid 552 | title['title'] = other_title.title 553 | output['unusual_job_titles'].append(title) 554 | 555 | return create_response(output, 200) 556 | 557 | class NormalizeSkillNameEndpoint(Resource): 558 | """Normalize Skill Name Endpoint Class""" 559 | 560 | def get(self): 561 | """GET operation for the endpoint class. 562 | 563 | Returns: 564 | A normalized version of a specified skill name. 565 | 566 | """ 567 | args = request.args 568 | 569 | if args is not None: 570 | if 'skill_name' in args.keys(): 571 | search_string = str(args['skill_name']) 572 | else: 573 | return create_error('Invalid parameter specified for skill name normalization', 400) 574 | 575 | search_string = search_string.replace('"','').strip() 576 | all_suggestions = [] 577 | 578 | results = SkillMaster.query.filter(SkillMaster.skill_name.contains(search_string)).all() 579 | 580 | if len(results) == 0: 581 | return create_error('No normalized skill names found', 404) 582 | 583 | for result in results: 584 | suggestion = OrderedDict() 585 | suggestion['uuid'] = result.uuid 586 | suggestion['skill_name'] = result.skill_name 587 | all_suggestions.append(suggestion) 588 | 589 | return create_response(all_suggestions, 200) 590 | else: 591 | return create_error('No normalized skill names found', 404) 592 | 593 | class AssociatedSkillsForJobEndpoint(Resource): 594 | """Associated Skills For Job Endpoint Class""" 595 | 596 | def get(self, id=None): 597 | """GET operation for the endpoint class. 598 | 599 | Returns: 600 | A collection of skills associated with a particular job UUID. 601 | 602 | """ 603 | if id is not None: 604 | #results = JobSkill.query.filter_by(job_uuid = id).all() 605 | results = SkillImportance.query.filter_by(job_uuid = id).all() 606 | job = JobMaster.query.filter_by(uuid = id).first() 607 | if not results: 608 | parent_uuid = None 609 | job = JobAlternateTitle.query.filter_by(uuid = id).first() 610 | if job: 611 | parent_uuid = job.job_uuid 612 | else: 613 | job = JobUnusualTitle.query.filter_by(uuid = id).first() 614 | if job: 615 | parent_uuid = job.job_uuid 616 | 617 | if parent_uuid is not None: 618 | #results = JobSkill.query.filter_by(job_uuid = parent_uuid).all() 619 | results = SkillImportance.query.filter_by(job_uuid = parent_uuid).all() 620 | 621 | if len(results) > 0: 622 | all_skills = OrderedDict() 623 | all_skills['job_uuid'] = id 624 | all_skills['job_title'] = job.title 625 | all_skills['normalized_job_title'] = job.nlp_a 626 | all_skills['skills'] = [] 627 | for result in results: 628 | skill = OrderedDict() 629 | skill_desc = SkillMaster.query.filter_by(uuid = result.skill_uuid).first() 630 | skill['skill_uuid'] = result.skill_uuid 631 | skill['skill_name'] = skill_desc.skill_name 632 | skill['skill_type'] = skill_desc.ksa_type 633 | skill['description'] = skill_desc.description 634 | skill['normalized_skill_name'] = skill_desc.nlp_a 635 | skill['importance'] = result.importance 636 | skill['level'] = result.level 637 | all_skills['skills'].append(skill) 638 | 639 | all_skills['skills'] = sorted(all_skills['skills'], key=lambda k: k['importance'], reverse=True) 640 | 641 | return create_response(all_skills, 200) 642 | else: 643 | return create_error('No associated skills found for job ' + id, 404) 644 | else: 645 | return create_error('No job UUID specified for query', 400) 646 | 647 | class AssociatedJobsForSkillEndpoint(Resource): 648 | """Associated Jobs For Skill Endpoint Class""" 649 | 650 | def get(self, id=None): 651 | """GET operation for the endpoint class. 652 | 653 | Returns: 654 | A collection of jobs associated with a specified skill UUID. 655 | 656 | """ 657 | if id is not None: 658 | #results = JobSkill.query.filter_by(skill_uuid = id).all() 659 | results = SkillImportance.query.filter_by(skill_uuid = id).all() 660 | if len(results) > 0: 661 | all_jobs = OrderedDict() 662 | skill = SkillMaster.query.filter_by(uuid = id).first() 663 | all_jobs['skill_uuid'] = id 664 | all_jobs['skill_name'] = skill.skill_name 665 | all_jobs['normalized_skill_name'] = skill.nlp_a 666 | all_jobs['jobs'] = [] 667 | for result in results: 668 | job = OrderedDict() 669 | job_desc = JobMaster.query.filter_by(uuid = result.job_uuid).first() 670 | job['job_uuid'] = result.job_uuid 671 | job['job_title'] = job_desc.title 672 | job['normalized_job_title'] = job_desc.nlp_a 673 | job['importance'] = result.importance 674 | job['level'] = result.level 675 | all_jobs['jobs'].append(job) 676 | 677 | all_jobs['jobs'] = sorted(all_jobs['jobs'], key=lambda k: k['importance'], reverse=True) 678 | return create_response(all_jobs, 200) 679 | else: 680 | return create_error('No associated jobs found for skill ' + id, 404) 681 | else: 682 | return create_error('No skill UUID specified for query', 400) 683 | 684 | class AssociatedJobsForJobEndpoint(Resource): 685 | """Associated Jobs For Job Endpoint Class""" 686 | 687 | def get(self, id=None): 688 | """GET operation for the endpoint class. 689 | 690 | Returns: 691 | A collection of jobs associated with a specified job UUID. 692 | 693 | """ 694 | if id is not None: 695 | parent_uuid = None 696 | result = JobMaster.query.filter_by(uuid = id).first() 697 | if result is None: 698 | result = JobAlternateTitle.query.filter_by(uuid = id).first() 699 | if result is None: 700 | result = JobUnusualTitle.query.filter_by(uuid = id).first() 701 | if result is not None: 702 | parent_uuid = result.job_uuid 703 | else: 704 | return create_error('No job found matching the specified uuid ' + id, 404) 705 | else: 706 | parent_uuid = result.job_uuid 707 | else: 708 | parent_uuid = result.uuid 709 | 710 | output = OrderedDict() 711 | output['related_job_titles'] = [] 712 | output['unusual_job_titles'] = [] 713 | 714 | # alternate job titles 715 | alt_titles = JobAlternateTitle.query.filter_by(job_uuid = parent_uuid).all() 716 | for alt_title in alt_titles: 717 | title = OrderedDict() 718 | title['uuid'] = alt_title.uuid 719 | title['title'] = alt_title.title 720 | title['parent_uuid'] = parent_uuid 721 | output['related_job_titles'].append(title) 722 | 723 | # unusual job titles 724 | other_titles = JobUnusualTitle.query.filter_by(job_uuid = parent_uuid).all() 725 | for other_title in other_titles: 726 | title = OrderedDict() 727 | title['uuid'] = other_title.uuid 728 | title['title'] = other_title.title 729 | title['parent_uuid'] = parent_uuid 730 | output['unusual_job_titles'].append(title) 731 | 732 | 733 | return create_response(output, 200) 734 | else: 735 | return create_error('No Job UUID specified for query', 400) 736 | 737 | class AssociatedSkillForSkillEndpoint(Resource): 738 | """Associated Skill For Skills Endpoint Class""" 739 | 740 | def get(self, id=None): 741 | """GET operation for the endpoint class. 742 | 743 | Returns: 744 | A collection of skills associated with a specified skill UUID. 745 | 746 | """ 747 | if id is not None: 748 | result = SkillMaster.query.filter_by(uuid = id).first() 749 | if result is not None: 750 | all_skills = OrderedDict() 751 | skills = SkillMaster.query.filter(SkillMaster.skill_name.contains(result.skill_name)).all() 752 | if len(skills) > 0: 753 | all_skills['skills'] = [] 754 | for skill in skills: 755 | output = OrderedDict() 756 | output['uuid'] = skill.uuid 757 | output['skill_name'] = skill.skill_name 758 | output['skill_type'] = skill.ksa_type 759 | output['normalized_skill_name'] = skill.nlp_a 760 | all_skills['skills'].append(output) 761 | return create_response(all_skills, 200) 762 | else: 763 | return create_error('Cannot find skills associated with id ' + id, 404) 764 | else: 765 | return create_error('No skill UUID specified for query', 400) 766 | 767 | class SkillNameAndFrequencyEndpoint(Resource): 768 | """Skill Name And Frequency Endpoint Class""" 769 | 770 | def get(self, id=None): 771 | """GET operation for the endpoint class. 772 | 773 | Returns: 774 | The name and frequency of all skills. 775 | 776 | """ 777 | if id is not None: 778 | result = SkillMaster.query.filter_by(uuid = id).first() 779 | if result is None: 780 | all_skills = OrderedDict() 781 | job = JobMaster.query.filter_by(onet_soc_code = id).first() 782 | if job is not None: 783 | search_uuid = job.uuid 784 | else: 785 | return create_error('Cannot find skills associated with id ' + id, 404) 786 | 787 | #results = JobSkill.query.filter_by(job_uuid = search_uuid).all() 788 | results = SkillImportance.query.filter_by(job_uuid = search_uuid).all() 789 | if len(results) == 0: 790 | return create_error('Cannot find skills associated with id ' + id, 404) 791 | else: 792 | all_skills['onet_soc_code'] = id 793 | all_skills['job_uuid'] = search_uuid 794 | all_skills['title'] = job.title 795 | all_skills['skills'] = [] 796 | for result in results: 797 | output = OrderedDict() 798 | output['skill_uuid'] = result.skill_uuid 799 | all_skills['skills'].append(output) 800 | 801 | return create_response(all_skills, 200) 802 | else: 803 | output = OrderedDict() 804 | output['uuid'] = result.uuid 805 | output['skill_name'] = result.skill_name 806 | output['description'] = result.description 807 | output['normalized_skill_name'] = result.nlp_a 808 | 809 | return create_response(output, 200) 810 | 811 | class AllUnusualJobsEndpoint(Resource): 812 | """All Unusual Jobs Endpoint Class""" 813 | 814 | def get(self): 815 | """GET operation for the endpoint class. 816 | 817 | Returns: 818 | A collection of job titles that fall outside the standard titles used for particular jobs. 819 | 820 | """ 821 | all_jobs = [] 822 | jobs = JobUnusualTitle.query.order_by(JobUnusualTitle.title.asc()).all() 823 | if jobs is not None: 824 | for job in jobs: 825 | job_response = OrderedDict() 826 | job_response['uuid'] = job.uuid 827 | job_response['title'] = job.title 828 | job_response['description'] = job.description 829 | job_response['job_uuid'] = job.job_uuid 830 | all_jobs.append(job_response) 831 | 832 | return create_response(all_jobs, 200) 833 | else: 834 | return create_error('No jobs were found', 404) 835 | 836 | 837 | class TitleCountsEndpoint(Resource): 838 | """All Jobs Endpoint Class""" 839 | 840 | def get(self): 841 | """GET operation for the endpoint class. 842 | 843 | Returns: 844 | A collection of jobs. 845 | 846 | Notes: 847 | The endpoint supports pagination. 848 | 849 | """ 850 | 851 | args = request.args 852 | limit, offset = get_limit_and_offset(args) 853 | 854 | all_jobs = [] 855 | links = OrderedDict() 856 | links['links'] = [] 857 | 858 | 859 | if args is not None: 860 | geography = None 861 | if 'fips' in args.keys(): 862 | fips = args['fips'] 863 | geography = Geography.query.filter_by( 864 | geography_type = 'CBSA', 865 | geography_name = fips 866 | ).first() 867 | if geography is None: 868 | return create_error('Core-Based Statistical Area FIPS code not found', 404) 869 | base_stmt = '''select 870 | tc.job_title, 871 | tc.job_uuid, 872 | round(avg(gtc.count), 2) 873 | from 874 | title_counts tc 875 | join geo_title_counts gtc using (job_uuid, quarter_id) 876 | where geography_id = %(geography_id)s 877 | group by 1, 2 order by 3 desc 878 | ''' 879 | job_results = db.engine.execute( 880 | '''{} 881 | limit %(limit)s 882 | offset %(offset)s 883 | '''.format(base_stmt), 884 | geography_id=geography.geography_id, 885 | limit=limit, 886 | offset=offset 887 | ) 888 | rows = [row[0] for row in db.engine.execute( 889 | 'select count(*) from ({}) q'.format(base_stmt), 890 | geography_id=geography.geography_id, 891 | )][0] 892 | else: 893 | base_stmt = '''select 894 | tc.job_title, 895 | tc.job_uuid, 896 | round(avg(tc.count), 2) 897 | from title_counts tc 898 | group by 1, 2 order by 3 desc 899 | ''' 900 | job_results = db.engine.execute( 901 | '''{} 902 | limit %(limit)s 903 | offset %(offset)s 904 | '''.format(base_stmt), 905 | limit=limit, 906 | offset=offset 907 | ) 908 | rows = [row[0] for row in db.engine.execute( 909 | 'select count(*) from ({}) q'.format(base_stmt) 910 | )][0] 911 | 912 | 913 | # compute pages 914 | url_link = '/title_counts?offset={}&limit={}' 915 | custom_headers = [] 916 | custom_headers.append('X-Total-Count = ' + str(rows)) 917 | 918 | total_pages = int(math.ceil(rows / limit)) 919 | current_page = compute_page(offset, limit) 920 | first = OrderedDict() 921 | prev = OrderedDict() 922 | next = OrderedDict() 923 | last = OrderedDict() 924 | current = OrderedDict() 925 | 926 | current['rel'] = 'self' 927 | current['href'] = url_link.format(str(offset), str(limit)) 928 | links['links'].append(current) 929 | 930 | first['rel'] = 'first' 931 | first['href'] = url_link.format(str(compute_offset(1, limit)), str(limit)) 932 | links['links'].append(first) 933 | 934 | if current_page > 1: 935 | prev['rel'] = 'prev' 936 | prev['href'] = url_link.format(str(compute_offset(current_page - 1, limit)), str(limit)) 937 | links['links'].append(prev) 938 | 939 | if current_page < total_pages: 940 | next['rel'] = 'next' 941 | next['href'] = url_link.format(str(compute_offset(current_page + 1, limit)), str(limit)) 942 | links['links'].append(next) 943 | 944 | last['rel'] = 'last' 945 | last['href'] = url_link.format(str(compute_offset(total_pages, limit)), str(limit)) 946 | links['links'].append(last) 947 | 948 | if job_results is not None: 949 | for job in job_results: 950 | title, job_uuid, count = job 951 | job_response = OrderedDict() 952 | job_response['uuid'] = job_uuid 953 | job_response['title'] = title 954 | job_response['count'] = float(count) 955 | job_response['normalized_job_title'] = None 956 | job_response['parent_uuid'] = None 957 | all_jobs.append(job_response) 958 | 959 | all_jobs.append(links) 960 | 961 | return create_response(all_jobs, 200, custom_headers) 962 | else: 963 | return create_error('No jobs were found', 404) 964 | --------------------------------------------------------------------------------