├── .gitignore ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── data_dictionary.pdf ├── data_lake ├── __init__.py ├── data_util.py ├── glassdoor_jobs.py ├── indeed_jobs.py ├── lake.cfg ├── linkedin_jobs.py ├── process_data.py └── stackoverflow_dev.py ├── data_pipeline ├── create_tables.sql ├── dags │ └── jobs_dag.py └── plugins │ ├── __init__.py │ └── operators │ ├── __init__.py │ ├── copy_redshift.py │ └── data_quality.py ├── readme_files ├── company_logos.png ├── data_model.png ├── data_pipeline.png ├── developer_tables.png ├── fetch_details_table.png ├── glassdoor_data_schema.png ├── indeed_data_schema.png ├── job_analysis_architecture.png ├── job_location_table.png ├── job_rating_table.png ├── job_salary_table.png ├── job_sector_table.png ├── job_table_file.png └── linkedin_data_schema.png ├── requirements.txt ├── sample_data ├── Data_Science_Jobs_Glassdoor │ ├── glassdoor_1.csv │ ├── glassdoor_2.csv │ └── glassdoor_3.csv ├── Data_Science_Jobs_Indeed │ ├── indeed_1.csv │ ├── indeed_2.csv │ ├── indeed_3.csv │ └── indeed_4.csv ├── Data_Science_Jobs_Linkedin │ ├── linkedin_1.csv │ └── linkedin_2.csv ├── Data_Science_Jobs_StackOverflow │ └── stackoverflow_1.csv └── output │ ├── developers │ └── df_devloper_stackoverflow.csv │ ├── job_data │ ├── df_jobs_glassdoor.csv │ ├── df_jobs_indeed.csv │ └── df_jobs_linkedin.csv │ ├── job_date_details │ ├── df_job_date_fetch_glassdoor.csv │ ├── df_job_date_fetch_indeed.csv │ └── df_job_date_fetch_linkedin.csv │ ├── job_location │ ├── df_job_location_glassdoor.csv │ └── df_job_location_linkedin.csv │ ├── job_rating │ ├── df_job_reviews_glassdoor.csv │ └── df_job_reviews_indeed.csv │ ├── job_salary │ └── df_job_salary_glassdoor.csv │ └── job_sector │ └── df_job_sectors_glassdoor.csv ├── script └── airflow_start.sh └── setup.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/LICENSE.md -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/README.md -------------------------------------------------------------------------------- /data_dictionary.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_dictionary.pdf -------------------------------------------------------------------------------- /data_lake/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_lake/data_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_lake/data_util.py -------------------------------------------------------------------------------- /data_lake/glassdoor_jobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_lake/glassdoor_jobs.py -------------------------------------------------------------------------------- /data_lake/indeed_jobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_lake/indeed_jobs.py -------------------------------------------------------------------------------- /data_lake/lake.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_lake/lake.cfg -------------------------------------------------------------------------------- /data_lake/linkedin_jobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_lake/linkedin_jobs.py -------------------------------------------------------------------------------- /data_lake/process_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_lake/process_data.py -------------------------------------------------------------------------------- /data_lake/stackoverflow_dev.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_lake/stackoverflow_dev.py -------------------------------------------------------------------------------- /data_pipeline/create_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_pipeline/create_tables.sql -------------------------------------------------------------------------------- /data_pipeline/dags/jobs_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_pipeline/dags/jobs_dag.py -------------------------------------------------------------------------------- /data_pipeline/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_pipeline/plugins/__init__.py -------------------------------------------------------------------------------- /data_pipeline/plugins/operators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_pipeline/plugins/operators/__init__.py -------------------------------------------------------------------------------- /data_pipeline/plugins/operators/copy_redshift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_pipeline/plugins/operators/copy_redshift.py -------------------------------------------------------------------------------- /data_pipeline/plugins/operators/data_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/data_pipeline/plugins/operators/data_quality.py -------------------------------------------------------------------------------- /readme_files/company_logos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/company_logos.png -------------------------------------------------------------------------------- /readme_files/data_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/data_model.png -------------------------------------------------------------------------------- /readme_files/data_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/data_pipeline.png -------------------------------------------------------------------------------- /readme_files/developer_tables.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/developer_tables.png -------------------------------------------------------------------------------- /readme_files/fetch_details_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/fetch_details_table.png -------------------------------------------------------------------------------- /readme_files/glassdoor_data_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/glassdoor_data_schema.png -------------------------------------------------------------------------------- /readme_files/indeed_data_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/indeed_data_schema.png -------------------------------------------------------------------------------- /readme_files/job_analysis_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/job_analysis_architecture.png -------------------------------------------------------------------------------- /readme_files/job_location_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/job_location_table.png -------------------------------------------------------------------------------- /readme_files/job_rating_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/job_rating_table.png -------------------------------------------------------------------------------- /readme_files/job_salary_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/job_salary_table.png -------------------------------------------------------------------------------- /readme_files/job_sector_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/job_sector_table.png -------------------------------------------------------------------------------- /readme_files/job_table_file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/job_table_file.png -------------------------------------------------------------------------------- /readme_files/linkedin_data_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/readme_files/linkedin_data_schema.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/requirements.txt -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Glassdoor/glassdoor_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Glassdoor/glassdoor_1.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Glassdoor/glassdoor_2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Glassdoor/glassdoor_2.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Glassdoor/glassdoor_3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Glassdoor/glassdoor_3.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Indeed/indeed_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Indeed/indeed_1.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Indeed/indeed_2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Indeed/indeed_2.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Indeed/indeed_3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Indeed/indeed_3.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Indeed/indeed_4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Indeed/indeed_4.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Linkedin/linkedin_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Linkedin/linkedin_1.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_Linkedin/linkedin_2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_Linkedin/linkedin_2.csv -------------------------------------------------------------------------------- /sample_data/Data_Science_Jobs_StackOverflow/stackoverflow_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/Data_Science_Jobs_StackOverflow/stackoverflow_1.csv -------------------------------------------------------------------------------- /sample_data/output/developers/df_devloper_stackoverflow.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/developers/df_devloper_stackoverflow.csv -------------------------------------------------------------------------------- /sample_data/output/job_data/df_jobs_glassdoor.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_data/df_jobs_glassdoor.csv -------------------------------------------------------------------------------- /sample_data/output/job_data/df_jobs_indeed.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_data/df_jobs_indeed.csv -------------------------------------------------------------------------------- /sample_data/output/job_data/df_jobs_linkedin.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_data/df_jobs_linkedin.csv -------------------------------------------------------------------------------- /sample_data/output/job_date_details/df_job_date_fetch_glassdoor.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_date_details/df_job_date_fetch_glassdoor.csv -------------------------------------------------------------------------------- /sample_data/output/job_date_details/df_job_date_fetch_indeed.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_date_details/df_job_date_fetch_indeed.csv -------------------------------------------------------------------------------- /sample_data/output/job_date_details/df_job_date_fetch_linkedin.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_date_details/df_job_date_fetch_linkedin.csv -------------------------------------------------------------------------------- /sample_data/output/job_location/df_job_location_glassdoor.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_location/df_job_location_glassdoor.csv -------------------------------------------------------------------------------- /sample_data/output/job_location/df_job_location_linkedin.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_location/df_job_location_linkedin.csv -------------------------------------------------------------------------------- /sample_data/output/job_rating/df_job_reviews_glassdoor.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_rating/df_job_reviews_glassdoor.csv -------------------------------------------------------------------------------- /sample_data/output/job_rating/df_job_reviews_indeed.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_rating/df_job_reviews_indeed.csv -------------------------------------------------------------------------------- /sample_data/output/job_salary/df_job_salary_glassdoor.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_salary/df_job_salary_glassdoor.csv -------------------------------------------------------------------------------- /sample_data/output/job_sector/df_job_sectors_glassdoor.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/sample_data/output/job_sector/df_job_sectors_glassdoor.csv -------------------------------------------------------------------------------- /script/airflow_start.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/script/airflow_start.sh -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayyan17/jobAnalytics_and_search/HEAD/setup.py --------------------------------------------------------------------------------