├── .gitignore ├── README.MD ├── bible_finder ├── 02_get_addr.py └── __init__.py ├── bs4_exam ├── __init__.py ├── bs4_01.py ├── index.html └── table.html ├── chrome └── mac │ └── chromedriver ├── com ├── __init__.py ├── chapter01 │ ├── 01_concate.py │ ├── 02_function.py │ └── 03_function_parameter.py ├── chapter03_python_begin │ ├── 01_variable_constants │ │ ├── 01_print_hello.py │ │ ├── 02_variable_constants.py │ │ ├── 03_variable_constants_2.py │ │ └── __init__.py │ ├── 02_if │ │ ├── 01_if.py │ │ ├── 02_else.py │ │ ├── 03_else_if.py │ │ ├── 04_else_if_2.py │ │ ├── 04_if_exam.py │ │ ├── 05_if_function.py │ │ └── __init__.py │ ├── 03_function │ │ ├── 01_make_function.py │ │ ├── 03_call_function.py │ │ ├── 03_make_function_pikachu.py │ │ ├── 04_call_function_pikachu.py │ │ ├── 05_parameter.py │ │ ├── 06_parameter_pikachu.py │ │ ├── 07_f_o_g.py │ │ ├── 08_score_grade.py │ │ └── __init__.py │ ├── 05_for │ │ ├── 01_make_for.py │ │ ├── 02_range.py │ │ ├── 03_list.py │ │ ├── 04_list_index.py │ │ ├── 05_print_list.py │ │ ├── 06_multiplication_table.py │ │ ├── 07_for_to_list2.py │ │ └── __init__.py │ ├── 06_number │ │ ├── 01_make_interger_data.py │ │ ├── 02_1_make_float_data.py │ │ ├── 02_2_make_float_data_2.py │ │ ├── 03_operator.py │ │ └── 06_number_practice.py │ ├── 07_string │ │ ├── 01_make_string_data.py │ │ ├── 02_string_concat.py │ │ ├── 03_explain_with_string.py │ │ ├── 04_string_indexing.py │ │ ├── 05_find.py │ │ ├── 06_split.py │ │ ├── 07_replace.py │ │ ├── 08_count.py │ │ └── 09_parsing_practice.py │ ├── 08_list │ │ ├── 01_make_list.py │ │ ├── 02_list_indexing.py │ │ ├── 03_list_indexing_2.py │ │ ├── 04_append_insert.py │ │ ├── 05_remove_del.py │ │ ├── 06_sort.py │ │ └── 07_count.py │ ├── 09_dictionary │ │ ├── 01_dictionray.py │ │ ├── 02_set_value.py │ │ ├── 03_number_key.py │ │ ├── 04_set_n_value.py │ │ ├── 05_get_value.py │ │ ├── 06_add_value.py │ │ ├── 07_remove_value.py │ │ ├── 08_get.py │ │ └── 09_keys_values_items.py │ └── __init__.py ├── chapter05_naver │ ├── 01_call_naver │ │ └── 01_naver_crawl.py │ ├── 02_bs4 │ │ ├── 01_basic │ │ │ ├── 02_get_second_li.py │ │ │ ├── 03_find_all.py │ │ │ └── __init__.py │ │ ├── 01_beautifulsoup │ │ │ ├── 01_beautiful_soup.py │ │ │ ├── 02_find_ul.py │ │ │ ├── 03_find_li.py │ │ │ ├── 04_find_li_text.py │ │ │ ├── 05_find_all.py │ │ │ ├── 06_find_all_index.py │ │ │ ├── 08_find_all_index_text.py │ │ │ └── __init__.py │ │ ├── 02_bs4_with_class │ │ │ ├── 01_find_ul.py │ │ │ ├── 02_tag_and_property.py │ │ │ └── __init__.py │ │ ├── 03_property │ │ │ ├── 01_extract_a_tag.py │ │ │ ├── 02_property_href.py │ │ │ └── __init__.py │ │ └── __init__.py │ ├── 03_naver_home │ │ ├── 01_naver_data.py │ │ ├── 02_naver_adjust_bs4.py │ │ ├── 03_extract_tag.py │ │ ├── 04_extract_a_tag.py │ │ └── 05_extract_text.py │ ├── 04_naver_menu │ │ ├── 01_naver_menu_ul.py │ │ ├── 02_naver_menu_li.py │ │ ├── 03_naver_menu_lis.py │ │ ├── 04_naver_menu_lis_for.py │ │ ├── 05_naver_menu_a.py │ │ ├── 06_naver_menu_span.py │ │ └── 07_naver_menu_text.py │ ├── 05_naver_news │ │ ├── 01_naver_news.py │ │ ├── 02_naver_news_newsnow.py │ │ ├── 03_naver_news_li.py │ │ ├── 04_naver_news_strong.py │ │ └── 05_naver_news_text.py │ └── __init__.py ├── chapter06_eosgo │ ├── 01_get_subpage_link │ │ ├── 01_eosgo_call.py │ │ ├── 02_bp_links.py │ │ ├── 03_bp_links_hrefs.py │ │ ├── 04_links_len.py │ │ ├── 05_link_select_range.py │ │ └── __init__.py │ ├── 02_subpage_contents │ │ ├── 01_call_subpage.py │ │ ├── 02_extract_bp_name.py │ │ ├── 03_extract_bp_name_h1.py │ │ ├── 04_extract_bp_name_text.py │ │ ├── 05_location.py │ │ ├── 06_location2.py │ │ ├── 07_location3.py │ │ ├── 08_link.py │ │ ├── 09_link2.py │ │ ├── 10_link_a_tag.py │ │ ├── 11_link_print.py │ │ ├── 12_bp_info_getter.py │ │ └── __init__.py │ ├── 03_bp_link_sub_contents │ │ ├── 01_bp_links.py │ │ ├── 02_bp_links_for.py │ │ └── __init__.py │ └── __init__.py ├── chapter07_shoppingmal │ ├── 01_shopping_mall │ │ ├── 01_jolse_mist.py │ │ ├── 02_jolse_mist_ul.py │ │ ├── 03_box.py │ │ ├── 04_product_name.py │ │ ├── 04_product_name_span.py │ │ ├── 05_find_all.py │ │ ├── 06_index.py │ │ ├── 07_price_ul.py │ │ ├── 08_price_ul_spans.py │ │ ├── 09_price_ul_spans_text.py │ │ ├── 10_return_name_price.py │ │ ├── 11_href.py │ │ ├── 12_href2.py │ │ ├── 14_product_info_list.py │ │ ├── 15_url_function.py │ │ └── 16_url_function.py │ ├── 02_page │ │ ├── 01_page.py │ │ ├── 01_page2.py │ │ └── 01_page_for.py │ └── 03_refactoring │ │ └── 01_get_page.py ├── chapter08_json │ ├── 01_json │ │ ├── 01_first.json │ │ ├── 02_.json │ │ ├── 03_n_datas.json │ │ └── 04_table_to_json.json │ └── 02_python_json │ │ ├── 01_json.py │ │ ├── 01_parse_json.py │ │ ├── 02_extract_data.py │ │ ├── 03_extract_n_data.py │ │ ├── 04_n_data_name.py │ │ ├── 05_retrieve_data_using_for.py │ │ ├── 06_retrieve_data_using_for2.py │ │ ├── 07_retrieve_data_using_for3.py │ │ └── 08_paste_to_excel.py ├── chapter09_naver_search_api │ ├── 01_api_call │ │ ├── 01_naver_api.py │ │ ├── 02_client_id_secret_keyword.py │ │ ├── 03_url_parameter.py │ │ ├── 04_requests_exam.py │ │ ├── 05_get_data.py │ │ ├── 06_requests_json1.py │ │ ├── 07_requests_json2.py │ │ ├── 08_requests_json_propetry.py │ │ ├── 09_requests_replace.py │ │ ├── 10_requests_link.py │ │ └── __init__.py │ ├── 02_url_parameter │ │ ├── 01_request.py │ │ ├── 02_url_parameter.py │ │ ├── 03_function.py │ │ ├── 04_paging.py │ │ ├── 05_print_result.py │ │ ├── 06_print_n_result.py │ │ └── 07_title_blogger_link.py │ ├── __init__.py │ └── json │ │ ├── 01_json.py │ │ ├── 02_json_2.py │ │ ├── 03_json_add_elements.py │ │ ├── 04_json_key_value.py │ │ ├── 05_json_with_friend.py │ │ ├── 06_json_5.py │ │ ├── 07_json_friends_for.py │ │ ├── 08_json_friends_for_name.py │ │ └── __init__.py ├── chapter10_grains │ ├── 01_receive_text.py │ ├── 02_receive_text.py │ ├── 03_convert_to_json.py │ ├── 04_print_first_item.py │ ├── 05_print_n_items.py │ ├── 06_print_for.py │ ├── 07_print_for2.py │ ├── 08_print_for3.py │ ├── 10_print_for5.py │ └── __init__.py ├── chapter11_public_data │ ├── 01_api_practice.py │ ├── 02_url_encoding.py │ ├── 03_search_pharmacy.py │ ├── 04_search_all_pharmacy.py │ ├── 05_search_pharmacy_in_special_condition.py │ ├── 06_career_info_structure.py │ ├── 07_job_income_over_4000.py │ ├── 08_job_distribution.py │ ├── 09_file_input_output.py │ ├── 10_save_api_result_in_file.py │ └── 11_processing_file_data.py ├── chapter12_naver_shopping │ ├── __init__.py │ ├── analyze.py │ ├── stage_naver_shopping.py │ └── stage_naver_shopping_paging.py └── chapter13_selinium │ ├── 01_random_request.py │ └── __init__.py ├── data_go_kr └── 01_data_go_kr.py ├── documents ├── 01_what_is_crawling.md ├── 02_naver_urlopen.md ├── 03_bs4.md ├── 04_naver_crawl.md ├── 05_json.md ├── 06_list.md ├── crawler_step_by_step.md ├── data_go_kr.md ├── file.md ├── matplot.md ├── naverKeywordResult.json ├── naver_extract_bookmark_text.md └── netd.md ├── excel ├── 01_xlrd.py ├── 02_xlrd.py ├── __init__.py ├── crawler.py ├── keywords.xlsx └── user_info.xlsx ├── gn_lib3 ├── __init__.py ├── crawler.py └── save_to_file.py ├── image ├── 01_downimage.py └── 02_downimage.py ├── lecture ├── 01_data_go_kr_real_estate.py ├── 01_gopax.py ├── 01_netd_crawl.py ├── 02_netd.py ├── __init__.py ├── apicall │ ├── 01_call_api.py │ └── 02_simple_api_call_code.py ├── chapter10_refactoring │ ├── 01_call_naver_news.py │ ├── 02_naver_api_news.py │ ├── 03_requests_json.py │ ├── 05_requests_json_function.py │ └── __init__.py ├── chapter16_inisfree │ ├── 01_inisfree.py │ └── 02_inis2.py ├── coupang │ ├── 02_get_products.py │ ├── 03_get_product.py │ ├── 04_collect_category.py │ └── __init__.py ├── graph │ ├── 01_graph.py │ └── __init__.py ├── jsonn │ └── 01_hello.py ├── lecture_3 │ ├── 01_function.py │ ├── 02_http_request.py │ ├── 03_bs4_01.py │ ├── 04_for.py │ ├── 05_list.py │ ├── 06_print_td.py │ ├── 07_http_request.py │ ├── 08_get_text.py │ ├── 09_get_bs_obj.py │ ├── 10_parsing_01.py │ ├── 11_parsing_for.py │ ├── 12_crawl_naver_webtoon.py │ ├── 13_bank.py │ ├── 14_jolse.py │ ├── 15_jolse.py │ ├── 16_jolse_n.py │ ├── 17_function_example.py │ ├── 18_function_examples.py │ ├── 19_return.py │ ├── 20_parameter2.py │ ├── 21_return_dict.py │ ├── 22_with_file.py │ ├── __init__.py │ └── jolse_packpeel_27.html ├── lecture_4 │ ├── 02_for.py │ ├── 03_for_2.py │ ├── 04_for_numbers.py │ ├── 05_2_dan.py │ ├── 06_function.py │ ├── 07_parameter.py │ ├── 08_print.py │ ├── 09_file_open.py │ ├── 10_crawl_codes.py │ ├── 11_crawl_pages.py │ ├── 12_for_each.py │ ├── __init__.py │ ├── codes.txt │ ├── infos.txt │ └── names.cvs ├── lecture_dorm │ ├── __init__.py │ ├── pocket_monster │ │ ├── 01_pikachu.py │ │ ├── 02_kkobuggi.py │ │ ├── 03_파이리.py │ │ └── __init__.py │ ├── week2 │ │ ├── 01_make_function_01.py │ │ ├── 02_make_pikkachu.py │ │ ├── 03_if.py │ │ ├── 04_else_if.py │ │ ├── 05_get_score.py │ │ ├── 06_function_if.py │ │ ├── 07_function_pika.py │ │ ├── 08_return.py │ │ └── __init__.py │ ├── week3 │ │ ├── 01_strange_seed.py │ │ ├── 02_urlopen.py │ │ ├── 03_urlopen2.py │ │ ├── 03_urlopen_naver.py │ │ ├── 07_urlopen.py │ │ └── __init__.py │ ├── week5_high_low_open_close │ │ ├── 01_hloc.py │ │ ├── 02_add_now_price.py │ │ ├── 03_bs4_find_all.py │ │ └── __init__.py │ ├── week5_requests_refactoring │ │ ├── 01_start.py │ │ ├── 02_complete.py │ │ ├── 03_get_price.py │ │ ├── 04_make_get_price.py │ │ ├── 05_make_get_price.py │ │ └── __init__.py │ ├── week6_jolse │ │ ├── 01_jolse.py │ │ ├── 02_jolse.py │ │ ├── 03_jolse_page.py │ │ ├── 13_hw_jm.py │ │ ├── 14_hw_sk.py │ │ └── __init__.py │ └── week7_naver_api │ │ ├── 01_call_api.py │ │ ├── 01_lsh.py │ │ ├── 02_use_requests.py │ │ ├── 03_use_requests.py │ │ ├── 04_use_req.py │ │ ├── 05예제.py │ │ ├── 06예제.py │ │ ├── 07예제.py │ │ └── __init__.py ├── lecture_dorm2 │ ├── __init__.py │ ├── week2 │ │ ├── 01_review.py │ │ ├── 02_esh_return.py │ │ ├── 03_gugudan.py │ │ ├── 04_gugudan_n.py │ │ ├── 05_gugudan_2_to_9.py │ │ ├── 06_crawl_naver_finance.py │ │ ├── 07_dict.py │ │ ├── 08_type.py │ │ └── __init__.py │ ├── week3 │ │ ├── __init__.py │ │ └── review │ │ │ ├── 01_gugudan_2_dan.py │ │ │ ├── 02_naver_finance_call_page.py │ │ │ ├── 03_get_bs_obj.py │ │ │ ├── 04_extract_value.py │ │ │ ├── 05_close.py │ │ │ ├── 06_list.py │ │ │ └── __init__.py │ ├── week4 │ │ ├── 01_crawl_naver_finance.py │ │ ├── 02_code_list.py │ │ ├── 03_dictionray.py │ │ └── __init__.py │ ├── week5 │ │ ├── 01_crawl.py │ │ ├── __init__.py │ │ └── jolse_lipliner.html │ └── week6 │ │ ├── 01_jolse.py │ │ ├── 02_file.py │ │ ├── __init__.py │ │ └── jolse.html ├── lecture_gn │ ├── __init__.py │ ├── week3 │ │ ├── 01_crawler.py │ │ ├── __init__.py │ │ ├── code.txt │ │ ├── code_lists.py │ │ └── crawler.py │ ├── week4 │ │ ├── 01_gugudan.py │ │ ├── 02_function.py │ │ ├── 03_parameter.py │ │ ├── 04_http_request.py │ │ ├── 05_http_request_call.py │ │ ├── 06_http_request_call_bs4.py │ │ ├── 07_bs4_find.py │ │ ├── 08_adjust_function.py │ │ └── __init__.py │ ├── week5 │ │ ├── 01_crawler.py │ │ ├── __init__.py │ │ ├── jolse_eyebrow.html │ │ └── jolse_stage.py │ ├── week6 │ │ ├── 01_review_crawl_parse_save.py │ │ ├── 02_open_and_parse.py │ │ ├── 03_review_requests.py │ │ ├── 04_open_and_parse.py │ │ ├── 05_api_call.py │ │ ├── 06_simple_api_call.py │ │ ├── 07_dict.py │ │ ├── 08_dict_2.py │ │ ├── 09_save_a_line.py │ │ └── __init__.py │ ├── week7 │ │ ├── 01_list.py │ │ ├── 02_pandas.py │ │ ├── 03_json.py │ │ ├── __init__.py │ │ ├── stage2.py │ │ └── user.json │ └── week8 │ │ ├── 01_analyze.py │ │ ├── 02_to_excel.py │ │ ├── 03_datetime.py │ │ ├── 04_groupby.py │ │ ├── 05_column.py │ │ ├── 06_copy.py │ │ ├── 07_filter.py │ │ ├── 08_sort.py │ │ ├── 09_head.py │ │ ├── 10_to_date.py │ │ ├── 11_index_of.py │ │ ├── 12_filter.py │ │ ├── stage.py │ │ └── stage2.py ├── lecture_gn2 │ ├── week5 │ │ ├── 01_file.py │ │ ├── 02_pandas.py │ │ ├── 03_pandas_coupang.py │ │ ├── __init__.py │ │ ├── naverApiStage.py │ │ └── stage.py │ ├── week6 │ │ ├── 01_pandas.py │ │ ├── 02_filter_and_print.py │ │ ├── 03_type_date.py │ │ ├── __init__.py │ │ └── naverApiStage.py │ └── week7 │ │ ├── analyzer.py │ │ ├── competitorAnalyzer.py │ │ ├── milkAnalyzeReportTable.json │ │ ├── pd_filter.py │ │ ├── result.json │ │ └── stage.py ├── lecture_gn3 │ ├── __init__.py │ ├── week2 │ │ ├── 07_for_for.py │ │ ├── 08_naver_finance_crawl.py │ │ ├── 09_bs4.py │ │ └── __init__.py │ ├── week3 │ │ ├── 01_crawl_naver.py │ │ ├── 02_file_open.py │ │ ├── 03_print_file.py │ │ ├── __init__.py │ │ ├── appendix_pandas.py │ │ └── jongmok.xlsx │ ├── week4 │ │ ├── 01_crawl_coupang.py │ │ ├── __init__.py │ │ ├── integrated_stage.py │ │ └── stage.py │ ├── week6 │ │ ├── 01_save_to_excel.py │ │ ├── 02_sort.py │ │ ├── 03_filter.py │ │ ├── 195297.json │ │ ├── __init__.py │ │ └── stage.py │ └── week7 │ │ ├── 01_list_append.py │ │ ├── 02_list.py │ │ ├── 02_sort.py │ │ ├── 03_filter.py │ │ ├── 03_print_list.py │ │ ├── 04_print_list_for.py │ │ ├── __init__.py │ │ ├── analyze.py │ │ ├── pandas_stage.py │ │ ├── stage.py │ │ └── stage_naver_api.py ├── lecture_gn4 │ ├── __init__.py │ ├── hello.py │ ├── readme.md │ ├── week3 │ │ ├── 01_stage.py │ │ ├── 02_naver_shopping_stage.py │ │ └── __init__.py │ ├── week4 │ │ ├── 01_list.py │ │ ├── 02_ex_list.py │ │ ├── 03_ex_list2.py │ │ ├── __init__.py │ │ └── stage_naver_shopping.py │ ├── week5 │ │ ├── __init__.py │ │ ├── analyze_mandoo.py │ │ ├── stage_coupang.py │ │ └── stage_naver_shopping.py │ ├── week6 │ │ ├── __init__.py │ │ ├── analyze_captain.py │ │ ├── analyze_water.py │ │ ├── stage_coupang.py │ │ └── stage_naver_api.py │ └── week7 │ │ ├── 01_crawl.py │ │ ├── 02_analyze.py │ │ ├── 03_total_count.py │ │ ├── __init__.py │ │ ├── macaron.json │ │ ├── result.json │ │ └── stage_coupang.py ├── lecture_gn5 │ ├── __init__.py │ ├── week1 │ │ ├── 01_hello.py │ │ ├── 02_function_def.py │ │ ├── 03_parameter.py │ │ ├── 04_return.py │ │ ├── 05_for.py │ │ ├── 06_gugudan.py │ │ └── __init__.py │ ├── week2 │ │ ├── 01_function.py │ │ ├── 02_gugudan.py │ │ ├── 03_crawl_naver_finance.py │ │ └── __init__.py │ ├── week3 │ │ ├── 01_crawl_naver_finance.py │ │ ├── 02_dict.py │ │ ├── 03_list.py │ │ ├── 04_crawl_coupang.py │ │ └── __init__.py │ ├── week4 │ │ ├── 01_crawl_coupang.py │ │ ├── 02_crawl_coupang2.py │ │ ├── 03_analyze.py │ │ ├── __init__.py │ │ └── test_make_number.py │ ├── week5 │ │ ├── 01_gugudan.py │ │ ├── 02_coupang_crawl.py │ │ ├── 03_save_file.py │ │ ├── 04_analyze.py │ │ ├── 05_replace.py │ │ ├── 06_naver_api.py │ │ ├── 07_naver_api_analyze.py │ │ └── __init__.py │ ├── week6 │ │ ├── 01_naver_api_crawl.py │ │ ├── 02_serialize.py │ │ ├── 03_analyze.py │ │ ├── 04_save_by_sheet.py │ │ └── __init__.py │ └── week7 │ │ ├── 01_coupang_crawl.py │ │ ├── 02_analyze.py │ │ ├── 03_chart.py │ │ ├── 04_analyze_and_chart.py │ │ ├── 05_pie.py │ │ ├── 06_brand_share.py │ │ ├── __init__.py │ │ ├── 생수.json │ │ ├── 어린이날.json │ │ ├── 추석.json │ │ └── 카네이션.json ├── lecture_gn6 │ ├── __init__.py │ ├── week1 │ │ ├── __init__.py │ │ └── com │ │ │ ├── 01_hello.py │ │ │ ├── 02_function.py │ │ │ ├── 03_ex_fn.py │ │ │ ├── 04_parameter.py │ │ │ ├── 05_var_const.py │ │ │ ├── 06_for.py │ │ │ ├── 07_gugudan.py │ │ │ ├── 08_return.py │ │ │ ├── __init__.py │ │ │ └── stage.py │ ├── week2 │ │ ├── 01_print_gugudan.py │ │ ├── 02_return.py │ │ ├── 03_crawl.py │ │ └── __init__.py │ ├── week3 │ │ ├── 01_return.py │ │ ├── 02_ex_return.py │ │ ├── 03_calculate_salary.py │ │ ├── 04_crawler.py │ │ ├── 05_list.py │ │ └── __init__.py │ ├── week4 │ │ ├── 01_make_function_return.py │ │ ├── 02_crawl_and_parse.py │ │ ├── 03_dictionary.py │ │ ├── 04_naver_shopping.py │ │ └── __init__.py │ ├── week5 │ │ ├── 01_naver_shopping.py │ │ └── __init__.py │ ├── week6 │ │ ├── 01_crawl.py │ │ ├── 02_crawl_and_parse.py │ │ └── __init__.py │ └── week7 │ │ ├── 01_1111.py │ │ ├── 01_naver_shopping.py │ │ ├── 02_file.py │ │ ├── 03_save_list_dictionary.py │ │ ├── 04_pandas.py │ │ ├── 05_list.py │ │ ├── 06_pd_filter.py │ │ ├── 07_pd_filter_str.py │ │ ├── 08_save_to_excel.py │ │ ├── __init__.py │ │ └── coffee.json ├── pr_pandas │ ├── 01_open_with_pd.py │ ├── 02_pandas_filter.py │ ├── 03_type_date.py │ ├── __init__.py │ └── naverKeywordResult.json ├── practice │ ├── 01_print_hello.py │ ├── 02_variable.py │ ├── 03_to_excel_csv.py │ ├── 04_two_items.py │ ├── 05_for.py │ ├── 06_for_2.py │ ├── 07_naver_api.py │ ├── 08_function.py │ ├── 09_to_excel.py │ ├── 10_save_to_csv.py │ ├── 11_stage.py │ ├── 12_analyze_pd.py │ ├── __init__.py │ └── gangnam.json ├── stage_netd.py ├── taling │ ├── __init__.py │ └── week1_function │ │ ├── 01_function.py │ │ ├── 02_pikachu.py │ │ ├── 03_kkobugi.py │ │ ├── 05_parameter.py │ │ ├── 06_if.py │ │ ├── 07_eles_if.py │ │ ├── 08_pikachu_level2.py │ │ ├── 09_get_score.py │ │ ├── 10_return.py │ │ ├── 11_get_grade_return.py │ │ ├── 12_variable.py │ │ ├── 13_pirie_return.py │ │ ├── 14_recursive.py │ │ └── __init__.py ├── week4 │ ├── 01_naver_api_call.py │ ├── 02_api_json.py │ ├── 04_replace.py │ └── 05_naver_finance_price.py └── week8 │ ├── 01_call_api.py │ ├── 02_call_api_cafe.py │ └── 03_call_api_some.py ├── libs ├── __init__.py ├── agrion │ ├── __init__.py │ └── parser.py ├── bible │ ├── __init__.py │ └── bibleAddressGetter.py ├── bibleFinder.py ├── bin │ └── chromedriver ├── browser.py ├── coupang │ ├── __init__.py │ ├── coupangCrawler.py │ ├── crawler.py │ ├── parser.py │ ├── productGetter.py │ ├── productTotalCountGetter.py │ ├── productsParser.py │ └── productsParser2.py ├── crawler.py ├── daily_sam │ ├── __init__.py │ └── dailySamCrawler.py ├── dorm_libs │ ├── 02_code_list.py │ ├── __init__.py │ ├── crawler.py │ ├── fileSaver.py │ └── parser_shoppingmall.py ├── excelSaver.py ├── gaeLines.json ├── jolse │ ├── __init__.py │ ├── jolse_crawl.py │ └── jolse_parser.py ├── jsonFileSaver.py ├── json_reducer.py ├── naver │ ├── __init__.py │ ├── apiCrawler.py │ ├── apiCrawler2.py │ ├── crawler.py │ └── parser.py ├── naver_api │ ├── __init__.py │ ├── api_call.py │ ├── naver_api_caller.py │ └── naver_api_caller2.py ├── naver_finance │ ├── __init__.py │ ├── crawler.py │ └── parser.py ├── naver_shopping │ ├── __init__.py │ ├── crawler.py │ └── parser.py ├── naver_shopping2 │ ├── __init__.py │ ├── crawler.py │ └── parser.py ├── ncbi │ ├── __init__.py │ └── ncbi_crawler.py ├── netd │ ├── __init__.py │ └── netd_crawler.py ├── patternMatcher.py └── singleExcelSaver.py ├── naver_api ├── __init__.py └── shopping │ ├── __init__.py │ └── main_call_naver_shopping_api.py ├── py_auto_gui └── 01_start.py ├── qna ├── 02_itooza.py └── __init__.py ├── question ├── 01_naver_finance_sise_market_sum │ ├── 01_crawl.py │ └── __init__.py ├── 02_naver_shopping_zzim │ ├── 01_zzim.py │ └── __init__.py ├── 03_youtube │ ├── 01_youtube.py │ └── __init__.py └── __init__.py ├── sites ├── __init__.py ├── bible │ ├── 01_read.py │ ├── 02_parse_index.py │ ├── __init__.py │ └── books │ │ ├── 1-01.txt │ │ ├── 1-02출애굽기.txt │ │ ├── 1-03레위기.txt │ │ ├── 1-04민수기.txt │ │ ├── 1-05신명기.txt │ │ ├── 1-06여호수아.txt │ │ ├── 1-07사사기.txt │ │ ├── 1-08룻기.txt │ │ ├── 1-09사무엘상.txt │ │ ├── 1-10사무엘하.txt │ │ ├── 1-11열왕기상.txt │ │ ├── 1-12열왕기하.txt │ │ ├── 1-13역대상.txt │ │ ├── 1-14역대하.txt │ │ ├── 1-15에스라.txt │ │ ├── 1-16느헤미야.txt │ │ ├── 1-17에스더.txt │ │ ├── 1-18욥기.txt │ │ ├── 1-19시편.txt │ │ ├── 1-20잠언.txt │ │ ├── 1-21전도서.txt │ │ ├── 1-22아가.txt │ │ ├── 1-23이사야.txt │ │ ├── 1-24예레미야.txt │ │ ├── 1-25예레미야애가.txt │ │ ├── 1-26에스겔.txt │ │ ├── 1-27다니엘.txt │ │ ├── 1-28호세아.txt │ │ ├── 1-29요엘.txt │ │ ├── 1-30아모스.txt │ │ ├── 1-31오바댜.txt │ │ ├── 1-32요나.txt │ │ ├── 1-33미가.txt │ │ ├── 1-34나훔.txt │ │ ├── 1-35하박국.txt │ │ ├── 1-36스바냐.txt │ │ ├── 1-37학개.txt │ │ ├── 1-38스가랴.txt │ │ ├── 1-39말라기.txt │ │ ├── 2-01마태복음.txt │ │ ├── 2-02마가복음.txt │ │ ├── 2-03누가복음.txt │ │ ├── 2-04요한복음.txt │ │ ├── 2-05사도행전.txt │ │ ├── 2-06로마서.txt │ │ ├── 2-07고린도전서.txt │ │ ├── 2-08고린도후서.txt │ │ ├── 2-09갈라디아서.txt │ │ ├── 2-10에베소서.txt │ │ ├── 2-11빌립보서.txt │ │ ├── 2-12골로새서.txt │ │ ├── 2-13데살로니가전서.txt │ │ ├── 2-14데살로니가후서.txt │ │ ├── 2-15디모데전서.txt │ │ ├── 2-16디모데후서.txt │ │ ├── 2-17디도서.txt │ │ ├── 2-18빌레몬서.txt │ │ ├── 2-19히브리서.txt │ │ ├── 2-20야고보서.txt │ │ ├── 2-21베드로전서.txt │ │ ├── 2-22베드로후서.txt │ │ ├── 2-23요한일서.txt │ │ ├── 2-24요한이서.txt │ │ ├── 2-25요한삼서.txt │ │ ├── 2-26유다서.txt │ │ ├── 2-27요한계시록.txt │ │ └── __init__.py ├── dart │ ├── 01_crawl_dart.py │ └── 02_specific.py ├── ecolife │ ├── 01_crawl.py │ └── __init__.py ├── holy_bible │ ├── 01_book_list.py │ ├── 02_parse_book_info.py │ ├── 03_specific_page.py │ ├── 04_analyze.py │ ├── __init__.py │ └── bookInfo.json ├── instagram │ ├── 01_tag.py │ └── 02_driver.py ├── lotte │ ├── 01_start.py │ ├── 02_http.py │ ├── 03_style_landa.py │ └── __init__.py ├── nalsam │ ├── 01_crawl_nalsam.py │ └── __init__.py ├── naver_finance │ ├── 01_call_samsung.py │ ├── 02_get_price.py │ ├── 03_get_price_function.py │ ├── 04_get_price_function_n.py │ ├── 05_get_price_function_n_for.py │ ├── 06_get_candle_chart_data.py │ ├── 07_pr_find_all.py │ ├── 08_pr_find_all_300.py │ ├── 09_get_candle_chart_data_high.py │ ├── 10_get_candle_chart_data_open.py │ ├── 11_get_candle_chart_data_n.py │ ├── 12_parse_bps.py │ ├── __init__.py │ └── target_site_file │ │ └── 005930.html ├── ncbi │ ├── 01_call.py │ ├── 02_separater.py │ ├── 03_seli.py │ ├── 04_links.py │ ├── 05_sub_page_threading.py │ ├── 08_sub_page_parsing.py │ ├── 09_reduce.py │ ├── 10_json_to_xml.py │ ├── 11_sub.py │ ├── __init__.py │ ├── analyze │ │ ├── 01_add_year_and_nation.py │ │ ├── 03_count_year.py │ │ ├── 04_count_keywords.py │ │ ├── 04_to_excel.py │ │ └── __init__.py │ └── links.txt ├── psyworld │ ├── 01_accept.py │ └── __init__.py ├── public_data │ ├── 01_public.py │ ├── 02_pd_read_csv.py │ ├── 03_requests.py │ └── __init__.py ├── seoul_data │ ├── 01_crawl_crossline.py │ └── __init__.py └── unipass │ ├── 01_unipass.py │ ├── __init__.py │ └── parser.py ├── sports ├── 01_hello.py ├── 02_import_json.py ├── bet356_prematch.json └── bet365_prematch_odds.json ├── test ├── 01_crawl.ipynb ├── 01_largest.py ├── 01_naver.py ├── 02_flight.py ├── 03_indices.py ├── 04_turtle_key_press.py ├── 05_pac_man.py ├── 08_ddd.py ├── 09_agrion_parser_test.py ├── 10_http_request_with_header.py ├── 11_hello.py ├── __init__.py └── agrion │ ├── 06_농업온.py │ ├── 07_농업온_data.py │ ├── __init__.py │ └── hello3.json └── test_data ├── 01_ppompu.py ├── 01_shoppingmal_jolse ├── 01_jolse_mist.py ├── 02_box.py ├── 03_boxes_findall.py ├── 04_box_name.py ├── 05_box_name_a.py ├── 06_box_name_a_title.py ├── 07_box_price_ul.py ├── 08_box_price_ul_findall.py └── 09_box_price_ul_findall_li_span.py ├── 01_slice.py ├── 02_hello.py ├── 06_naver_finance └── 01_naver_exchange.py ├── chapter11_naver_trand ├── 01_call.py └── 02_simple_call.py ├── chapter12_selenium ├── 01_login_bank.py ├── 01_login_daum.py ├── 01_login_naver.py ├── 02_my_info.py └── 05_hello.py └── chapter14_file ├── 01_file_hello.py ├── 02_open_file.py ├── 03_save_line_to_file.py ├── 04_replace.py ├── 05_random_number_to_file.py └── 06_draw_chart_using_file.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /bible_finder/02_get_addr.py: -------------------------------------------------------------------------------- 1 | from libs.bible.bibleAddressGetter import getAddr 2 | 3 | text = "(민수기 12:9~16)" 4 | res = getAddr(text) 5 | print(res) 6 | -------------------------------------------------------------------------------- /bible_finder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/bible_finder/__init__.py -------------------------------------------------------------------------------- /bs4_exam/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/bs4_exam/__init__.py -------------------------------------------------------------------------------- /bs4_exam/bs4_01.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup # 라이브러리 import 2 | 3 | # 파일 불러오기 4 | html_text = open("index.html", encoding="utf-8").read() 5 | 6 | # BeautifulSoup으로 읽어오기 7 | bsobj = BeautifulSoup(html_text, "html.parser") 8 | 9 | # 출력하기 10 | print(bsobj) -------------------------------------------------------------------------------- /bs4_exam/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | HTML구조 6 | 7 | 8 |

제목1

9 | 14 | 15 | -------------------------------------------------------------------------------- /bs4_exam/table.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | 9 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
10 | 이름나이
김경록36
함정호24
22 | 23 | -------------------------------------------------------------------------------- /chrome/mac/chromedriver: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/chrome/mac/chromedriver -------------------------------------------------------------------------------- /com/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/__init__.py -------------------------------------------------------------------------------- /com/chapter01/01_concate.py: -------------------------------------------------------------------------------- 1 | message = "hello" 2 | who = "kyeongrok" 3 | 4 | print(message + " " + who) 5 | 6 | url = "www.naver.com" 7 | query = "?query=영화" 8 | print(url + query) 9 | -------------------------------------------------------------------------------- /com/chapter01/02_function.py: -------------------------------------------------------------------------------- 1 | def pikachu(): 2 | return "electric attack" 3 | 4 | print(pikachu()) 5 | # hydro pump 6 | 7 | def kkobuggi(): 8 | return "hydro pump" -------------------------------------------------------------------------------- /com/chapter01/03_function_parameter.py: -------------------------------------------------------------------------------- 1 | def pikachu(skill_number): 2 | if skill_number == 1: 3 | # print("몸통박치기") 4 | return "몸통박치기" 5 | elif skill_number == 2: 6 | return "전기공격" 7 | # print("전기공격") 8 | 9 | print(pikachu(1)) 10 | print(pikachu(2)) 11 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/01_variable_constants/01_print_hello.py: -------------------------------------------------------------------------------- 1 | print("hello") 2 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/01_variable_constants/02_variable_constants.py: -------------------------------------------------------------------------------- 1 | result = 10 2 | print(result) 3 | result = 20 4 | print(result) 5 | 6 | print("10") 7 | print("20") 8 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/01_variable_constants/03_variable_constants_2.py: -------------------------------------------------------------------------------- 1 | hello = "world" 2 | world = "hello" 3 | 4 | print(hello) 5 | print(world) -------------------------------------------------------------------------------- /com/chapter03_python_begin/01_variable_constants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter03_python_begin/01_variable_constants/__init__.py -------------------------------------------------------------------------------- /com/chapter03_python_begin/02_if/01_if.py: -------------------------------------------------------------------------------- 1 | password = 1234 2 | 3 | if password == 1234: 4 | print("문을 열어줍니다.") 5 | else: 6 | print("아무것도 하지 않습니다.") -------------------------------------------------------------------------------- /com/chapter03_python_begin/02_if/02_else.py: -------------------------------------------------------------------------------- 1 | password = 1235 2 | 3 | if password == 1234: 4 | print("문을 열어줍니다.") 5 | else: 6 | print("아무것도 하지 않습니다.") -------------------------------------------------------------------------------- /com/chapter03_python_begin/02_if/03_else_if.py: -------------------------------------------------------------------------------- 1 | score = 86 2 | 3 | if score >= 90: 4 | print("A") 5 | elif score >= 80: 6 | print("B") 7 | else: 8 | print("C") -------------------------------------------------------------------------------- /com/chapter03_python_begin/02_if/04_else_if_2.py: -------------------------------------------------------------------------------- 1 | score = 65 2 | 3 | if score >= 90: 4 | print("A") 5 | elif score >= 80: 6 | print("B") 7 | elif score >= 70: 8 | print("C") 9 | else: 10 | print("F") -------------------------------------------------------------------------------- /com/chapter03_python_begin/02_if/04_if_exam.py: -------------------------------------------------------------------------------- 1 | # 학생들 점수 2 | # scores에 있는 점수의 등급을 매기는 if문을 짜보세요 3 | # >=90 A, >=80 B, >=70 C, < 70 F 4 | 5 | scores = [86, 75, 42, 98] 6 | 7 | for score in scores: 8 | if score >= 90: 9 | print("A") 10 | elif score >= 80: 11 | print("B") 12 | elif score >= 70: 13 | print("C") 14 | else: 15 | print("F") 16 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/02_if/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter03_python_begin/02_if/__init__.py -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/01_make_function.py: -------------------------------------------------------------------------------- 1 | def print_hello(): 2 | print("hello") 3 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/03_call_function.py: -------------------------------------------------------------------------------- 1 | def print_hello(): 2 | print("hello") 3 | 4 | print_hello() -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/03_make_function_pikachu.py: -------------------------------------------------------------------------------- 1 | def pikachu(): 2 | print("전기공격") 3 | 4 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/04_call_function_pikachu.py: -------------------------------------------------------------------------------- 1 | def pikachu(): 2 | print("전기공격") 3 | 4 | pikachu() 5 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/05_parameter.py: -------------------------------------------------------------------------------- 1 | def print_message(p_message): 2 | print(p_message) 3 | 4 | print_message("bye") 5 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/06_parameter_pikachu.py: -------------------------------------------------------------------------------- 1 | def pikachu(skill_number): 2 | if skill_number == 1: 3 | print("전기공격") 4 | elif skill_number == 2: 5 | print("몸통박치기") 6 | 7 | pikachu(2) -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/07_f_o_g.py: -------------------------------------------------------------------------------- 1 | # f라는 함수 선언 2 | def f(param1): 3 | return param1 + 10 4 | 5 | # g함수 선언 6 | def g(param1): 7 | return param1 * 20 8 | 9 | # 합성함수 10 | # f o g(10) = 210 11 | print(f(g(10))) -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/08_score_grade.py: -------------------------------------------------------------------------------- 1 | def get_grade(score): 2 | # score를 넣으면 grade 3 | if score >= 90: 4 | return "A" 5 | elif score >= 80: 6 | return "B" 7 | elif score >= 70: 8 | return "C" 9 | else: 10 | return "F" 11 | 12 | def plus_ipnida(grade): 13 | # grade를 넣으면 입니다. 를 붙여줌 14 | return grade + "입니다." 15 | 16 | # g o f(80) = g(f(80)) 17 | # result = B입니다. 18 | result = plus_ipnida(get_grade(80)) 19 | print(result) -------------------------------------------------------------------------------- /com/chapter03_python_begin/03_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter03_python_begin/03_function/__init__.py -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/01_make_for.py: -------------------------------------------------------------------------------- 1 | for item in range(0, 10): 2 | print(item) -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/02_range.py: -------------------------------------------------------------------------------- 1 | print(list(range(1, 10))) -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/03_list.py: -------------------------------------------------------------------------------- 1 | list_number = [0, 1, 2, 2, 4] 2 | list_string = ["kyeongrok", "yeongduk", "jihyun"] 3 | 4 | print(list_number) 5 | print(list_string) -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/04_list_index.py: -------------------------------------------------------------------------------- 1 | num_list = [0, 1, 2, 2, 4] 2 | 3 | print(num_list) 4 | print(num_list[3]) -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/05_print_list.py: -------------------------------------------------------------------------------- 1 | for item in [0, 1, 2, 2, 4]: 2 | print(item) -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/06_multiplication_table.py: -------------------------------------------------------------------------------- 1 | for item in range(1, 10): 2 | print(2, "*", item, "=", 2 * item) -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/07_for_to_list2.py: -------------------------------------------------------------------------------- 1 | num_list = [1, 2, 2, 4] 2 | 3 | # num_list 각각에 2를 곱한 결과 4 | results = [num * 2 for num in num_list] 5 | print(results) 6 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/05_for/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter03_python_begin/05_for/__init__.py -------------------------------------------------------------------------------- /com/chapter03_python_begin/06_number/01_make_interger_data.py: -------------------------------------------------------------------------------- 1 | a = 10 2 | b = -20 3 | c = 30 4 | total = a + b + c 5 | print(total) 6 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/06_number/02_1_make_float_data.py: -------------------------------------------------------------------------------- 1 | a = 12.5 2 | b = 35.8 3 | c = 50 4 | total = a + b + c 5 | average = total / 3 6 | print(average) 7 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/06_number/02_2_make_float_data_2.py: -------------------------------------------------------------------------------- 1 | print(30/4) 2 | print(4/30) 3 | print(5/10) 4 | print(10/5) 5 | print(1/1) 6 | print(type(1)) 7 | print(type(1/1)) 8 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/06_number/03_operator.py: -------------------------------------------------------------------------------- 1 | print(5 + 4) 2 | print(5.0 - 3.0) 3 | print(10.5 * 4) 4 | print(200 / 10) 5 | print(200 % 10) 6 | print(20 / 7) 7 | print(20 % 7) 8 | print(20 // 7) 9 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/06_number/06_number_practice.py: -------------------------------------------------------------------------------- 1 | cheolsu_kor = 88 2 | cheolsu_eng = 76 3 | cheolsu_math = 95 4 | younghee_kor = 100 5 | younghee_eng = 67 6 | younghee_math = 80 7 | 8 | cheolsu_average = (cheolsu_kor + cheolsu_eng + cheolsu_math) / 3 9 | younghee_average = (younghee_kor + younghee_eng + younghee_math) / 3 10 | 11 | if (cheolsu_average > younghee_average): 12 | print("철수가 높아요") 13 | elif (cheolsu_average == younghee_average): 14 | print("둘이 같아요") 15 | else: 16 | print("영희가 높아요") 17 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/01_make_string_data.py: -------------------------------------------------------------------------------- 1 | crawling = "Data crawling is fun" 2 | parsing = 'Data parsing is also fun' 3 | 4 | print(crawling) 5 | print(parsing) 6 | print(type(crawling)) 7 | print(type(parsing)) 8 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/02_string_concat.py: -------------------------------------------------------------------------------- 1 | crawling = "Data crawling is fun" 2 | parsing = 'Data parsing is also fun' 3 | 4 | print(crawling + " " + parsing) 5 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/03_explain_with_string.py: -------------------------------------------------------------------------------- 1 | r = 10 2 | pie = 3.141592 3 | area = (r ** 2) * pie 4 | 5 | print("원의 넓이 : " + str(area)) 6 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/04_string_indexing.py: -------------------------------------------------------------------------------- 1 | crawling = "Data crawling is fun" 2 | parsing = 'Data parsing is also fun' 3 | 4 | data = crawling[0:4] 5 | data_crawling = crawling[0:13] 6 | fun = crawling[17:] 7 | n = crawling[19] 8 | n_range = crawling[-1:] 9 | also = parsing[16:16 + 4] 10 | 11 | print(data) 12 | print(data_crawling) 13 | print(fun) 14 | print(n) 15 | print(n_range) 16 | print(also) 17 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/05_find.py: -------------------------------------------------------------------------------- 1 | crawling = "Data crawling is fun" 2 | parsing = 'Data parsing is also fun' 3 | 4 | print(crawling.find("Data")) 5 | print(crawling.find("fun")) 6 | print(parsing.find("Data")) 7 | print(parsing.find("fun")) 8 | print(crawling.find("parsing")) 9 | print(parsing.find("crawling")) 10 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/06_split.py: -------------------------------------------------------------------------------- 1 | str_data = "random:data:choice:sampling:mini-batch:unpooling:training" 2 | split_str_data = str_data.split(":") 3 | print(str_data) 4 | print(split_str_data) 5 | 6 | for i in range(0, len(split_str_data)): 7 | print(split_str_data[i]) 8 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/07_replace.py: -------------------------------------------------------------------------------- 1 | crawling = "Data crawling is fun" 2 | 3 | print(crawling.replace("search", "analyze")) 4 | print(crawling.replace("Data", "Web")) 5 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/07_string/09_parsing_practice.py: -------------------------------------------------------------------------------- 1 | str_data = "{ a1 : 20 }, { a2 : 30 }, { a3 : 15 }, { a4 : 50 }, { a5 : -15 }, { a6 : 80 }, { a7 : 0 }, { a8 : -110 }" 2 | 3 | total = 0 4 | split_str_data = str_data.split(",") 5 | for i in range(0, len(split_str_data)): 6 | str_tmp = split_str_data[i].split(":")[1].split("}")[0] 7 | num_tmp = int(str_tmp) 8 | total += num_tmp 9 | 10 | print(total) 11 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/08_list/01_make_list.py: -------------------------------------------------------------------------------- 1 | name = [] 2 | print(name) 3 | print(type(name)) 4 | 5 | 6 | name = ["kyeongrok"] 7 | 8 | print(name) 9 | 10 | 11 | int_list = [1, 2, 3, 4, 5] 12 | float_list = [1.1, 2.2, 3.3, 4.4, 5.5] 13 | string_list = ["crawling", "parsing", "data", "extract", "pre-processing"] 14 | all_list = [int_list, float_list, string_list] 15 | 16 | print(int_list) 17 | print(float_list) 18 | print(string_list) 19 | print(all_list) 20 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/08_list/02_list_indexing.py: -------------------------------------------------------------------------------- 1 | int_list = [1, 2, 3, 4, 5] 2 | float_list = [1.1, 2.2, 3.3, 4.4, 5.5] 3 | string_list = ["crawling", "parsing", "data", "extract", "pre-processing"] 4 | all_list = [int_list, float_list, string_list] 5 | 6 | print(int_list[2:4]) 7 | print(float_list[-1:]) 8 | print(string_list[1]) 9 | print(all_list[0][4]) 10 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/08_list/03_list_indexing_2.py: -------------------------------------------------------------------------------- 1 | int_list = [1, 2, 3, 4, 5] 2 | float_list = [1.1, 2.2, 3.3, 4.4, 5.5] 3 | string_list = ["crawling", "parsing", "data", "extract", "pre-processing"] 4 | all_list = [int_list, float_list, string_list] 5 | 6 | for i in range(0, len(int_list)): 7 | print(int_list[i]) 8 | for i in range(0, len(float_list)): 9 | print(float_list[i]) 10 | for i in range(0, len(string_list)): 11 | print(string_list[i]) 12 | for i in range(0, len(all_list)): 13 | print(all_list[i]) 14 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/08_list/04_append_insert.py: -------------------------------------------------------------------------------- 1 | int_list = [1, 2, 3, 4, 5] 2 | 3 | int_list.append(6) 4 | print(int_list) 5 | int_list.insert(0, -1) 6 | print(int_list) 7 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/08_list/05_remove_del.py: -------------------------------------------------------------------------------- 1 | int_list = [1, 2, 3, 4, 5, 4] 2 | 3 | int_list.remove(4) 4 | print(int_list) 5 | 6 | del int_list[3:] 7 | print(int_list) 8 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/08_list/06_sort.py: -------------------------------------------------------------------------------- 1 | int_list = [3, 1, 5, 4, 2] 2 | float_list = [4.4, 2.2, 3.3, 5.5, 1.1] 3 | string_list = ["crawling", "parsing", "data", "extract", "pre-processing"] 4 | 5 | int_list.sort() 6 | float_list.sort() 7 | string_list.sort() 8 | 9 | print(int_list) 10 | print(float_list) 11 | print(string_list) 12 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/08_list/07_count.py: -------------------------------------------------------------------------------- 1 | int_list = [1, 5, 4, 3, 7, 5, 4, 2, 4] 2 | string_list = ["car", "cat", "can", "cut", "cat", "cnn", "cure", "cat"] 3 | 4 | print(int_list.count(4)) 5 | print(string_list.count("cat")) 6 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/01_dictionray.py: -------------------------------------------------------------------------------- 1 | user = {} 2 | print(user) 3 | print(type(user)) -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/02_set_value.py: -------------------------------------------------------------------------------- 1 | user = {"name":"kyeongrok"} 2 | print(user) -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/03_number_key.py: -------------------------------------------------------------------------------- 1 | class1 = {0:"kyeongrok", 1:"victoria"} 2 | print(class1) 3 | print(class1[0]) 4 | print(class1[1]) -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/04_set_n_value.py: -------------------------------------------------------------------------------- 1 | naver = { 2 | "name": "naver", 3 | "url": "www.naver.com", 4 | "userid": "nv", 5 | "passwd": "1234" 6 | } 7 | 8 | google = { 9 | "name": "google", 10 | "url": "www.naver.com", 11 | "userid": "gg", 12 | "passwd": "1234" 13 | } 14 | 15 | print(naver) 16 | print(google) 17 | print(type(naver)) 18 | print(type(google)) 19 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/05_get_value.py: -------------------------------------------------------------------------------- 1 | site = { 2 | "naver": "www.naver.com", 3 | "google": "www.google.com" 4 | } 5 | 6 | print(site["naver"]) 7 | print(site["google"]) 8 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/06_add_value.py: -------------------------------------------------------------------------------- 1 | site = { 2 | "naver": "www.naver.com", 3 | "google": "www.google.com" 4 | } 5 | 6 | site["daum"] = "www.daum.net" 7 | print(site) 8 | print(site["daum"]) 9 | 10 | site["yahoo"] = "www.yahoo.com" 11 | print(site) 12 | print(site["yahoo"]) 13 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/07_remove_value.py: -------------------------------------------------------------------------------- 1 | site = { 2 | "naver": "www.naver.com", 3 | "google": "www.google.com", 4 | "daum": "nodata" 5 | } 6 | 7 | site["daum"] = "www.daum.net" 8 | print(site["daum"]) 9 | 10 | del site["daum"] 11 | print(site) 12 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/08_get.py: -------------------------------------------------------------------------------- 1 | site = { 2 | "naver": "www.naver.com", 3 | "google": "www.google.com" 4 | } 5 | 6 | print(site.get("naver")) 7 | print(site.get("daum")) 8 | 9 | insert_key = "daum" 10 | if (site.get(insert_key) == None): 11 | print(insert_key + " 에 대한 데이터가 없습니다 ") 12 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/09_dictionary/09_keys_values_items.py: -------------------------------------------------------------------------------- 1 | site = { 2 | "naver": "www.naver.com", 3 | "google": "www.google.com", 4 | "daum": "www.daum.net" 5 | } 6 | 7 | print(site.keys()) 8 | print(site.values()) 9 | print(site.items()) 10 | 11 | for key in site.keys(): 12 | print(key) 13 | for value in site.values(): 14 | print(value) 15 | for item in site.items(): 16 | print(item) 17 | -------------------------------------------------------------------------------- /com/chapter03_python_begin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter03_python_begin/__init__.py -------------------------------------------------------------------------------- /com/chapter05_naver/01_call_naver/01_naver_crawl.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | 3 | url = "https://www.naver.com/" 4 | html = urlopen(url) 5 | 6 | print(html.read()) -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_basic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter05_naver/02_bs4/01_basic/__init__.py -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/01_beautiful_soup.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | html_str = "
hello
" 3 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 4 | 5 | print(type(bs_obj)) 6 | print(bs_obj) 7 | print(bs_obj.find("div")) 8 | -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/02_find_ul.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | html_str = """ 4 | 5 | 6 | 11 | 12 | 13 | """ 14 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 15 | 16 | lis = bs_obj.find("ul") 17 | -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/03_find_li.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | html_str = """ 4 | 5 | 6 | 11 | 12 | 13 | """ 14 | 15 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 16 | 17 | ul = bs_obj.find("ul") 18 | li = ul.find("li") 19 | print(li) -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/04_find_li_text.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | html_str = """ 4 | 5 | 6 | 11 | 12 | 13 | """ 14 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 15 | 16 | ul = bs_obj.find("ul") 17 | li = ul.find("li") 18 | print(li.text) 19 | -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/05_find_all.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | html_str = """ 4 | 5 | 6 | 11 | 12 | 13 | """ 14 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 15 | 16 | ul = bs_obj.find("ul") 17 | lis = ul.findAll("li") 18 | print(lis) 19 | -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/06_find_all_index.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | html_str = """ 4 | 5 | 6 | 11 | 12 | 13 | """ 14 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 15 | 16 | ul = bs_obj.find("ul") 17 | lis = ul.findAll("li") 18 | print(lis[1]) 19 | -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/08_find_all_index_text.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | html_str = """ 4 | 5 | 6 | 11 | 12 | 13 | """ 14 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 15 | 16 | ul = bs_obj.find("ul") 17 | lis = ul.findAll("li") 18 | print(lis[1]) 19 | print(type({})) 20 | 21 | for i in range(0, 10): 22 | for j in range(1, 10): 23 | print("hello") -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/01_beautifulsoup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter05_naver/02_bs4/01_beautifulsoup/__init__.py -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/02_bs4_with_class/01_find_ul.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | html_str = """ 4 | 5 | 6 | 11 | 16 | 17 | 18 | """ 19 | 20 | bs_obj = bs4.BeautifulSoup(html_str, "html.parser") 21 | 22 | ul = bs_obj.find("ul") 23 | print(ul) -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/02_bs4_with_class/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter05_naver/02_bs4/02_bs4_with_class/__init__.py -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/03_property/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter05_naver/02_bs4/03_property/__init__.py -------------------------------------------------------------------------------- /com/chapter05_naver/02_bs4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter05_naver/02_bs4/__init__.py -------------------------------------------------------------------------------- /com/chapter05_naver/03_naver_home/01_naver_data.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | 3 | url = "https://www.naver.com/" 4 | html = urllib.request.urlopen(url) 5 | 6 | print(html.read()) -------------------------------------------------------------------------------- /com/chapter05_naver/03_naver_home/02_naver_adjust_bs4.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | print(bs_obj) -------------------------------------------------------------------------------- /com/chapter05_naver/03_naver_home/03_extract_tag.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | top_right = bs_obj.find("div", {"class":"area_links"}) 10 | print(top_right) 11 | -------------------------------------------------------------------------------- /com/chapter05_naver/03_naver_home/04_extract_a_tag.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | top_right = bs_obj.find("div", {"class":"area_links"}) 10 | first_a = top_right.find("a") 11 | print(first_a) 12 | -------------------------------------------------------------------------------- /com/chapter05_naver/03_naver_home/05_extract_text.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | top_right = bs_obj.find("div", {"class":"area_links"}) 10 | first_a = top_right.find("a") 11 | print(first_a.text) 12 | -------------------------------------------------------------------------------- /com/chapter05_naver/04_naver_menu/01_naver_menu_ul.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"an_l"}) 10 | print(ul) -------------------------------------------------------------------------------- /com/chapter05_naver/04_naver_menu/02_naver_menu_li.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"an_l"}) 10 | 11 | for li in ul: 12 | print(li) -------------------------------------------------------------------------------- /com/chapter05_naver/04_naver_menu/03_naver_menu_lis.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"an_l"}) 10 | 11 | lis = ul.findAll("li") 12 | print(lis) -------------------------------------------------------------------------------- /com/chapter05_naver/04_naver_menu/04_naver_menu_lis_for.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"an_l"}) 10 | 11 | lis = ul.findAll("li") 12 | 13 | for li in lis: 14 | print(li) 15 | 16 | -------------------------------------------------------------------------------- /com/chapter05_naver/04_naver_menu/05_naver_menu_a.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"an_l"}) 10 | 11 | lis = ul.findAll("li") 12 | 13 | for li in lis: 14 | a_tag = li.find("a") 15 | print(a_tag) 16 | -------------------------------------------------------------------------------- /com/chapter05_naver/04_naver_menu/06_naver_menu_span.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"an_l"}) 10 | 11 | lis = ul.findAll("li") 12 | 13 | for li in lis: 14 | a_tag = li.find("a") 15 | span = a_tag.find("span", {"class":"an_txt"}) 16 | print(span) 17 | -------------------------------------------------------------------------------- /com/chapter05_naver/04_naver_menu/07_naver_menu_text.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://www.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"an_l"}) 10 | 11 | lis = ul.findAll("li") 12 | 13 | for li in lis: 14 | a_tag = li.find("a") 15 | span = a_tag.find("span", {"class":"an_txt"}) 16 | print(span.text) -------------------------------------------------------------------------------- /com/chapter05_naver/05_naver_news/01_naver_news.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "http://news.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | print(bs_obj) -------------------------------------------------------------------------------- /com/chapter05_naver/05_naver_news/02_naver_news_newsnow.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "http://news.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | newsnow_txarea = bs_obj.find("ul", {"class":"newsnow_txarea"}) 10 | print(newsnow_txarea) -------------------------------------------------------------------------------- /com/chapter05_naver/05_naver_news/03_naver_news_li.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "http://news.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | newsnow_txarea = bs_obj.find("ul", {"class":"newsnow_txarea"}) 10 | lis = newsnow_txarea.findAll("li") 11 | 12 | for li in lis: 13 | print(li) -------------------------------------------------------------------------------- /com/chapter05_naver/05_naver_news/04_naver_news_strong.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "http://news.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | newsnow_txarea = bs_obj.find("ul", {"class":"newsnow_txarea"}) 10 | lis = newsnow_txarea.findAll("li") 11 | 12 | for li in lis: 13 | strong = li.find("strong") 14 | print(strong) -------------------------------------------------------------------------------- /com/chapter05_naver/05_naver_news/05_naver_news_text.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "http://news.naver.com/" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | 9 | newsnow_txarea = bs_obj.find("ul", {"class":"newsnow_txarea"}) 10 | lis = newsnow_txarea.findAll("li") 11 | 12 | for li in lis: 13 | strong = li.find("strong") 14 | print(strong.text) -------------------------------------------------------------------------------- /com/chapter05_naver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter05_naver/__init__.py -------------------------------------------------------------------------------- /com/chapter06_eosgo/01_get_subpage_link/01_eosgo_call.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/" 5 | 6 | result = requests.get(url =url) 7 | 8 | bs_obj = BeautifulSoup(result.content, "html.parser") 9 | print(bs_obj) 10 | 11 | -------------------------------------------------------------------------------- /com/chapter06_eosgo/01_get_subpage_link/02_bp_links.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/" 5 | 6 | result = requests.get(url =url) 7 | 8 | bs_obj = BeautifulSoup(result.content, "html.parser") 9 | 10 | lf_items = bs_obj.findAll("div", {"class":"lf-item"}) 11 | print(lf_items) 12 | -------------------------------------------------------------------------------- /com/chapter06_eosgo/01_get_subpage_link/03_bp_links_hrefs.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/" 5 | 6 | result = requests.get(url =url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | lf_items = bs_obj.findAll("div", {"class":"lf-item"}) 10 | 11 | hrefs = [div.find("a")['href'] for div in lf_items ] 12 | print(hrefs) -------------------------------------------------------------------------------- /com/chapter06_eosgo/01_get_subpage_link/04_links_len.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/" 5 | 6 | result = requests.get(url =url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | lf_items = bs_obj.findAll("div", {"class":"lf-item"}) 10 | 11 | hrefs = [div.find("a")['href'] for div in lf_items ] 12 | 13 | print(len(hrefs)) 14 | 15 | -------------------------------------------------------------------------------- /com/chapter06_eosgo/01_get_subpage_link/05_link_select_range.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/" 5 | 6 | result = requests.get(url =url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | lf_items = bs_obj.findAll("div", {"class":"lf-item"}) 10 | 11 | hrefs = [div.find("a")['href'] for div in lf_items ] 12 | 13 | print(hrefs[0:5]) 14 | print(len(hrefs[0:5])) 15 | 16 | -------------------------------------------------------------------------------- /com/chapter06_eosgo/01_get_subpage_link/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter06_eosgo/01_get_subpage_link/__init__.py -------------------------------------------------------------------------------- /com/chapter06_eosgo/02_subpage_contents/01_call_subpage.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/listing/eos-canada/" 5 | 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | print(bs_obj) 9 | -------------------------------------------------------------------------------- /com/chapter06_eosgo/02_subpage_contents/02_extract_bp_name.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/listing/eos-canada/" 5 | 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | profile_name = bs_obj.find("div", {"class":"profile-name"}) 10 | 11 | print(profile_name) -------------------------------------------------------------------------------- /com/chapter06_eosgo/02_subpage_contents/03_extract_bp_name_h1.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/listing/eos-canada/" 5 | 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | profile_name = bs_obj.find("div", {"class":"profile-name"}) 10 | 11 | h1_bp_name = profile_name.find("h1") 12 | print(h1_bp_name) -------------------------------------------------------------------------------- /com/chapter06_eosgo/02_subpage_contents/04_extract_bp_name_text.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/listing/eos-canada/" 5 | 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | profile_name = bs_obj.find("div", {"class":"profile-name"}) 10 | 11 | h1_bp_name = profile_name.find("h1") 12 | bp_name = h1_bp_name.text 13 | print(bp_name) -------------------------------------------------------------------------------- /com/chapter06_eosgo/02_subpage_contents/05_location.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/listing/eos-canada/" 5 | 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | profile_name = bs_obj.find("div", {"class":"profile-name"}) 10 | 11 | h1_bp_name = profile_name.find("h1") 12 | bp_name = h1_bp_name.text 13 | 14 | cover_buttons = bs_obj.find("div", {"class":"cover-buttons"}) 15 | print(cover_buttons) 16 | -------------------------------------------------------------------------------- /com/chapter06_eosgo/02_subpage_contents/06_location2.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://bp.eosgo.io/listing/eos-canada/" 5 | 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | profile_name = bs_obj.find("div", {"class":"profile-name"}) 10 | 11 | h1_bp_name = profile_name.find("h1") 12 | bp_name = h1_bp_name.text 13 | 14 | cover_buttons = bs_obj.find("div", {"class":"cover-buttons"}) 15 | 16 | button_label = bs_obj.find("span", {"class":"button-label"}) 17 | print(button_label) -------------------------------------------------------------------------------- /com/chapter06_eosgo/02_subpage_contents/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter06_eosgo/02_subpage_contents/__init__.py -------------------------------------------------------------------------------- /com/chapter06_eosgo/03_bp_link_sub_contents/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter06_eosgo/03_bp_link_sub_contents/__init__.py -------------------------------------------------------------------------------- /com/chapter06_eosgo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter06_eosgo/__init__.py -------------------------------------------------------------------------------- /com/chapter07_shoppingmal/01_shopping_mall/02_jolse_mist_ul.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import bs4 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(result.content, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"prdList column5"}) 10 | 11 | print(ul) -------------------------------------------------------------------------------- /com/chapter07_shoppingmal/01_shopping_mall/03_box.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import bs4 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(result.content, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"prdList column5"}) 10 | 11 | boxes = ul.findAll("div", {"class":"box"}) 12 | 13 | for box in boxes: 14 | print(box) 15 | 16 | -------------------------------------------------------------------------------- /com/chapter07_shoppingmal/01_shopping_mall/04_product_name.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import bs4 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(result.content, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"prdList column5"}) 10 | 11 | boxes = ul.findAll("div", {"class":"box"}) 12 | 13 | for box in boxes: 14 | ptag = box.find("p", {"class": "name"}) 15 | print(ptag) 16 | 17 | -------------------------------------------------------------------------------- /com/chapter07_shoppingmal/01_shopping_mall/04_product_name_span.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import bs4 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(result.content, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"prdList column5"}) 10 | 11 | boxes = ul.findAll("div", {"class":"box"}) 12 | 13 | for box in boxes: 14 | ptag = box.find("p", {"class": "name"}) 15 | span = ptag.find("span") 16 | print(span) -------------------------------------------------------------------------------- /com/chapter07_shoppingmal/01_shopping_mall/05_find_all.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import bs4 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(result.content, "html.parser") 8 | 9 | ul = bs_obj.find("ul", {"class":"prdList column5"}) 10 | 11 | boxes = ul.findAll("div", {"class":"box"}) 12 | 13 | for box in boxes: 14 | ptag = box.find("p", {"class": "name"}) 15 | spans = ptag.findAll("span") 16 | print(spans) 17 | 18 | -------------------------------------------------------------------------------- /com/chapter07_shoppingmal/03_refactoring/01_get_page.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import bs4 3 | 4 | # url을 넣으면 해당 페이지의 bs_obj을 return하는 5 | # get_page_from_url이라는 함수를 만들어 보세요 6 | def get_page_from_url(url): 7 | result = requests.get(url) 8 | bs_obj = bs4.BeautifulSoup(result.content, "html.parser") 9 | return bs_obj 10 | 11 | url = "http://jolse.com/category/tonermist/43/" 12 | print(get_page_from_url(url)) -------------------------------------------------------------------------------- /com/chapter08_json/01_json/01_first.json: -------------------------------------------------------------------------------- 1 | {"name":"kyeongrok"} -------------------------------------------------------------------------------- /com/chapter08_json/01_json/02_.json: -------------------------------------------------------------------------------- 1 | {"class":"an_l"} -------------------------------------------------------------------------------- /com/chapter08_json/01_json/03_n_datas.json: -------------------------------------------------------------------------------- 1 | {"name":"kyeongrok", "age":"32"} 2 | -------------------------------------------------------------------------------- /com/chapter08_json/01_json/04_table_to_json.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"name":"kyeongrok", "age":"32", "math":"90"}, 3 | {"name":"bomi", "age":"25", "math":"85"} 4 | ] 5 | -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/01_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '{"name":"kyeongrok", "age":"32"}' 3 | json_obj = json.loads(json_str) 4 | 5 | print(json_obj) 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | ''' 14 | # list기준으로 item은 한개이다. 15 | 왜냐하면 {}중괄호가 1쌍이므로. 16 | 17 | {}는 json object이다. 18 | # json 오브젝트 기준으로는 item이 2개이다. 19 | 왜냐하면 ,컴마로 구분 하기 때문 20 | ''' 21 | -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/01_parse_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | print(json_obj) 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | ''' 14 | # list기준으로 item은 한개이다. 15 | 왜냐하면 {}중괄호가 1쌍이므로. 16 | 17 | {}는 json object이다. 18 | # json 오브젝트 기준으로는 item이 2개이다. 19 | 왜냐하면 ,컴마로 구분 하기 때문 20 | ''' 21 | -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/02_extract_data.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | print(json_obj) 6 | print(json_obj[0]) 7 | -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/03_extract_n_data.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}, {"name":"bomi", "age":"25"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | print(json_obj[1]) 6 | print(json_obj[0:2]) 7 | 8 | 9 | 10 | ''' 11 | list기준으로는 item이 두개. 왜냐하면 중괄호가 두개이기 때문. 그리고 ,컴마로 구분 되어 있다. 12 | 13 | ''' 14 | 15 | ''' 16 | print(json_obj[2]) 17 | IndexError: list index out of rangeㅆ 18 | list는 0번 index부터 시작합니다. 19 | ''' -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/04_n_data_name.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}, {"name":"bomi", "age":"25"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | print(json_obj[0]) 6 | print(json_obj[0]['name']) 7 | print(json_obj[0]['age']) -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/05_retrieve_data_using_for.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}, {"name":"bomi", "age":"25"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | for student in json_obj: 6 | print(student) -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/06_retrieve_data_using_for2.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}, {"name":"bomi", "age":"25"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | for student in json_obj: 6 | print(student['name']) -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/07_retrieve_data_using_for3.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}, {"name":"bomi", "age":"25"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | for student in json_obj: 6 | print(student['name'], student['age']) -------------------------------------------------------------------------------- /com/chapter08_json/02_python_json/08_paste_to_excel.py: -------------------------------------------------------------------------------- 1 | import json 2 | json_str = '[{"name":"kyeongrok", "age":"32"}, {"name":"bomi", "age":"25"}]' 3 | json_obj = json.loads(json_str) 4 | 5 | for student in json_obj: 6 | print(student['name'] + "," + student['age']) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/04_requests_exam.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | print(result.json()) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/05_get_data.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | 10 | json_obj = result.json() 11 | print(json_obj['lastBuildDate']) 12 | print(json_obj['total']) 13 | print(json_obj['start']) 14 | print(json_obj['display']) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/06_requests_json1.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | 10 | json_obj = result.json() 11 | print(json_obj['items']) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/07_requests_json2.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | json_obj = result.json() 10 | for item in json_obj['items']: 11 | print(item) 12 | -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/08_requests_json_propetry.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | 10 | json_obj = result.json() 11 | for item in json_obj['items']: 12 | print(item['title']) 13 | -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/09_requests_replace.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | 10 | json_obj = result.json() 11 | for item in json_obj['items']: 12 | print(item['title'].replace("","").replace("", "")) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/10_requests_link.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역 운동" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | 10 | json_obj = result.json() 11 | for item in json_obj['items']: 12 | print(item['title'].replace("","").replace("", ""), 13 | item['link']) 14 | -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/01_api_call/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter09_naver_search_api/01_api_call/__init__.py -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/02_url_parameter/01_request.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "강남역" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | 10 | json_obj = result.json() 11 | print(json_obj['display']) 12 | print(json_obj['start']) 13 | print(len(json_obj['items'])) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter09_naver_search_api/__init__.py -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/01_json.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok"} 2 | 3 | print(json_obj) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/02_json_2.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok", "age":"32"} 2 | 3 | print(json_obj) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/03_json_add_elements.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok", 2 | "age":"32", 3 | "where":"역삼동", 4 | "phone_number":"010-3588-0000" 5 | } 6 | 7 | print(json_obj) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/04_json_key_value.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok", 2 | "age":"32", 3 | "where":"역삼동", 4 | "phone_number":"010-3588-0000" 5 | } 6 | 7 | print(json_obj['name']) 8 | print(json_obj['phone_number']) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/05_json_with_friend.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok", 2 | "age":"32", 3 | "where":"역삼동", 4 | "phone_number":"010-3588-6265", 5 | "friends":[{"name":"sian", "age":"32"}] 6 | } 7 | 8 | print(json_obj['friends']) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/06_json_5.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok", 2 | "age":"32", 3 | "where":"역삼동", 4 | "phone_number":"010-3588-6265", 5 | "friends":[ 6 | {"name":"sian", "age":"32"}, 7 | {"name":"kyuri", "age":"24"} 8 | ] 9 | } 10 | 11 | friends = json_obj['friends'] 12 | print(friends) -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/07_json_friends_for.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok", 2 | "age":"32", 3 | "where":"역삼동", 4 | "phone_number":"010-3588-6265", 5 | "friends":[ 6 | {"name":"sian", "age":"32"}, 7 | {"name":"kyuri", "age":"24"} 8 | ] 9 | } 10 | 11 | friends = json_obj['friends'] 12 | for friend in friends: 13 | print(friend) 14 | -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/08_json_friends_for_name.py: -------------------------------------------------------------------------------- 1 | json_obj = {"name":"kyeongrok", 2 | "age":"32", 3 | "where":"역삼동", 4 | "phone_number":"010-3588-6265", 5 | "friends":[ 6 | {"name":"sian", "age":"32"}, 7 | {"name":"kyuri", "age":"24"} 8 | ] 9 | } 10 | 11 | friends = json_obj['friends'] 12 | for friend in friends: 13 | print(friend['age']) 14 | -------------------------------------------------------------------------------- /com/chapter09_naver_search_api/json/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter09_naver_search_api/json/__init__.py -------------------------------------------------------------------------------- /com/chapter10_grains/01_receive_text.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | 3 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/1972-01-01/edate/2018-04-28" 4 | text = urlopen(url) 5 | 6 | print(text.read()) -------------------------------------------------------------------------------- /com/chapter10_grains/02_receive_text.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | 3 | from_date = "2010-01-01" 4 | to_date = "2018-04-28" 5 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 6 | text = urlopen(url) 7 | 8 | print(text.read()) -------------------------------------------------------------------------------- /com/chapter10_grains/03_convert_to_json.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | import json 3 | 4 | from_date = "2010-01-01" 5 | to_date = "2018-04-28" 6 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 7 | text = urlopen(url) 8 | 9 | json_obj = json.load(text) 10 | print(json_obj) 11 | 12 | -------------------------------------------------------------------------------- /com/chapter10_grains/04_print_first_item.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | import json 3 | 4 | from_date = "2010-01-01" 5 | to_date = "2018-04-28" 6 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 7 | text = urlopen(url) 8 | json_objs = json.load(text) 9 | 10 | print(json_objs[0]) 11 | -------------------------------------------------------------------------------- /com/chapter10_grains/05_print_n_items.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | import json 3 | 4 | from_date = "2010-01-01" 5 | to_date = "2018-04-28" 6 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 7 | text = urlopen(url) 8 | 9 | json_objs = json.load(text) 10 | print(json_objs[0:10]) 11 | -------------------------------------------------------------------------------- /com/chapter10_grains/06_print_for.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | import json 3 | 4 | from_date = "2010-01-01" 5 | to_date = "2018-04-28" 6 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 7 | text = urlopen(url) 8 | 9 | json_objs = json.load(text) 10 | 11 | for item in json_objs: 12 | print(item) 13 | 14 | -------------------------------------------------------------------------------- /com/chapter10_grains/07_print_for2.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | import json 3 | 4 | from_date = "2010-01-01" 5 | to_date = "2018-04-28" 6 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 7 | text = urlopen(url) 8 | 9 | json_objs = json.load(text) 10 | 11 | for item in json_objs[0:10]: 12 | print(item) 13 | 14 | -------------------------------------------------------------------------------- /com/chapter10_grains/08_print_for3.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | import json 3 | 4 | from_date = "2010-01-01" 5 | to_date = "2018-04-28" 6 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 7 | text = urlopen(url) 8 | 9 | json_objs = json.load(text) 10 | for item in json_objs: 11 | print(item['date'] + "@" + item['settlement']) 12 | 13 | -------------------------------------------------------------------------------- /com/chapter10_grains/10_print_for5.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | import json 3 | 4 | from_date = "2018-01-01" 5 | to_date = "2018-01-31" 6 | url = "http://grains.krei.re.kr/chart/main_chart/index/kind/W/sdate/" + from_date + "/edate/" + to_date 7 | text = urlopen(url) 8 | 9 | json_obj = json.load(text) 10 | 11 | datas = [item['settlement'] for item in json_obj] 12 | 13 | print(datas) 14 | 15 | 16 | -------------------------------------------------------------------------------- /com/chapter10_grains/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter10_grains/__init__.py -------------------------------------------------------------------------------- /com/chapter11_public_data/02_url_encoding.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import quote 2 | 3 | seoul = "서울특별시" 4 | print(seoul) 5 | 6 | seoul = quote(seoul) 7 | print(seoul) 8 | -------------------------------------------------------------------------------- /com/chapter11_public_data/09_file_input_output.py: -------------------------------------------------------------------------------- 1 | f = open("C:\json_data\sample_data.txt", 'w') 2 | data = "easy_python_crawler" 3 | f.write(data) 4 | f.close() 5 | 6 | f = open("C:\json_data\sample_data.txt", 'r') 7 | for str in f.readlines(): 8 | print(str) 9 | f.close() 10 | -------------------------------------------------------------------------------- /com/chapter12_naver_shopping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter12_naver_shopping/__init__.py -------------------------------------------------------------------------------- /com/chapter12_naver_shopping/analyze.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./tumbler.json") 4 | 5 | print(df.count()) 6 | 7 | writer = pd.ExcelWriter("./tumbler.xlsx") 8 | df.to_excel(writer, "sheet1") 9 | #openpyxl 10 | -------------------------------------------------------------------------------- /com/chapter12_naver_shopping/stage_naver_shopping.py: -------------------------------------------------------------------------------- 1 | from libs.naver_shopping2.crawler import crawl 2 | from libs.naver_shopping2.parser import parse 3 | import json 4 | 5 | page_string = crawl('액박+패드') 6 | products = parse(page_string) 7 | print(len(products)) 8 | 9 | file = open("./products.json", "w+") 10 | file.write(json.dumps(products)) -------------------------------------------------------------------------------- /com/chapter13_selinium/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/com/chapter13_selinium/__init__.py -------------------------------------------------------------------------------- /data_go_kr/01_data_go_kr.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | url = 'http://openapi.molit.go.kr/OpenAPI_ToolInstallPackage/service/rest/RTMSOBJSvc/getRTMSDataSvcNrgTrade' 4 | queryParams = '?' + 'LAWD_CD=' +'11110' \ 5 | + '&DEAL_YMD=' + '201811' \ 6 | + '&serviceKey=' + 'QOIyVxx7oVtU%2B03Na7PG6BWsVsElYmp8921ouUzpePsIOHFnNAOXhlYy4lrwBeG31dKKRfdKRmTY2CY034o2qw%3D%3D' 7 | 8 | url = url + queryParams 9 | 10 | result = requests.get(url) 11 | bs_obj = BeautifulSoup(result.content, "html.parser") 12 | print(bs_obj) -------------------------------------------------------------------------------- /documents/02_naver_urlopen.md: -------------------------------------------------------------------------------- 1 | ## 네이버 첫 페이지 불러오기 2 | ```python 3 | import urllib.request 4 | import bs4 5 | 6 | url = "https://www.naver.com/" 7 | html = urllib.request.urlopen(url) 8 | 9 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 10 | 11 | print(bs_obj) 12 | ``` 13 | 1.urllib.request를 import한다. 14 | 2.bs4를 import한다. 15 | 3.bs_obj변수에 BeautifulSoup한 결과를 저장한다. 16 | 4.출력한다. 17 | -------------------------------------------------------------------------------- /documents/06_list.md: -------------------------------------------------------------------------------- 1 | ## []에 대해서 2 | * [] = list 3 | * []에는 값이 0개 한개 또는 여러개 들어갈 수 있습니다. 4 | * 각 값들은 ,로 구분 합니다. 5 | * ex) [1, 2, 3], [
  • ,
  • ] 6 | * 이렇게 생긴걸 list라는 자료구조 라고 합니다. 7 | 8 | ## tree, list 자료구조 9 | * tree는 나뭇가지 처럼 뻗어 나가는 것 10 | * ex) 책의 장, 절 11 | * 조직도 12 | * list는 순서대로 나열 13 | * ex) 식당 기다리는 순서 14 | * 학생 목록(출석부) 15 | 16 | ## List에서 하나만 뽑기 17 | ```python 18 | list = [1, 2, 3] 19 | extracted_item = list[0] 20 | print(list) 21 | ``` -------------------------------------------------------------------------------- /documents/data_go_kr.md: -------------------------------------------------------------------------------- 1 | # 공공데이터 API 호출하기 - 부동산 실거래가 2 | * https://www.data.go.kr/ 3 | * https://www.data.go.kr/dataset/3050988/openapi.do 4 | 5 | ## API 사용 신청하기 6 | * 공공데이터는 사용 신청을 해야 사용 할 수 있습니다. 7 | 8 | ## 시크릿 발급 받기 9 | * 한입에 웹 크롤링 -------------------------------------------------------------------------------- /documents/file.md: -------------------------------------------------------------------------------- 1 | ## File 2 | ```python 3 | with open("./hello.txt", mode="w+") as f: 4 | f.write("hello \n") 5 | ``` 6 | 파일에 데이터 저장하기 7 | 8 | 9 | ```python 10 | f1 = open("./hello.txt", mode='r') 11 | lines = f1.readlines() 12 | 13 | print(lines) 14 | ``` 15 | 16 | 여러줄 저장하기 17 | ```python 18 | import random 19 | 20 | file_name = "./save_line.txt" 21 | f1 = open(file_name, mode='w+') 22 | for item in range(0, 10): 23 | f1.write(str(random.randint(1, 100)) + "\n") 24 | f1.close() 25 | ``` 26 | -------------------------------------------------------------------------------- /documents/matplot.md: -------------------------------------------------------------------------------- 1 | ## python으로 차트 그리기 2 | 3 | ```python 4 | import matplotlib.pylab as plt 5 | 6 | x = [1, 2, 3] 7 | y = [1, 2, 3] 8 | 9 | plt.plot(x, y) 10 | plt.show() 11 | ``` 12 | 13 | -------------------------------------------------------------------------------- /documents/netd.md: -------------------------------------------------------------------------------- 1 | # 서브페이지에서 데이터 수집하기 2 | * 서브페이지의 데이터를 수집하려면 서브페이지의 링크를 알아야 합니다. 3 | * 서브 url수집하기 4 | * http://www.netd.ac.za/ 크롤링 5 | 6 | ## 페이지 분석 7 | * ol -> findAll("li") -> for li.find("a") -------------------------------------------------------------------------------- /excel/01_xlrd.py: -------------------------------------------------------------------------------- 1 | import xlrd 2 | from libs.naver_shopping.crawler import crawl 3 | from libs.naver_shopping.parser import parse 4 | 5 | workbook = xlrd.open_workbook('user_info.xlsx') 6 | worksheet = workbook.sheet_by_name('db') 7 | list = worksheet._cell_values 8 | 9 | for row in list[1:]: 10 | keyword = row[0] 11 | result = parse(crawl(keyword)) 12 | print(keyword, result) 13 | 14 | -------------------------------------------------------------------------------- /excel/02_xlrd.py: -------------------------------------------------------------------------------- 1 | import xlrd 2 | workbook = xlrd.open_workbook('user_info.xlsx') 3 | worksheet = workbook.sheet_by_name('db') 4 | list = worksheet._cell_values 5 | 6 | for row in list[1:]: 7 | keyword = row[0] 8 | print(keyword) 9 | 10 | -------------------------------------------------------------------------------- /excel/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/excel/__init__.py -------------------------------------------------------------------------------- /excel/crawler.py: -------------------------------------------------------------------------------- 1 | def crawl(keyword): 2 | return [keyword, " 함수가 실행 되었습니다."] -------------------------------------------------------------------------------- /excel/keywords.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/excel/keywords.xlsx -------------------------------------------------------------------------------- /excel/user_info.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/excel/user_info.xlsx -------------------------------------------------------------------------------- /gn_lib3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/gn_lib3/__init__.py -------------------------------------------------------------------------------- /gn_lib3/save_to_file.py: -------------------------------------------------------------------------------- 1 | 2 | def saveToFile(list, fileLocationName): 3 | file = open(fileLocationName, "w+") 4 | 5 | for item in list: 6 | file.write(str(item['name'])+"\n") 7 | file.close() -------------------------------------------------------------------------------- /image/01_downimage.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | 3 | 4 | url = "http://jolse.com/web/product/medium/201809/c3ba634b8a379ed945a9d853d2f7a5a5.jpg" 5 | filename = "./study_room2.jpg" 6 | 7 | urllib.request.urlretrieve(url, filename) 8 | -------------------------------------------------------------------------------- /image/02_downimage.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | 3 | 4 | info = {"url":"http://jolse.com/web/product/medium/201809/c3ba634b8a379ed945a9d853d2f7a5a5.jpg", 5 | "name":"01.jpg" 6 | } 7 | 8 | urllib.request.urlretrieve(info['url'], info['name']) 9 | -------------------------------------------------------------------------------- /lecture/01_gopax.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | def get_bs_obj(url): 5 | html = requests.get(url) 6 | 7 | print(html.content) 8 | return BeautifulSoup(html.content, "html.parser") 9 | 10 | url = "https://www.gopax.co.kr/" 11 | bs_obj = get_bs_obj(url) 12 | 13 | print(bs_obj) -------------------------------------------------------------------------------- /lecture/01_netd_crawl.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://www.netd.ac.za/?action=browse&category=Affiliation&order=asc" 5 | result = requests.get(url) 6 | bs_obj = BeautifulSoup(result.content, "html.parser") 7 | 8 | ol = bs_obj.find("ol") 9 | lis = ol.findAll("li") 10 | urlPrefix = "http://www.netd.ac.za/" 11 | for li in lis: 12 | print(urlPrefix + li.find("a")["href"]) 13 | -------------------------------------------------------------------------------- /lecture/02_netd.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://www.netd.ac.za/portal/?action=browse&category=Affiliation&maxresults=10&start=1&order=asc" 5 | result = requests.get(url) 6 | bs_obj = BeautifulSoup(result.content, "html.parser") 7 | # http://www.netd.ac.za/?action=view&identifier=oai%3Aunion.ndltd.org%3Acput%2Foai%3Alocalhost%3A20.500.11838%2F727 8 | 9 | ol = bs_obj.find("ol") 10 | lis = ol.findAll("li") 11 | for li in lis: 12 | print("http://www.netd.ac.za/" + li.find("a")["href"]) 13 | 14 | -------------------------------------------------------------------------------- /lecture/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/__init__.py -------------------------------------------------------------------------------- /lecture/apicall/02_simple_api_call_code.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | 4 | keyword = "디퓨저" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword # json 결과 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"JyP8ZGwBvgwzajxGWs7m", 8 | "X-Naver-Client-Secret":"WC3_t8TCMu"}) 9 | json_obj = result.json() 10 | print(json_obj) 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /lecture/chapter10_refactoring/01_call_naver_news.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | def get_bs_obj(url): 5 | result = requests.get(url) 6 | bs_obj = BeautifulSoup(result.content, "html.parser") 7 | return bs_obj 8 | 9 | url = "https://news.naver.com/" 10 | result = get_bs_obj(url) 11 | print(result) -------------------------------------------------------------------------------- /lecture/chapter10_refactoring/02_naver_api_news.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/chapter10_refactoring/02_naver_api_news.py -------------------------------------------------------------------------------- /lecture/chapter10_refactoring/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/chapter10_refactoring/__init__.py -------------------------------------------------------------------------------- /lecture/coupang/02_get_products.py: -------------------------------------------------------------------------------- 1 | from libs.crawler import crawl 2 | from bs4 import BeautifulSoup 3 | 4 | def getProducts(string): 5 | bsObj = BeautifulSoup(string, "html.parser") 6 | 7 | ul = bsObj.find("ul", {"id":"productList"}) 8 | lis = ul.findAll("li") 9 | li = lis[1] 10 | print(li) 11 | return [] 12 | 13 | 14 | url = "https://www.coupang.com/np/categories/186764?page=1" 15 | pageString = crawl(url) 16 | print(getProducts(pageString)) 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /lecture/coupang/03_get_product.py: -------------------------------------------------------------------------------- 1 | from libs.crawler import crawl 2 | from bs4 import BeautifulSoup 3 | from libs.coupang.productsParser import getProducts 4 | 5 | url = "https://www.coupang.com/np/categories/186764?page=2" 6 | pageString = crawl(url) 7 | products = getProducts(pageString) 8 | -------------------------------------------------------------------------------- /lecture/coupang/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/coupang/__init__.py -------------------------------------------------------------------------------- /lecture/graph/01_graph.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pylab as plt 3 | 4 | ''' 5 | x = y 6 | 1 1 7 | 2 2 8 | 0.5 0.5 9 | ''' 10 | x = np.arange(-5, 5, 0.1) 11 | y = lambda x: np.array(x > 2, dtype=int) 12 | 13 | print(y(x)) 14 | 15 | plt.plot(x, y(x)) 16 | plt.grid() 17 | plt.show() -------------------------------------------------------------------------------- /lecture/graph/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/graph/__init__.py -------------------------------------------------------------------------------- /lecture/jsonn/01_hello.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | f = open("./sample_data.json", 'r', encoding='utf-8') 5 | data = f.read() 6 | 7 | json_str = json.dumps(data) 8 | json_obj = json.loads(json_str) 9 | 10 | print(type(json_obj)) 11 | print(json_obj) 12 | -------------------------------------------------------------------------------- /lecture/lecture_3/01_function.py: -------------------------------------------------------------------------------- 1 | # 라이브러리 2 | import requests 3 | 4 | def get_page(url): 5 | result = requests.get(url) 6 | print(result.content) 7 | 8 | url = "https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2" 9 | get_page(url) -------------------------------------------------------------------------------- /lecture/lecture_3/02_http_request.py: -------------------------------------------------------------------------------- 1 | # 라이브러리 2 | import requests 3 | 4 | def get_page(url): 5 | result = requests.get(url) 6 | print(result.content) 7 | 8 | url = "https://finance.naver.com/marketindex/worldDailyQuote.nhn?marketindexCd=OIL_CL&fdtc=2" 9 | get_page(url) -------------------------------------------------------------------------------- /lecture/lecture_3/04_for.py: -------------------------------------------------------------------------------- 1 | def print_hello_using_for(): 2 | for num in range(0, 10): 3 | print(num) 4 | 5 | print_hello_using_for() -------------------------------------------------------------------------------- /lecture/lecture_3/05_list.py: -------------------------------------------------------------------------------- 1 | list = [2, 4, 3, 8] 2 | 3 | def print_list(): 4 | for num in list: 5 | print(num) 6 | 7 | print(list) 8 | print_list() -------------------------------------------------------------------------------- /lecture/lecture_3/06_print_td.py: -------------------------------------------------------------------------------- 1 | tds = ["2018.10.05", "74.32"] 2 | 3 | print(tds) 4 | for td in tds: 5 | print(td) -------------------------------------------------------------------------------- /lecture/lecture_3/07_http_request.py: -------------------------------------------------------------------------------- 1 | # http request 2 | 3 | # plus_one(x) = x + 1 4 | # f(1) = 2 5 | # f(2) = 3 6 | # f(url) = httprequest(url)-> 텍스트 7 | def get_text(url): 8 | return url + 1 9 | 10 | # fn(), g() 11 | 12 | url = "" 13 | y = get_text(url) 14 | print(y) -------------------------------------------------------------------------------- /lecture/lecture_3/08_get_text.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | def get_text(url): 4 | middle = requests.get(url) 5 | return middle.content 6 | 7 | 8 | url = "https://finance.naver.com/marketindex/exchangeDetail.nhn?marketindexCd=FX_USDKRW" 9 | result = get_text(url) 10 | print(result) 11 | -------------------------------------------------------------------------------- /lecture/lecture_3/09_get_bs_obj.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | def get_bs_obj(url): 5 | middle = requests.get(url) 6 | bs_obj = BeautifulSoup(middle.content, "html.parser") 7 | return bs_obj 8 | 9 | 10 | # url = "https://finance.naver.com/marketindex/exchangeDetail.nhn?marketindexCd=FX_USDKRW" 11 | url = "https://finance.naver.com/marketindex/exchangeDailyQuote.nhn?marketindexCd=FX_USDKRW" 12 | bs_obj = get_bs_obj(url) 13 | print(bs_obj) 14 | -------------------------------------------------------------------------------- /lecture/lecture_3/12_crawl_naver_webtoon.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url="https://comic.naver.com/webtoon/list.nhn?titleId=20853" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj=bs4.BeautifulSoup(html,"html.parser") 8 | table = bs_obj.find("table",{"class":"viewList"}) 9 | td = table.findAll("td",{"class":"title"}) 10 | 11 | print(type(td)) 12 | print(td) 13 | print(td[0]) 14 | 15 | for number in range(0, len(td)): 16 | print(td[number].text) 17 | -------------------------------------------------------------------------------- /lecture/lecture_3/14_jolse.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | bs_obj = BeautifulSoup(result.content, "html.parser") 7 | 8 | box = bs_obj.find("div", {"class":"box"}) 9 | aaa = box.find("a").find("img") 10 | productName = aaa['title'] 11 | 12 | ppp = box.find("p", {"class":"name"}) 13 | # saver.save("./1.html", productName) 14 | print(productName) 15 | 16 | -------------------------------------------------------------------------------- /lecture/lecture_3/15_jolse.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | bs_obj = BeautifulSoup(result.content, "html.parser") 7 | 8 | box = bs_obj.find("div", {"class":"box"}) 9 | aaa = box.find("a").find("img") 10 | productName = aaa['title'] 11 | 12 | lis = box.findAll("li", {"class":"xans-record-"}) 13 | spans = lis[1].findAll("span") 14 | print(spans[1].text) 15 | -------------------------------------------------------------------------------- /lecture/lecture_3/17_function_example.py: -------------------------------------------------------------------------------- 1 | # 함수로 값 넘기기 2 | def f(x): 3 | return x + 2 4 | 5 | def g(x): 6 | return x * 10 7 | # f o g(10) = f(g(10)) 8 | # f = getProductInfo() 9 | # g = print() 10 | # f o g 11 | 12 | 13 | numbers = [0, 1] 14 | 15 | print(f(0), f(1)) -------------------------------------------------------------------------------- /lecture/lecture_3/18_function_examples.py: -------------------------------------------------------------------------------- 1 | def plus(val1, val2): 2 | return val1 + val2 3 | 4 | plus(10, 20) -------------------------------------------------------------------------------- /lecture/lecture_3/19_return.py: -------------------------------------------------------------------------------- 1 | def getHello(): 2 | return "hello" 3 | 4 | result = getHello() 5 | print(result) 6 | -------------------------------------------------------------------------------- /lecture/lecture_3/20_parameter2.py: -------------------------------------------------------------------------------- 1 | # parameter가 1개인 function 2 | def printMessage(pMessage): 3 | print(pMessage) 4 | 5 | def printMessageWho(pMessage, pWho): 6 | print(pMessage, pWho) 7 | 8 | printMessage("hello") 9 | printMessageWho("hello","kyeongrok") 10 | -------------------------------------------------------------------------------- /lecture/lecture_3/22_with_file.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | 3 | html = open("./jolse_packpeel_27.html", encoding="utf-8") 4 | print(html) 5 | bs_obj = BeautifulSoup(html, "html.parser") 6 | 7 | boxes = bs_obj.findAll("div", {"class":"box"}) 8 | def getProductInfo(box): 9 | aaa = box.find("a").find("img") 10 | productName = aaa['title'] 11 | 12 | lis = box.findAll("li", {"class":"xans-record-"}) 13 | spans = lis[1].findAll("span") 14 | return spans[1].text 15 | 16 | box = boxes[0] 17 | print(getProductInfo(box)) 18 | -------------------------------------------------------------------------------- /lecture/lecture_3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_4/02_for.py: -------------------------------------------------------------------------------- 1 | 삼반 = ["철수", "영희", "태희"] 2 | 3 | for 학생 in 삼반: 4 | print(학생) 5 | -------------------------------------------------------------------------------- /lecture/lecture_4/03_for_2.py: -------------------------------------------------------------------------------- 1 | numbers = list(range(1, 10+1)) 2 | #array 배열 list 3 | print(numbers) 4 | print(numbers[3]) 5 | print(numbers[4]) -------------------------------------------------------------------------------- /lecture/lecture_4/04_for_numbers.py: -------------------------------------------------------------------------------- 1 | numbers = list(range(1, 10000+1)) 2 | print(numbers) 3 | 4 | # 반복문으로 numbers에서 number를 5 | # 한개씩 꺼내서 출력 해보세요 6 | for number in numbers: 7 | print(number) 8 | -------------------------------------------------------------------------------- /lecture/lecture_4/05_2_dan.py: -------------------------------------------------------------------------------- 1 | numbers = list(range(1, 10)) 2 | # 2 * 1 = 2 .... 3 | # 2 * 2 = 4 4 | # ..... 5 | # 2 * 9 = 18 6 | for number in numbers: 7 | print("2", "*", number, "=", 2*number) 8 | -------------------------------------------------------------------------------- /lecture/lecture_4/06_function.py: -------------------------------------------------------------------------------- 1 | # 함수 펑션 function 2 | # f(x) = x + 2 3 | def print_hello(): 4 | print("hello") 5 | 6 | print_hello() -------------------------------------------------------------------------------- /lecture/lecture_4/07_parameter.py: -------------------------------------------------------------------------------- 1 | # parameter 파라메터 매개변수 2 | # 함수이름옆 괄호안에 들어가는 것 3 | # 값을 함수로 전달 할 때 쓴다. 4 | def print_messge(p_message): 5 | print(p_message) 6 | 7 | print_messge("2") 8 | print_messge("3") 9 | -------------------------------------------------------------------------------- /lecture/lecture_4/08_print.py: -------------------------------------------------------------------------------- 1 | solars = ['a', 'b', 'c'] 2 | print('hello') 3 | 4 | print('{}월 {}일'.format(10, 27)) -------------------------------------------------------------------------------- /lecture/lecture_4/09_file_open.py: -------------------------------------------------------------------------------- 1 | file = open('names.cvs') 2 | lines = file.readlines() 3 | print(lines) 4 | 5 | print("%d %s" % (10, "월")) 6 | print("%d %s" % ("aa", "월")) 7 | -------------------------------------------------------------------------------- /lecture/lecture_4/10_crawl_codes.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://bigdata-trader.com/itemcodehelp.jsp" 5 | result = requests.get(url) 6 | bsObj = BeautifulSoup(result.content, "html.parser") 7 | 8 | trs = bsObj.find("table").findAll("tr") 9 | 10 | codes = [tr.find("a").text + "\n" for tr in trs] 11 | 12 | file = open("./codes.txt", "w+") 13 | file.writelines(codes) 14 | -------------------------------------------------------------------------------- /lecture/lecture_4/11_crawl_pages.py: -------------------------------------------------------------------------------- 1 | import gn_lib3.crawler as crawler 2 | codes = open("./codes.txt").readlines() 3 | infos = open("./infos.txt", "w+") 4 | 5 | for code in codes[:10]: 6 | companyInfo = crawler.getPrice(code) 7 | message = "{}@{}".format(companyInfo['company_name'], companyInfo['now']) 8 | infos.write(message + "\n") 9 | print(message) 10 | 11 | # getPrice 얘는 code 12 | # save to excel 얘는 index 13 | # -------------------------------------------------------------------------------- /lecture/lecture_4/12_for_each.py: -------------------------------------------------------------------------------- 1 | codes = ["000660", "091245"] 2 | 3 | # for each 4 | for code in codes: 5 | print("each=>", code) 6 | 7 | # for index 8 | for iii in range(0, 2): 9 | print("index=>", iii) 10 | -------------------------------------------------------------------------------- /lecture/lecture_4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_4/infos.txt: -------------------------------------------------------------------------------- 1 | 3S@2,270 2 | AJ네트웍스@4,720 3 | AJ렌터카@12,050 4 | AK홀딩스@46,650 5 | APS홀딩스@4,300 6 | AP시스템@26,600 7 | AP위성@7,090 8 | -------------------------------------------------------------------------------- /lecture/lecture_4/names.cvs: -------------------------------------------------------------------------------- 1 | hello 2 | world 3 | bye 4 | welcome 5 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/pocket_monster/01_pikachu.py: -------------------------------------------------------------------------------- 1 | # 전기공격을 하는 피카츄 2 | def pikachu(): 3 | print("전기공격") 4 | 5 | # 몬스터볼인데 피카츄가 들어있음 6 | pikachu() -------------------------------------------------------------------------------- /lecture/lecture_dorm/pocket_monster/02_kkobuggi.py: -------------------------------------------------------------------------------- 1 | # 물대포를 쏘는 꼬부기 2 | def 꼬부기(): 3 | print("물대포") 4 | 5 | # 몬스터볼 6 | 꼬부기() -------------------------------------------------------------------------------- /lecture/lecture_dorm/pocket_monster/03_파이리.py: -------------------------------------------------------------------------------- 1 | def 파이리(): 2 | print("파이어볼") 3 | 4 | 파이리() -------------------------------------------------------------------------------- /lecture/lecture_dorm/pocket_monster/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/pocket_monster/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/01_make_function_01.py: -------------------------------------------------------------------------------- 1 | def print_bye(): 2 | print("bye") 3 | 4 | print_bye() -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/02_make_pikkachu.py: -------------------------------------------------------------------------------- 1 | def pikachu(): 2 | print("전기공격") 3 | 4 | pikachu() 5 | 6 | def kkobugi(): 7 | print("물대포") 8 | 9 | def pirie(): 10 | print("파이어볼") 11 | kkobugi() 12 | pirie() -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/03_if.py: -------------------------------------------------------------------------------- 1 | res_code = 200 2 | if res_code == 200: 3 | print("ok") 4 | else: 5 | print("error") 6 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/04_else_if.py: -------------------------------------------------------------------------------- 1 | res_code = 300 2 | 3 | if res_code == 200: 4 | print("ok") 5 | elif res_code == 300: 6 | print("ok too") 7 | else: 8 | print("error") -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/05_get_score.py: -------------------------------------------------------------------------------- 1 | # 학생들 점수 2 | # score에 있는 점수의 등급을 매기는 if문을 짜보세요 3 | # >=90 A, >=80 B, >=70 C, < 70 F 4 | 5 | score = 75 # 86, 90, 75, 32 6 | 7 | if score >= 90: 8 | print("A") 9 | elif score >= 80: 10 | print("B") 11 | elif score >= 70: 12 | print("C") 13 | else: 14 | print("F") -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/06_function_if.py: -------------------------------------------------------------------------------- 1 | def get_grade(score): 2 | #score>=90 A score>=80 B score>=70 C F 3 | if score>=90: 4 | print("A") 5 | elif score>=80: 6 | print("B") 7 | elif score>=70: 8 | print("C") 9 | else: 10 | print("F") 11 | 12 | # 86, 90, 75, 32 13 | get_grade(86) 14 | get_grade(90) 15 | get_grade(75) 16 | get_grade(32) -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/07_function_pika.py: -------------------------------------------------------------------------------- 1 | #전기공격을 하는 피카츄 2 | #1 전기공격 #2 몸통박치기 #3 speed up 3 | def pikachu(number): 4 | if number == 1: 5 | print("전기공격") 6 | elif number == 2: 7 | print("몸통박치기") 8 | else: 9 | print("speed up") 10 | pikachu(3) -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/08_return.py: -------------------------------------------------------------------------------- 1 | def plus(val1, val2): 2 | print(val1 + val2) 3 | 4 | result = plus(10, 20) 5 | 6 | print(result) -------------------------------------------------------------------------------- /lecture/lecture_dorm/week2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/week2/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/week3/01_strange_seed.py: -------------------------------------------------------------------------------- 1 | # 이상해씨 만들기 1덩굴채찍 2잎날가르기 3솔라빔 2 | # 13:08 3 | 4 | def strange(skill_number): 5 | if skill_number == 1: 6 | return("whip") 7 | elif skill_number == 2: 8 | return("잎날가르기") 9 | elif skill_number == 3: 10 | return("solar beam") 11 | 12 | print(strange(2)) 13 | print(strange(1)) 14 | print(strange(3)) -------------------------------------------------------------------------------- /lecture/lecture_dorm/week3/02_urlopen.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "http://ko.pokemon.wikia.com/wiki/%EC%9D%B4%EC%83%81%ED%95%B4%EC%94%A8_(%ED%8F%AC%EC%BC%93%EB%AA%AC)/1%EC%84%B8%EB%8C%80_%EA%B8%B0%EC%88%A0" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | print(bs_obj) 9 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week3/03_urlopen2.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "http://ko.pokemon.wikia.com/wiki/%EC%9D%B4%EC%83%81%ED%95%B4%EC%94%A8_(%ED%8F%AC%EC%BC%93%EB%AA%AC)/1%EC%84%B8%EB%8C%80_%EA%B8%B0%EC%88%A0" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | # print(bs_obj) 9 | headline = bs_obj.find("span", {"class":"mw-headline"}) 10 | explore_activity = bs_obj.find("a", {"data-tracking":"explore-activity"}) 11 | print(headline.text) 12 | print(explore_activity.text) 13 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week3/03_urlopen_naver.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | # url을 받아서 bs_obj를 return하는 function 5 | # 으로 만들어보세요 6 | 7 | def get_bs_obj(url): 8 | html = urllib.request.urlopen(url) 9 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 10 | return bs_obj 11 | url="https://www.naver.com" 12 | bs_obj = get_bs_obj(url) 13 | to_start = bs_obj.find("a",{"data-clk":"top.mkhome"}) 14 | print(to_start.text) -------------------------------------------------------------------------------- /lecture/lecture_dorm/week3/07_urlopen.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://finance.naver.com/world/sise.nhn?symbol=DJI@DJI&fdtc=0" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html, "html.parser") 8 | print(bs_obj) 9 | 10 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/week3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/week5_high_low_open_close/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/week5_high_low_open_close/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/week5_requests_refactoring/01_start.py: -------------------------------------------------------------------------------- 1 | # crawling 하는 코드 2 | import requests 3 | from bs4 import BeautifulSoup 4 | 5 | url = "https://finance.naver.com/item/main.nhn?code=005930" 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | print(bs_obj) 9 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week5_requests_refactoring/02_complete.py: -------------------------------------------------------------------------------- 1 | # crawling 하는 코드 2 | import requests 3 | from bs4 import BeautifulSoup 4 | 5 | # url을 넣어서 bs_obj를 return하는 function 6 | def get_bs_obj(url): 7 | result = requests.get(url) 8 | bs_obj = BeautifulSoup(result.content, "html.parser") 9 | return bs_obj 10 | 11 | url = "https://finance.naver.com/item/main.nhn?code=005930" 12 | bs_obj = get_bs_obj(url) 13 | print(bs_obj) 14 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week5_requests_refactoring/03_get_price.py: -------------------------------------------------------------------------------- 1 | # crawling 하는 코드 2 | import requests 3 | from bs4 import BeautifulSoup 4 | 5 | # url을 넣어서 bs_obj를 return하는 function 6 | def get_bs_obj(url): 7 | result = requests.get(url) 8 | bs_obj = BeautifulSoup(result.content, "html.parser") 9 | return bs_obj 10 | 11 | url = "https://finance.naver.com/item/main.nhn?code=005930" 12 | bs_obj = get_bs_obj(url) 13 | no_today = bs_obj.find("p", {"class":"no_today"}) 14 | blind = no_today.find("span", {"class":"blind"}) 15 | 16 | print(blind.text) 17 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week5_requests_refactoring/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/week5_requests_refactoring/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/week6_jolse/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/week6_jolse/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/week7_naver_api/02_use_requests.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.request import urlparse 3 | 4 | keyword = "강남역 탁구장" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | print(result.json()) -------------------------------------------------------------------------------- /lecture/lecture_dorm/week7_naver_api/03_use_requests.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.request import urlparse 3 | 4 | keyword = "강남역 탁구장" 5 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword 6 | result = requests.get(urlparse(url).geturl(), 7 | headers={"X-Naver-Client-Id":"H1DC13DICQ8zIK84XwWn", 8 | "X-Naver-Client-Secret":"8sUQqFukKQ"}) 9 | json_obj = result.json() 10 | 11 | for item in json_obj['items']: 12 | print(item) 13 | -------------------------------------------------------------------------------- /lecture/lecture_dorm/week7_naver_api/04_use_req.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/week7_naver_api/04_use_req.py -------------------------------------------------------------------------------- /lecture/lecture_dorm/week7_naver_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm/week7_naver_api/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm2/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/01_review.py: -------------------------------------------------------------------------------- 1 | def esh(p_skill_num): 2 | if p_skill_num == 1: 3 | print ("태클") 4 | elif p_skill_num == 2: 5 | print ("울부짖기") 6 | elif p_skill_num == 3: 7 | print ("씨뿌리기") 8 | 9 | esh(1) 10 | esh(2) 11 | esh(3) -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/02_esh_return.py: -------------------------------------------------------------------------------- 1 | def print_esh(p_skill_num): 2 | if p_skill_num == 1: 3 | return "태클" 4 | elif p_skill_num == 2: 5 | return "울부짖기" 6 | elif p_skill_num == 3: 7 | return "씨뿌리기" 8 | 9 | print(print_esh(1)) 10 | print(print_esh(2)) 11 | print(print_esh(3)) -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/03_gugudan.py: -------------------------------------------------------------------------------- 1 | # 2 | def print_table (): 3 | for x in range(2,10): 4 | print("2","X",x,"=",2*x) 5 | 6 | print(print_table()) -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/04_gugudan_n.py: -------------------------------------------------------------------------------- 1 | def gugudann(dan): 2 | for num in range(1,10): 3 | print(dan,"*",num,"=",dan*num) 4 | 5 | gugudann(4) 6 | -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/05_gugudan_2_to_9.py: -------------------------------------------------------------------------------- 1 | def gugudann(dan): 2 | for num in range(1,10): 3 | print(dan,"*",num,"=",dan*num) 4 | 5 | 6 | for dan in range(2, 10): 7 | gugudann(dan) 8 | 9 | -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/07_dict.py: -------------------------------------------------------------------------------- 1 | price = 1000 2 | print(price) 3 | 4 | user = {"name":"kyeongrok", "age":"32", "level":10} 5 | print(user) 6 | print(user['name']) 7 | print(user['level']) 8 | print(user['level'] * 1000) 9 | print(user['32'] * 1000) 10 | 11 | 12 | -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/08_type.py: -------------------------------------------------------------------------------- 1 | print(10 + 10) 2 | print("10" + "10") -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm2/week2/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm2/week3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week3/review/01_gugudan_2_dan.py: -------------------------------------------------------------------------------- 1 | def multiple(a,b): 2 | return a*b 3 | 4 | for a in range (2,10): 5 | for number in range(1,10): 6 | print(a,"x", number, "=", multiple(a, number)) 7 | -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week3/review/02_naver_finance_call_page.py: -------------------------------------------------------------------------------- 1 | from urllib. request import urlopen 2 | 3 | def get_text(url): 4 | html = urlopen(url) 5 | return html.read().decode("euc-kr") 6 | 7 | url = "https://finance.naver.com/item/main.nhn?code=215600" 8 | html = get_text(url) 9 | print(html) 10 | 11 | -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week3/review/03_get_bs_obj.py: -------------------------------------------------------------------------------- 1 | from urllib. request import urlopen 2 | from bs4 import BeautifulSoup # tree 구조로 접근할 수 있게 해줌 3 | 4 | def get_bs_obj(url): 5 | html = urlopen(url) 6 | bs_obj = BeautifulSoup(html.read().decode("euc-kr")) 7 | return bs_obj 8 | 9 | 10 | url = "https://finance.naver.com/item/main.nhn?code=215600" 11 | bs_obj = get_bs_obj(url) 12 | print(bs_obj) 13 | 14 | -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week3/review/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm2/week3/review/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week4/02_code_list.py: -------------------------------------------------------------------------------- 1 | list = [ 2 | "060310", 3 | "095570", 4 | "190150" 5 | ] 6 | 7 | for item in list: 8 | print(item) -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week4/03_dictionray.py: -------------------------------------------------------------------------------- 1 | def getPrice(code): 2 | # 봉차트 candle 3 | # 시가(open) 2000 4 | # 고가(high) 3000 5 | # 저가(low) 1000 6 | # 종가(close) 2500 7 | # price = 종가 8 | 9 | return {"close":1000, "open":2000, 10 | "high":3000, "low":1000} 11 | 12 | result = getPrice("234233") 13 | print(result['open']) -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm2/week4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm2/week5/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week6/01_jolse.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import libs.dorm_libs.parser_shoppingmall as parser 3 | 4 | bs_obj = BeautifulSoup(open("./jolse.html", 5 | encoding="utf-8").read(), "html.parser") 6 | 7 | result = parser.parse(bs_obj) 8 | file = open("./product_infos.csv", "w+") 9 | 10 | for item in result: 11 | file.write(item['name']+","+item['price']+"\n") 12 | file.close() -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week6/02_file.py: -------------------------------------------------------------------------------- 1 | file = open("./hello.txt", "w+") 2 | list = [ 3 | {"name":"hello"}, 4 | {"name":"world"} 5 | ] 6 | for item in list: 7 | file.write(item['name']+"\n") 8 | file.close() -------------------------------------------------------------------------------- /lecture/lecture_dorm2/week6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_dorm2/week6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn/week3/01_crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://finance.naver.com/item/main.nhn?code=068270" 5 | result = requests.get(url) 6 | bs_obj = BeautifulSoup(result.content, "html.parser") 7 | p_no_today = bs_obj.find("p", {"class":"no_today"}) # 서울시 강남구 역삼동 8 | span_blind = p_no_today.find("span", {"class":"blind"}) 9 | 10 | print(span_blind.text) # 228,000 -------------------------------------------------------------------------------- /lecture/lecture_gn/week3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn/week3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn/week3/code_lists.py: -------------------------------------------------------------------------------- 1 | import gn_lib3.crawler as crawler 2 | codes = [ 3 | "060310", 4 | "095570", 5 | "000660", # 67000 6 | "161490" 7 | ] 8 | for code in codes: 9 | try: 10 | print("=>", crawler.getPrice(code)) 11 | except: 12 | print("error", code) 13 | 14 | # print(crawler.getPrice(codes[0])) -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/01_gugudan.py: -------------------------------------------------------------------------------- 1 | # 구구단 2단 출력하는 함수를 만들고 실행 해보세요 2 | # 유투브 '파이썬 크롤링'검색 3 | # 단톡방 링크 스크롤 스크롤 4 | 5 | # 구구단 2단 6 | # 구구단 3단 7 | # parameter 8 | def gugudan(dan): 9 | for number in range(1, 9+1): 10 | print("{} * {} = {}".format(dan, number,dan*number)) 11 | 12 | # 2단 ~ 9단 동시에 13 | for dan in range(2, 9+1): 14 | print("-------------------") 15 | gugudan(dan) 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/02_function.py: -------------------------------------------------------------------------------- 1 | # 함수 만들기 2 | def printHello(): 3 | print("hello") 4 | 5 | printHello() -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/03_parameter.py: -------------------------------------------------------------------------------- 1 | # 파라메터 parameter 매개변수 args 2 | # argument 아규먼트 알규먼트 3 | def printMessage(pMessage): 4 | print(pMessage) 5 | 6 | printMessage("welcome") 7 | 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/04_http_request.py: -------------------------------------------------------------------------------- 1 | # 라이브러리 임포트하기 2 | import urllib.request 3 | 4 | # http request(urllib.re~~)를 보내는 방법이 써있는책 5 | # 크롤링 하는 방법(한입에~)이 써있는 책 6 | 7 | # 라이브러리 library 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/05_http_request_call.py: -------------------------------------------------------------------------------- 1 | # 라이브러리 임포트하기 2 | import urllib.request 3 | 4 | url = "https://finance.naver.com/item/main.nhn?code=068270" 5 | html = urllib.request.urlopen(url) 6 | 7 | print(html.read()) -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/07_bs4_find.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | url = "https://finance.naver.com/item/main.nhn?code=068270" 5 | html = urllib.request.urlopen(url) 6 | 7 | bs_obj = bs4.BeautifulSoup(html.read().decode("euc-kr"), "html.parser") 8 | #

    ___

    # 서울시 강남구 9 | no_today = bs_obj.find("p", {"class":"no_today"}) 10 | blind = no_today.find("span", {"class":"blind"}) 11 | 12 | print(blind.text) 13 | 14 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/08_adjust_function.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import bs4 3 | 4 | def getPrice(): 5 | url = "https://finance.naver.com/item/main.nhn?code=068270" 6 | html = urllib.request.urlopen(url) 7 | 8 | bs_obj = bs4.BeautifulSoup(html.read().decode("euc-kr"), "html.parser") 9 | #

    ___

    # 서울시 강남구 10 | no_today = bs_obj.find("p", {"class":"no_today"}) 11 | blind = no_today.find("span", {"class":"blind"}) 12 | # 13 | print(blind.text) 14 | 15 | getPrice("068270") 16 | getPrice("000660") 17 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn/week4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn/week5/01_crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | url = "http://jolse.com/category/eyebrow/68/?page=1" 4 | result = requests.get(url) 5 | 6 | file = open("./eyebrow_1.html","w+") 7 | file.write(str(result.content)) 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn/week5/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn/week5/jolse_stage.py: -------------------------------------------------------------------------------- 1 | import libs.jolse.jolse_parser as parser 2 | 3 | def getProdList(fileName): 4 | file = open(fileName, encoding="utf-8") 5 | str = file.read() 6 | prodList = parser.parse(str) 7 | return prodList 8 | 9 | prodList1 = getProdList("./troublecare_1.html") 10 | prodList2 = getProdList("./troublecare_2.html") 11 | 12 | mergedList = prodList1 + prodList2 13 | 14 | file = open("./product_info.csv", "w+") 15 | for item in mergedList: 16 | file.write(item['name'] + "@" + item['price'] + "\n") 17 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/01_review_crawl_parse_save.py: -------------------------------------------------------------------------------- 1 | import libs.jolse.jolse_crawl as crawler 2 | 3 | url = "http://jolse.com/category/porecare/52/?page=1" 4 | crawler.crawl(url, "./jolse_01.html") 5 | 6 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/02_open_and_parse.py: -------------------------------------------------------------------------------- 1 | import libs.jolse.jolse_parser as parser 2 | import gn_lib3.save_to_file as saver 3 | 4 | file = open("./jolse_01.html") 5 | 6 | result = parser.parse(file.read()) 7 | print("=>",result) 8 | print(len(result)) 9 | 10 | saver.saveToFile(result, "./jolse_01.csv") 11 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/03_review_requests.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | # url을 가지고 str 4 | url = "http://jolse.com/category/cleansing/58/" 5 | def crawl(url, fileLocationName): 6 | result = requests.get(url) 7 | file = open(fileLocationName, "w+") 8 | file.write(str(result.content)) 9 | 10 | crawl(url, "./jolse_02.html") 11 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/04_open_and_parse.py: -------------------------------------------------------------------------------- 1 | import libs.jolse.jolse_parser as parser 2 | 3 | file = open("./jolse_02.html") 4 | result = parser.parse(file.read()) 5 | print(len(result)) 6 | 7 | savedFile = open("./jolse_02.csv", "w+") 8 | for item in result: 9 | savedFile.write(str(item['name']) + "\n") -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/07_dict.py: -------------------------------------------------------------------------------- 1 | thisdict = { 2 | "brand":"Ford", 3 | "moder":"Mustang", 4 | "year": 1964, 5 | "total":12383838 6 | } 7 | 8 | print(thisdict) -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/08_dict_2.py: -------------------------------------------------------------------------------- 1 | # key, value 2 | user = {"name":"kyeongrok", "age":32} 3 | 4 | print(user["name"]) 5 | print(user["age"]) 6 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/09_save_a_line.py: -------------------------------------------------------------------------------- 1 | file = open("./hello.csv", "w+") 2 | file.write("hello" + "\n") 3 | file.write("bye" + "\n") 4 | 5 | list = ["welcome", "glad"] 6 | for item in list: 7 | file.write(item + "\n") 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn/week6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn/week7/01_list.py: -------------------------------------------------------------------------------- 1 | list = [] 2 | subList1page = [{"title":"삼겹살"}, 3 | {"title":"파스타"}] 4 | subList2page = [{"title":"왕만두"}, 5 | {"title":"컵밥"}] 6 | list = list + subList1page 7 | list = list + subList2page 8 | print(list) -------------------------------------------------------------------------------- /lecture/lecture_gn/week7/02_pandas.py: -------------------------------------------------------------------------------- 1 | # python pandas 2 | # data분석 툴 3 | # 파이썬 안에서 표 형태의 데이터를 분석할때 씁니다. 4 | # excel like 5 | import pandas as pd 6 | 7 | dataframe = pd.read_json("./data_file.json") 8 | print(dataframe['bloggername']) 9 | 10 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week7/03_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | list = [ 3 | {"title":"hello", "blogger":"world"} 4 | ] 5 | file = open("./data_file.json", "w+") 6 | file.write(json.dumps(list)) 7 | 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week7/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn/week7/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn/week7/stage2.py: -------------------------------------------------------------------------------- 1 | from libs.naver_api.naver_api_caller import get1000result 2 | file = open("./result.json", "w+", encoding="utf-8") 3 | 4 | list = [] 5 | keywords = ["강남역 맛집", "강남역 공연", "강남역 python"] 6 | for keyword in keywords: 7 | list = list + get1000result(keyword) 8 | 9 | import json 10 | file = open("./data_file.json", "w+") 11 | file.write(json.dumps(list)) 12 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week7/user.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"title":"왕만두집", "blogger":"kyeongrok"}, 3 | {"title":"찐만두집", "blogger":"kyeongrok"}, 4 | {"title":"치킨집", "blogger":"chenenem"} 5 | ] -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/01_analyze.py: -------------------------------------------------------------------------------- 1 | #01_analyze.py 2 | import pandas as pd 3 | 4 | df = pd.read_json("./search_result.json") 5 | print(df.count()) 6 | 7 | 8 | # pandas는 data를 dataframe으로 만들어 줍니다. 9 | # dataframe안에 데이터를 넣으면 표처럼 쓸 수 있다. 10 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/02_to_excel.py: -------------------------------------------------------------------------------- 1 | #01_analyze.py 2 | #openpyxl 3 | import pandas as pd 4 | 5 | df = pd.read_json("./search_result.json") 6 | print(df.count()) 7 | 8 | writer = pd.ExcelWriter('output.xlsx') 9 | df.to_excel(writer,'Sheet1') 10 | writer.save() -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/03_datetime.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./search_result.json") 4 | print(df.count()) 5 | print(df['postdate']) 6 | df['postdate'] = pd.to_datetime(df['postdate'], format="%Y%m%d") 7 | 8 | print(df['postdate']) 9 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/04_groupby.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | df = pd.read_json("./search_result.json") 3 | 4 | print(df.groupby(['bloggername']).count()) 5 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/05_column.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | df = pd.read_json("./search_result.json") 3 | 4 | dfLink = df['link'] 5 | print(dfLink) 6 | 7 | writer = pd.ExcelWriter('links.xlsx') 8 | dfLink.to_excel(writer,'link') 9 | writer.save() 10 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/06_copy.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | df = pd.read_json("./search_result.json") 3 | 4 | print(df.columns) 5 | 6 | df2 = df[['postdate', 'title', 'bloggername', 'link']] 7 | df2Sorted = df2.sort_values(['postdate'], ascending=[0]) 8 | print(df2Sorted) 9 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/07_filter.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | df = pd.read_json("./search_result.json") 3 | 4 | df2 = df[['bloggername', 'link', 'title']] 5 | df3 = df[df['bloggername'] == '허니드롭'] 6 | 7 | print(df3['title']) 8 | for link in df3['link']: 9 | print(link) 10 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/08_sort.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | df = pd.read_json("./search_result.json") 3 | 4 | print(df.columns) 5 | 6 | df2 = df[['postdate', 'title', 'bloggername', 'link']] 7 | df2Sorted = df2.sort_values(['postdate'], ascending=[0]) 8 | print(df2Sorted) 9 | 10 | print(df2Sorted.head(), df2Sorted.columns) 11 | print(df2Sorted[['postdate', 'title']].head(100)) 12 | # writer = pd.ExcelWriter('sorted.xlsx') 13 | # df2Sorted.to_excel(writer,'sorted') 14 | # writer.save() 15 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/09_head.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | df = pd.read_json("./search_result.json") 3 | 4 | df2 = df[['postdate', 'title', 'bloggername', 'link']] 5 | df2Sorted = df2.sort_values(['postdate'], ascending=[0]) 6 | df2SortedAndTop10 = df2Sorted[['postdate', 'title', 'link']].head(10) 7 | 8 | for item in df2SortedAndTop10['link']: 9 | print(item) 10 | 11 | # writer = pd.ExcelWriter('top10.xlsx') 12 | # df2SortedAndTop10.to_excel(writer,'top10') 13 | # writer.save() 14 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/10_to_date.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./search_result.json") 4 | # print(df['postdate']) 5 | 6 | df2 = df[df['postdate'] != ''] 7 | df2['postdate'] = pd.to_datetime(df2['postdate'], format="%Y%m%d") 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/11_index_of.py: -------------------------------------------------------------------------------- 1 | title = "강남역 고기집은 00가 맛있어요" 2 | 3 | print(title.index("고기집")) 4 | print(title.find("고기집")) 5 | print(title.find("횟집")) 6 | 7 | -------------------------------------------------------------------------------- /lecture/lecture_gn/week8/stage.py: -------------------------------------------------------------------------------- 1 | from libs.naver_api.naver_api_caller import get1000Result 2 | import json 3 | 4 | keywords = ["강남역 공연", "강남역 맛집", "강남역 오피스텔"] 5 | list = [] 6 | for keyword in keywords: 7 | result = get1000Result("강남역 공연") 8 | list = list + result 9 | print(len(list)) 10 | file = open("./search_result.json", "w+") 11 | file.write(json.dumps(list)) 12 | 13 | -------------------------------------------------------------------------------- /lecture/lecture_gn2/week5/01_file.py: -------------------------------------------------------------------------------- 1 | import json 2 | jsonObj = {"name":"다우니", "price":"12000"} 3 | 4 | file = open("./product.json", "w+") 5 | file.write(json.dumps(jsonObj)) -------------------------------------------------------------------------------- /lecture/lecture_gn2/week5/02_pandas.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./results.json") 4 | 5 | writer = pd.ExcelWriter('output.xlsx') 6 | df.to_excel(writer,'Sheet1') 7 | writer.save() 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn2/week5/03_pandas_coupang.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./coupangResults.json") 4 | 5 | writer = pd.ExcelWriter('outputCoupang.xlsx') 6 | df.to_excel(writer,'Sheet1') 7 | writer.save() 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn2/week5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn2/week5/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn2/week5/naverApiStage.py: -------------------------------------------------------------------------------- 1 | # lib import하기 2 | import json 3 | from libs.naver.apiCrawler import crawl 4 | 5 | def getSearchResults(keyword): 6 | results = [] 7 | for num in range(0, 10): 8 | items = crawl(keyword, 100 * num + 1)['items'] 9 | print(items) 10 | results += items 11 | return results 12 | 13 | results = getSearchResults("강남역 저녁") 14 | print(results) 15 | file = open("./results.json", "w+") 16 | file.write(json.dumps(results)) 17 | -------------------------------------------------------------------------------- /lecture/lecture_gn2/week6/01_pandas.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | # pandas로 데이터 불러오기 4 | df = pd.read_json("./naverKeywordResult.json") 5 | 6 | # 잘 불러왔느지 확인하기 7 | print(df.count()) 8 | 9 | # 데이터가 어떻게 생겼는지 알아보기 10 | 11 | # column명, 필드(field)명, 열제목 12 | 13 | # 블로거 이름을 가지고 필터링 하기 14 | # 블로거 이름이 기준 15 | 16 | # top5 블로거가 검색결과 1000개중에 글을 몇개나 썼을까요? 17 | # 가장 많이 글을 쓴 블로거는 누구일까요? 18 | 19 | print(df.head(5)['bloggername']) 20 | 21 | dfResult = df[df['bloggername'] == "명랑 뮬란의 오늘 뭐먹지?"] 22 | 23 | 24 | -------------------------------------------------------------------------------- /lecture/lecture_gn2/week6/02_filter_and_print.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | pd.set_option('display.max_colwidth', -1) 3 | # pandas로 데이터 불러오기 4 | df = pd.read_json("./naverKeywordResult.json") 5 | 6 | for bloggername in df.head(100)['bloggername']: 7 | dfResult = df[df['bloggername'] == bloggername] 8 | print(bloggername, len(dfResult)) 9 | 10 | dfTop1 = df[df['bloggername'] == "책과 음식과 영화이야기"] 11 | for item in dfTop1['link']: 12 | print(item.replace("amp;", "")) 13 | -------------------------------------------------------------------------------- /lecture/lecture_gn2/week6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn2/week6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn2/week7/analyzer.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./result.json") 4 | 5 | print(df.count()) 6 | 7 | writer = pd.ExcelWriter('output.xlsx') 8 | df.to_excel(writer,'Sheet1') 9 | writer.save() 10 | 11 | # 형태소 분석, text 분석 -------------------------------------------------------------------------------- /lecture/lecture_gn2/week7/milkAnalyzeReportTable.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn2/week7/milkAnalyzeReportTable.json -------------------------------------------------------------------------------- /lecture/lecture_gn3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn3/week2/07_for_for.py: -------------------------------------------------------------------------------- 1 | # 1 to 20 2 | def printNumbers(pageNum): 3 | list = [] 4 | for num in range(1, 20 + 1): 5 | list.append(num * pageNum) 6 | return list 7 | 8 | # 1 page to 17 page 9 | for num in range(1, 18): 10 | print(printNumbers(num)) 11 | 12 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week2/08_naver_finance_crawl.py: -------------------------------------------------------------------------------- 1 | # hello를 return하는 crawl이라는 이름의 함수를 만들어보세요 2 | import requests 3 | 4 | # url -> string = 크롤링 -> 크롤러 crawler 5 | # string -> [], {} 파싱 -> 파서 parser 6 | 7 | def crawl(): 8 | url = "https://finance.naver.com/item/main.nhn?code=068270" 9 | data = requests.get(url) 10 | return data.content 11 | 12 | print(crawl()) -------------------------------------------------------------------------------- /lecture/lecture_gn3/week2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn3/week2/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn3/week3/01_crawl_naver.py: -------------------------------------------------------------------------------- 1 | from libs.NaverFinance.crawl import crawl 2 | from libs.NaverFinance.parser import getCompanyInfo 3 | 4 | #025980 아난티, 000660 sk하이닉스 5 | 6 | codes = ["000660", "025980", "005930"] # 종목코드표 7 | 8 | for code in codes: 9 | print(getCompanyInfo(crawl(code))) 10 | 11 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week3/02_file_open.py: -------------------------------------------------------------------------------- 1 | file = open("./codes.csv", "w+") 2 | file.write("000660" + "\n") 3 | file.write("025980" + "\n") 4 | file.write("005930" + "\n") 5 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week3/03_print_file.py: -------------------------------------------------------------------------------- 1 | from libs.NaverFinance.crawl import crawl 2 | from libs.NaverFinance.parser import getCompanyInfo 3 | 4 | file = open("./codes.csv") 5 | lines = file.readlines() 6 | print(lines) 7 | for line in lines: 8 | code = line.replace("\n", "") 9 | print(getCompanyInfo(crawl(code))) 10 | 11 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn3/week3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn3/week3/appendix_pandas.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_excel("./jongmok.xlsx") 4 | print(df['code']) 5 | 6 | for item in df['code']: 7 | print(item.replace("'", "")) 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week3/jongmok.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn3/week3/jongmok.xlsx -------------------------------------------------------------------------------- /lecture/lecture_gn3/week4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn3/week4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn3/week4/stage.py: -------------------------------------------------------------------------------- 1 | from libs.naver.crawler import crawl 2 | from libs.naver.parser import parse 3 | 4 | # 무대 -> 배우, 장치, 조명 5 | # crawl호출 parse호출 결과 출력 6 | code = "000660" 7 | string = crawl(code) 8 | companyInfo = parse(string) 9 | print(companyInfo) -------------------------------------------------------------------------------- /lecture/lecture_gn3/week6/01_save_to_excel.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from libs.excelSaver import save 3 | 4 | pd.set_option('display.max_colwidth', -1) 5 | df = pd.read_json("./195297.json") 6 | dicts = [ 7 | {'df':df, 'sheetName': 'total'}, 8 | ] 9 | save(dicts, "초콜렛.xlsx") 10 | 11 | print(df.count()) 12 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week6/02_sort.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from libs.excelSaver import save 3 | 4 | pd.set_option('display.max_colwidth', -1) 5 | 6 | df = pd.read_json("./195297.json") 7 | print(df.count()) 8 | 9 | dfSorted = df.sort_values(['price'], ascending=[0]) 10 | dfTop10 = dfSorted.head(10) 11 | 12 | save(dfTop10, 'priceTop10.xlsx') 13 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn3/week6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn3/week6/stage.py: -------------------------------------------------------------------------------- 1 | from libs.coupang.crawler import crawl 2 | from libs.coupang.parser import parse 3 | 4 | def getCategory(categoryNo): 5 | result = [] 6 | for pageNo in range(1, 17 + 1): 7 | pageString = crawl(categoryNo, pageNo) 8 | products = parse(pageString) 9 | result = result + products 10 | return result 11 | 12 | result = getCategory("195297") # 1020 13 | for item in result: 14 | print(item) 15 | print(len(result)) 16 | 17 | 18 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/01_list_append.py: -------------------------------------------------------------------------------- 1 | #
  • 가 5개 들어있는 list를 만들어보세요. 2 | list = [] 3 | list.append("
  • ") 4 | list.append("
  • ") 5 | list.append("
  • ") 6 | list.append("
  • ") 7 | list.append("
  • ") 8 | 9 | print(list) 10 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/02_list.py: -------------------------------------------------------------------------------- 1 | #
  • 가 5개 들어있는 list를 만들어보세요. 2 | # .append() 쓰지 않고 3 | list = ["
  • ","
  • ","
  • ","
  • ","
  • "] 4 | print(len(list)) 5 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/02_sort.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from libs.excelSaver import save 3 | 4 | pd.set_option('display.max_colwidth', -1) 5 | 6 | df = pd.read_json("./195297.json") 7 | print(df.count()) 8 | 9 | dfSorted = df.sort_values(['price'], ascending=[0]) 10 | dfTop10 = dfSorted.head(10) 11 | 12 | save(dfTop10, 'priceTop10.xlsx') 13 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/03_print_list.py: -------------------------------------------------------------------------------- 1 | #
  • 가 5개 들어있는 list를 만들어보세요. 2 | # .append() 쓰지 않고 3 | list = ["
  • ","
  • ","
  • ","
  • ","
  • "] 4 | print(list) 5 | 6 | 7 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/04_print_list_for.py: -------------------------------------------------------------------------------- 1 | #
  • 가 5개 들어있는 list를 만들어보세요. 2 | # .append() 쓰지 않고 3 | list = ["
  • ","
  • ","
  • ","
  • ","
  • "] 4 | 5 | for li in list: 6 | print(li) 7 | 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn3/week7/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/analyze.py: -------------------------------------------------------------------------------- 1 | #android6265 pw:1q2w3e4r 2 | import pandas as pd 3 | 4 | #dataframe 5 | df = pd.read_json("./195297.json") 6 | print(df.count()) 7 | 8 | print(df['price']) 9 | writer = pd.ExcelWriter('output.xlsx') 10 | df.to_excel(writer,'Sheet1') 11 | writer.save() -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/stage.py: -------------------------------------------------------------------------------- 1 | from libs.naver_shopping.crawler import crawl 2 | from libs.naver_shopping.parser import parse 3 | from libs.jsonFileSaver import save 4 | 5 | results = [] 6 | keywords = ["숨셔바요", "탈취제", "애완동물 냄세"] 7 | 8 | for keyword in keywords: 9 | pageString = crawl(keyword) 10 | products = parse(pageString) 11 | results = results + products 12 | print(len(results)) 13 | 14 | save(results, "./products.json") 15 | # file = open(fileName, "w+") 16 | # file.write(json.dumps(content)) 17 | -------------------------------------------------------------------------------- /lecture/lecture_gn3/week7/stage_naver_api.py: -------------------------------------------------------------------------------- 1 | from libs.naver_api.crawler import crawl 2 | 3 | apiResult = crawl("강남역 평양 냉면") 4 | 5 | items = apiResult['items'] 6 | 7 | print(len(items)) 8 | print(items[0]) -------------------------------------------------------------------------------- /lecture/lecture_gn4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn4/hello.py: -------------------------------------------------------------------------------- 1 | from libs.crawler import crawl 2 | from bs4 import BeautifulSoup 3 | 4 | url = "https://search.shopping.naver.com/search/all.nhn?query=%EC%88%A8%EC%85%94%EB%B0%94%EC%9A%94&cat_id=&frm=NVSHATC" 5 | string = crawl(url) 6 | # print(string) 7 | 8 | bsObj = BeautifulSoup(string, "html.parser") 9 | 10 | print(bsObj) 11 | 12 | goodsList = bsObj.find("ul", {"class":"goods_list"}) 13 | print(goodsList) -------------------------------------------------------------------------------- /lecture/lecture_gn4/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn4/readme.md -------------------------------------------------------------------------------- /lecture/lecture_gn4/week3/01_stage.py: -------------------------------------------------------------------------------- 1 | # 네이버 금융 페이지 불러오기 2 | from libs.naver_finance.crawler import crawl 3 | from libs.naver_finance.parser import parse 4 | 5 | # sk하이닉스, 신한지주 6 | print(parse(crawl("000660"))) 7 | print(parse(crawl("055550"))) 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn4/week3/02_naver_shopping_stage.py: -------------------------------------------------------------------------------- 1 | # 클로러와 2 | # 파서 불러오기 3 | from libs.naver_shopping.crawler import crawl 4 | from libs.naver_shopping.parser import parse 5 | 6 | pageString = crawl() 7 | products = parse(pageString) 8 | print(products) 9 | print(len(products)) -------------------------------------------------------------------------------- /lecture/lecture_gn4/week3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn4/week3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn4/week4/01_list.py: -------------------------------------------------------------------------------- 1 | list = [] 2 | 3 | list2 = [1] 4 | print(len(list)) 5 | print(len(list2)) 6 | print(len(list), len(list2)) 7 | 8 | list3 = [1, 2, 1] 9 | list4 = ["
  • "] 10 | list5 = ["
  • ", "
  • "] 11 | 12 | # list의 특징 추가, 삭제 가능 13 | # 순서가 있다 14 | # 연산이 가능 15 | # 중복 허용 16 | # for문을 돌릴 수 있다. 17 | # iterator 이터레이터 반복 할 수 있는 18 | for item in list5: 19 | print(item) 20 | 21 | for num in [1, 2]: 22 | print(num) 23 | 24 | -------------------------------------------------------------------------------- /lecture/lecture_gn4/week4/02_ex_list.py: -------------------------------------------------------------------------------- 1 | # 1부터 10까지 들어있는 리스트를 만들어 주세요 2 | nums = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 3 | 4 | nums2 = [] 5 | for num in range(1, 11): 6 | nums2.append(num) 7 | 8 | nums3 = list(range(1, 11)) 9 | 10 | print(len(nums), len(nums2), len(nums3)) 11 | -------------------------------------------------------------------------------- /lecture/lecture_gn4/week4/03_ex_list2.py: -------------------------------------------------------------------------------- 1 | # 1부터 20까지 들어있는 list를 만들어 주세요 2 | nums = list(range(1, 21)) 3 | # multiple이라는 이름의 함수를 만들어 주세요. 4 | # 함수의 기능은 'hello'를 출력 하는 것입니다. 5 | # num이라는 parameter를 추가 해주세요. 6 | # 받은 num을 return하는 기능을 추가 해주세요. 7 | # multiple이 받은 parameter에 2를 곱해서 리턴하게 바꿔주세요. 8 | def multiple(num): 9 | return num * 2 10 | 11 | # 위에서 만든 20개의 숫자에 각각 2를 곱한 결과를 출력해주세요. 12 | # 곱할때는 앞에서 만든 함수를 이용해주세요. 13 | for num in nums: 14 | result = multiple(num) 15 | print(result) 16 | 17 | print("nums:", len(nums)) 18 | 19 | print(list(range(1, 10000))) -------------------------------------------------------------------------------- /lecture/lecture_gn4/week4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn4/week4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn4/week5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn4/week5/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn4/week6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn4/week6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn4/week6/analyze_captain.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./captin_marble.json") 4 | print(df.count()) 5 | 6 | def save(list, filename): 7 | writer = pd.ExcelWriter(filename) 8 | 9 | for dict in list: 10 | dict['df'].to_excel(writer, dict['sheetName']) 11 | writer.save() 12 | 13 | save([{"df":df, "sheetName":"sheet1"}], "captain_marble.xlsx") 14 | 15 | -------------------------------------------------------------------------------- /lecture/lecture_gn4/week7/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn4/week7/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn4/week7/stage_coupang.py: -------------------------------------------------------------------------------- 1 | from libs.coupang.productTotalCountGetter import getKeywordTotalCount 2 | from libs.coupang.productGetter import getKeywordResults 3 | 4 | totalCount = getKeywordTotalCount("마카롱") 5 | iTotalCount = int(totalCount) 6 | print(iTotalCount) 7 | print(type(iTotalCount)) 8 | 9 | endPage = iTotalCount // 36 10 | print("endPage:", endPage) 11 | 12 | results = getKeywordResults("마카롱", 10) 13 | print(len(results)) 14 | 15 | import json 16 | file = open("./macaron.json", "w+") 17 | file.write(json.dumps(results)) -------------------------------------------------------------------------------- /lecture/lecture_gn5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn5/week1/01_hello.py: -------------------------------------------------------------------------------- 1 | print("hello") -------------------------------------------------------------------------------- /lecture/lecture_gn5/week1/02_function_def.py: -------------------------------------------------------------------------------- 1 | def printHello(): 2 | print("hello") 3 | 4 | # 호출 불러주세요 5 | printHello() 6 | 7 | # bye가 출력되는 printBye함수를 만들어 보세요 8 | # 선언(만들)하고 호출까지 해보세요 9 | def printBye(): 10 | print("bye") 11 | printBye() 12 | 13 | 14 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week1/03_parameter.py: -------------------------------------------------------------------------------- 1 | # parameter 파라메터 매개변수 2 | # 변수 변하는 수(값) ""가 없음 3 | # 상수 항상 같은 수(값) ""가 있음 4 | def printMessage(message): 5 | print(message) 6 | 7 | printMessage("happy spring") 8 | 9 | # 차이점 10 | # 1. 함수 이름 옆에 괄호에 뭐가 있다 없다. 11 | # 2. 함수를 호출 할 때 괄호안에 뭐가 있고 없고 12 | # 3. print()함수에 "hello", message 13 | 14 | # ctrl + tab 이전 파일로 가기 15 | # ctrl + tab + tab 전전으로 가기 16 | # ctrl + e 17 | # command + e -------------------------------------------------------------------------------- /lecture/lecture_gn5/week1/05_for.py: -------------------------------------------------------------------------------- 1 | def print1To20(): 2 | for num in range(1, 10 + 1): 3 | print(num) 4 | 5 | # print1To20() 6 | 7 | # print2To80 이라는 이름의 함수를 만들고 8 | # 2부터 80까지 출력을 해보세요. 2 4 6 8 10 9 | def print2To80(): 10 | for num in range(2, 80): 11 | print(num) 12 | 13 | # print2To80() 14 | 15 | def print1To79(): 16 | for num in range(0, 40): 17 | print(num * 2 + 1) 18 | 19 | print1To79() 20 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week1/06_gugudan.py: -------------------------------------------------------------------------------- 1 | # 2단을 출력하는 printGugudan함수를 만들고 2 | # 2단을 출력 해보세요. 3 | 4 | def printDan(): 5 | for dan in range(2, 10): 6 | for num in range(1, 10): 7 | print(dan, "*", num, "=", dan * num ) 8 | 9 | printDan() 10 | 11 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/week1/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn5/week2/02_gugudan.py: -------------------------------------------------------------------------------- 1 | # 2 * 1 = 2 를 출력하는 gugudan이라는 이름의 함수를 만들고 2 | # 2 * 1 = 2 를 콘솔에 출력 해보세요. 3 | def multiple(val1, val2): 4 | return val1 * val2 5 | 6 | def gugudan(dan): 7 | # 반복문 이용해서 9번 출력 해보세요 8 | for num in range(1, 10): 9 | print("{} * {} = {}".format(dan, num, dan * num)) 10 | # 2 * 2 = 4 11 | # 2 * 3 = 6 12 | 13 | for dan in range(2, 10): 14 | gugudan(dan) 15 | # 2단, 3단 동시에 16 | 17 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/week2/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn5/week3/02_dict.py: -------------------------------------------------------------------------------- 1 | # dictionary 딕셔너리 dict 사전 2 | # price : 가격 물건이나 서비스의 가격, 대가 이런 의미로도 쓰인다 3 | # {key:value} 4 | 5 | # 특정 종목(한국전력)을 dict로 표현 해보세요 6 | # 이름, 가격, 코드, 시가총액, 시가, 고가, 저가 7 | company = {"name":"한국전력", "price":"29200", "code":"112345", 8 | "total":"1800000000", "open":"29900", "high":"30150", 9 | "low":"29100"} 10 | print(company) 11 | print(company['name'], company['code']) -------------------------------------------------------------------------------- /lecture/lecture_gn5/week3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/week3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn5/week4/03_analyze.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from libs.singleExcelSaver import save 3 | 4 | df = pd.read_json("./candy.json") 5 | print(df.count()) 6 | 7 | save(df, "./candy.xlsx") -------------------------------------------------------------------------------- /lecture/lecture_gn5/week4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/week4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn5/week4/test_make_number.py: -------------------------------------------------------------------------------- 1 | # 1부터 10까지 숫자를 만들어 보세요. 2 | # 파이썬 1부터 10까지 3 | # 구글에 검색해서 찾아보세요. 4 | 5 | 6 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week5/01_gugudan.py: -------------------------------------------------------------------------------- 1 | # 구구단 2단을 출력하는 함수를 만들고 2 | # 구구단 2단을 출력 해보세요 3 | # 단이 끝날 때마다 구분선을 넣어보세요. 4 | 5 | def gugudan(dan): 6 | for num in range(1, 9 + 1): 7 | print("{} * {} = {}".format(dan, num, dan * num)) 8 | 9 | for dan in range(2, 9 + 1): 10 | gugudan(dan) 11 | print("---------------") 12 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week5/03_save_file.py: -------------------------------------------------------------------------------- 1 | # keywordResult 2 | products = [ 3 | {"name":"바디워시1", "price":1000}, 4 | {"name":"바디워시2", "price":2000} 5 | ] 6 | 7 | import json 8 | file = open("result.json", "w+") 9 | file.write(json.dumps(products)) 10 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week5/04_analyze.py: -------------------------------------------------------------------------------- 1 | # .json을 pandas로 불러와서 개수 세기 2 | import pandas as pd 3 | 4 | df = pd.read_json("./bodywash.json") 5 | df = df[['price', 'link']] 6 | 7 | print(df.count()) 8 | 9 | dfSorted = df.sort_values(['price']) 10 | print(dfSorted.head(10)) 11 | print(dfSorted.tail(10)) 12 | print(type(dfSorted['price'][0])) 13 | 14 | 15 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week5/05_replace.py: -------------------------------------------------------------------------------- 1 | word = "hello" 2 | price = "1,000" 3 | price2 = "1,000,000" 4 | # l을 a로 바꾸고 싶습니다. 어떻게 해야 할까요? 5 | # .replace()를 검색해서 6 | # hello -> heaao로 바꿔보세요 7 | 8 | replaced = word.replace("l", "a") 9 | print(replaced) 10 | print(price.replace(",","")) 11 | print(price2.replace(",","")) 12 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week5/07_naver_api_analyze.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./nangmyn.json") 4 | 5 | print(df.count()) -------------------------------------------------------------------------------- /lecture/lecture_gn5/week5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/week5/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn5/week6/02_serialize.py: -------------------------------------------------------------------------------- 1 | for num in range(0, 10): 2 | print(1 + num * 100) 3 | -------------------------------------------------------------------------------- /lecture/lecture_gn5/week6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/week6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn5/week7/03_chart.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pylab as plt 2 | 3 | 4 | plt.plot(["a_dep", "b_dep", "c_dep", "d_dep"], [1, 2, 3, 4], label="carnation") 5 | plt.plot(["a_dep", "b_dep", "c_dep", "d_dep"], [10, 20, 30, 40], label="childrensday") 6 | plt.legend(loc='upper right') 7 | plt.show() -------------------------------------------------------------------------------- /lecture/lecture_gn5/week7/05_pie.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | 3 | plt.pie([10, 20, 30]) 4 | plt.show() -------------------------------------------------------------------------------- /lecture/lecture_gn5/week7/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn5/week7/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week1/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/01_hello.py: -------------------------------------------------------------------------------- 1 | print("bye") 2 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/02_function.py: -------------------------------------------------------------------------------- 1 | # function 펑션 2 | # def부터 함수라고 선언하는 명령어 3 | # 어디까지냐면 들여쓰기가 있는 동안 4 | def printHello(): 5 | print("hello") 6 | 7 | printHello() 8 | 9 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/03_ex_fn.py: -------------------------------------------------------------------------------- 1 | # bye를 출력하는 함수를 만들고 2 | # 콘솔에 bye가 나오게 해보세요. 3 | # 함수이름 printBye 4 | # ctrl + tab 이전 파일로 가기 5 | # ctrl + tab + tab 전전 파일로 가기 6 | def printBye(): 7 | print("bye") 8 | 9 | printBye() -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/05_var_const.py: -------------------------------------------------------------------------------- 1 | 2 | # printHello, printGreet 3 | # hello, nice to meet you 4 | 5 | def printHello(): 6 | print("hello") # 상수 ""가 있음 7 | 8 | def printGreeting(greet): 9 | print(greet) # 변수 ""가 없음 10 | 11 | printHello() 12 | printGreeting("nice to meet you") 13 | 14 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/06_for.py: -------------------------------------------------------------------------------- 1 | # 반복문 for문 반복을 해줌 나대신 2 | def printNumbers(): 3 | for num in range(0, 1000000, 2): 4 | print(num) 5 | 6 | printNumbers() 7 | # shift + f6 파일명 바꾸기 8 | 9 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/07_gugudan.py: -------------------------------------------------------------------------------- 1 | # 구구단 2단 출력하기 2 | # 2 * 1 = 2를 console에 출력 해보세요. 3 | # 2 * 9 = 18까지 출력 해보세요 4 | def printGugudan(dan): 5 | for num in range(1, 9 + 1): 6 | print(dan, "*", num, "=", dan * num) # 20 7 | print("-------------") 8 | 9 | for dan in range(2, 100000 + 1): 10 | printGugudan(dan) 11 | 12 | # 1. 여기에서 함수 pringGugudan()은 몇번줄까지일까요? 9 13 | # 2. 6번줄 14 | # 3. 15 | 16 | # type 타입 형 문자형, 숫자형 17 | #"1" => 문자 계산 못함 18 | #1 => 숫자 계산 가능 19 | 20 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/08_return.py: -------------------------------------------------------------------------------- 1 | def plus(val1, val2): 2 | return val1 + val2 3 | 4 | def minus(val1, val2): 5 | return val1 - val2 6 | 7 | def multiple(val1, val2): 8 | print(val1 * val2) 9 | 10 | def divide(val1, val2): 11 | print(val1 / val2) 12 | 13 | re1 = plus(10, 20) # 30 14 | re2 = minus(10, 20) # -10 15 | print(re1, re2) 16 | multiple(re1, re2) # 200 17 | 18 | # 30 multiple -10 = -300 -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week1/com/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week1/com/stage.py: -------------------------------------------------------------------------------- 1 | # 기능들을 조합해서 실행 하는 곳 2 | from libs.naverFinanceCrawler import getCompanyInfo 3 | 4 | print(getCompanyInfo("")) -------------------------------------------------------------------------------- /lecture/lecture_gn6/week2/01_print_gugudan.py: -------------------------------------------------------------------------------- 1 | # 2 * 1 = 2 를 console에 출력 해보세요. 2 | # printGugudan() 이라는 함수를 만들어서 출력해보세요. 3 | 4 | # 함수 선언 5 | def printGugudan(dan): 6 | for num in range(1, 9 + 1): 7 | print(dan, "*", num,"=", dan * num) 8 | print("-------------") 9 | 10 | 11 | # 함수 호출 12 | for dan in range(2, 9 + 1): 13 | printGugudan(dan) 14 | 15 | # 2단, 3단 동시에 출력하기 16 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week2/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week3/02_ex_return.py: -------------------------------------------------------------------------------- 1 | # plus, minus 함수를 만들어 보세요. 2 | def plus(val1, val2): 3 | return val1 + val2 4 | 5 | def minus(val1, val2): 6 | return val1 - val2 7 | 8 | # 560 + 890 - 720 9 | 10 | result1 = plus(560, 890) 11 | result2 = minus(result1, 720) 12 | print(result2, 560 + 890 - 720) -------------------------------------------------------------------------------- /lecture/lecture_gn6/week3/03_calculate_salary.py: -------------------------------------------------------------------------------- 1 | # 연봉 계산기를 만들어 보세요 2 | # 어떻게 계산하냐면 연차 * 0.9 3 | # 함수 이름 getSalary 4 | # parameter이름은 year 5 | 6 | def getSalary(year): 7 | return year * 9000000 8 | 9 | mySalary = getSalary(7) 10 | print(mySalary) 11 | 12 | def getVat(amount): 13 | return amount * 0.1 14 | 15 | myVat = getVat(mySalary) 16 | print("myVat:", myVat) 17 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week3/05_list.py: -------------------------------------------------------------------------------- 1 | # list = [] 대괄호 2 | number = 1 3 | print(number) 4 | 5 | numbers = [1, 2, 3, 4, 6, 7, 9, 10] # 한개의 변수에 여러개 값을 넣기 위해 6 | print(numbers) 7 | 8 | code = "000660" 9 | # print(code) 10 | 11 | # codes를 만들어 보세요. 000660, 055550 12 | for number in numbers: 13 | print(number) 14 | 15 | # list는 반입니다. 학생이 여러명 들어갈 수 있습니다. 16 | # for in을 이용해서 반에서 한명씩 꺼낼 수 있습니다. 17 | codes = ["000660", "055550"] 18 | for code in codes: 19 | print(code) 20 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week3/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week4/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week5/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week5/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week6/__init__.py -------------------------------------------------------------------------------- /lecture/lecture_gn6/week7/02_file.py: -------------------------------------------------------------------------------- 1 | # file로 저장하기 2 | file = open("./hello.txt", "w+") 3 | file.write("hello") 4 | 5 | 6 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week7/03_save_list_dictionary.py: -------------------------------------------------------------------------------- 1 | import json 2 | # json 3 | result = [ 4 | {"name":"coffee1", "price":10000}, 5 | {"name":"coffee2", "price":20000} 6 | ] 7 | 8 | file = open("./jsonexam.json", "w+") 9 | file.write(json.dumps(result)) 10 | 11 | # naver shopping 1 ~ 10 -> [{}, {}] 12 | # file로 저장 해보세요. coffee.json 13 | 14 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week7/05_list.py: -------------------------------------------------------------------------------- 1 | list = [ 2 | ['name', 'price', 'link'], 3 | ["hello", 1000, "www.aaa.bbb"], 4 | ["hello", 1000, "www.aaa.bbb"] 5 | ] 6 | 7 | print(list) 8 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week7/06_pd_filter.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | pd.set_option('display.max_colwidth', -1) 3 | pd.set_option('display.max_columns', 100) 4 | 5 | # df = 표 6 | df = pd.read_json("./coffee.json") 7 | dfSorted = df.sort_values(['price']) 8 | dfFiltered = dfSorted[(dfSorted['price'] >= 10000) & (dfSorted['price'] < 20000)] 9 | 10 | print(dfFiltered[['price', 'link']]) 11 | 12 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week7/08_save_to_excel.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | pd.set_option('display.max_colwidth', -1) 3 | pd.set_option('display.max_columns', 100) 4 | 5 | # df = 표 6 | df = pd.read_json("./coffee.json") 7 | def save(df, filename): 8 | writer = pd.ExcelWriter(filename) 9 | df.to_excel(writer, 'Sheet1', index=False) 10 | writer.save() 11 | 12 | dfSorted = df.sort_values(['price'], ascending=False) 13 | 14 | save(dfSorted, "./coffee.xlsx") 15 | -------------------------------------------------------------------------------- /lecture/lecture_gn6/week7/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/lecture_gn6/week7/__init__.py -------------------------------------------------------------------------------- /lecture/pr_pandas/01_open_with_pd.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./naverKeywordResult.json") 4 | print(df.count()) 5 | 6 | # 상위 10개 블로거 이름 7 | dfTop5 = df.head(5) 8 | print(dfTop5['bloggername']) 9 | -------------------------------------------------------------------------------- /lecture/pr_pandas/02_pandas_filter.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | pd.set_option('display.max_colwidth', -1) 4 | 5 | df = pd.read_json("./naverKeywordResult.json") 6 | print(df.count()) 7 | 8 | bloggerName = "명랑 뮬란의 오늘 뭐먹지?" 9 | dfFiltered = df[df['bloggername'] == "명랑 뮬란의 오늘 뭐먹지?"] 10 | print(dfFiltered.count()) 11 | 12 | print(dfFiltered['bloggername']) 13 | print(dfFiltered['link'][1].replace("amp;", "")) 14 | -------------------------------------------------------------------------------- /lecture/pr_pandas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/pr_pandas/__init__.py -------------------------------------------------------------------------------- /lecture/practice/01_print_hello.py: -------------------------------------------------------------------------------- 1 | print({"name":"product1", "price":100}) 2 | -------------------------------------------------------------------------------- /lecture/practice/02_variable.py: -------------------------------------------------------------------------------- 1 | result = {"name":"product2", "price":1000} 2 | print(result) -------------------------------------------------------------------------------- /lecture/practice/03_to_excel_csv.py: -------------------------------------------------------------------------------- 1 | result = {"name":"product1", "price":100} 2 | print(result["name"]) 3 | print(result["price"]) 4 | print(result["name"], result["price"]) 5 | print(result["name"] + "," + str(result["price"])) 6 | -------------------------------------------------------------------------------- /lecture/practice/04_two_items.py: -------------------------------------------------------------------------------- 1 | list = [ 2 | {"name":"product1", "price":100}, 3 | {"name":"product2", "price":200}, 4 | {"name":"product3", "price":300} 5 | ] 6 | 7 | print(list) 8 | -------------------------------------------------------------------------------- /lecture/practice/05_for.py: -------------------------------------------------------------------------------- 1 | # 반복문 for문 2 | list = [ 3 | {"name":"producet1", "price":100}, 4 | {"name":"producet2", "price":200}, 5 | {"name":"producet3", "price":300} 6 | ] 7 | 8 | # list -> [] 9 | # item -> {} 10 | # [ {}, {}, ...] 11 | 12 | # 첫번째 item list에서 꺼내기 13 | print(list[1]) 14 | print(list[2]) 15 | print(list[0]) 16 | -------------------------------------------------------------------------------- /lecture/practice/06_for_2.py: -------------------------------------------------------------------------------- 1 | list = [ 2 | {"name":"product1", "price":100}, 3 | {"name":"product2", "price":200}, 4 | {"name":"product3", "price":300}, 5 | {"name":"product4", "price":400} 6 | ] 7 | 8 | for item in list: 9 | print(item['price']) 10 | 11 | 12 | -------------------------------------------------------------------------------- /lecture/practice/08_function.py: -------------------------------------------------------------------------------- 1 | def plus(val1, val2): 2 | return val1 + val2 3 | 4 | def multiple(val1, val2): 5 | return val1 * val2 6 | 7 | result = plus(10, 20) 8 | result1 = multiple(result, 40) 9 | print(result1) 10 | 11 | # (50 + 20) * 40 12 | 13 | -------------------------------------------------------------------------------- /lecture/practice/10_save_to_csv.py: -------------------------------------------------------------------------------- 1 | file = open("./hello.csv", "w+") 2 | file.write("hello" + "\n") 3 | file.write("bye" + "\n") 4 | -------------------------------------------------------------------------------- /lecture/practice/11_stage.py: -------------------------------------------------------------------------------- 1 | from libs.naver_api.naver_api_caller import get1000Result 2 | import json 3 | 4 | list = [] 5 | result = get1000Result("강남역 맛집") 6 | result2 = get1000Result("강남역 찻집") 7 | list = list + result + result2 8 | 9 | file = open("./gangnam.json", "w+") 10 | file.write(json.dumps(list)) 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lecture/practice/12_analyze_pd.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./gangnam.json") 4 | print(bloggernames) -------------------------------------------------------------------------------- /lecture/practice/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/practice/__init__.py -------------------------------------------------------------------------------- /lecture/stage_netd.py: -------------------------------------------------------------------------------- 1 | from libs.netd.netd_crawler import getLinks 2 | 3 | links = [] 4 | for num in range(0, 2): 5 | links = links + getLinks(num * 10 + 1) 6 | print(len(links)) -------------------------------------------------------------------------------- /lecture/taling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/taling/__init__.py -------------------------------------------------------------------------------- /lecture/taling/week1_function/01_function.py: -------------------------------------------------------------------------------- 1 | # function 펑션 함수 2 | def print_hello(): 3 | print("hello") 4 | 5 | print_hello() -------------------------------------------------------------------------------- /lecture/taling/week1_function/02_pikachu.py: -------------------------------------------------------------------------------- 1 | def pikachu(): 2 | print("전기공격") 3 | 4 | pikachu() -------------------------------------------------------------------------------- /lecture/taling/week1_function/03_kkobugi.py: -------------------------------------------------------------------------------- 1 | def 꼬부기(): 2 | print("물대포") 3 | 4 | 꼬부기() -------------------------------------------------------------------------------- /lecture/taling/week1_function/05_parameter.py: -------------------------------------------------------------------------------- 1 | # 파라메터 parameter param p 매개변수 2 | def print_message(p_message): 3 | print(p_message) 4 | 5 | print_message("안녕히가세요") 6 | -------------------------------------------------------------------------------- /lecture/taling/week1_function/06_if.py: -------------------------------------------------------------------------------- 1 | # 어른이면 안녕하세요, 그밖에는 안녕을 출력하는 프로그램 2 | def exam_if(is_adult): 3 | if(is_adult == "어른"): 4 | print("안녕하세요") 5 | else: 6 | print("안녕") 7 | 8 | 9 | exam_if("어른") 10 | exam_if("someone") 11 | -------------------------------------------------------------------------------- /lecture/taling/week1_function/07_eles_if.py: -------------------------------------------------------------------------------- 1 | # is_adult가 어른이면 안녕하세요 2 | # is_adult가 애들이면 안녕 3 | # is_adult가 외국인이면 hello 4 | # 그밖에는 모르겠습니다 5 | 6 | def exam_elif(is_adult): 7 | if(is_adult == "어른"): 8 | print("안녕하세요") 9 | elif(is_adult == "애들"): 10 | print("안녕") 11 | elif (is_adult == "외국인"): 12 | print("hello") 13 | else: 14 | print("모르겠습니다.") 15 | 16 | exam_elif("어른") 17 | exam_elif("애들") 18 | exam_elif("외국인") 19 | exam_elif("외계인") -------------------------------------------------------------------------------- /lecture/taling/week1_function/08_pikachu_level2.py: -------------------------------------------------------------------------------- 1 | # 피카츄 1전기공격, 2피뢰침 2 | # 그밖에 번호를 넣으면 레벨업을 더 하세요 가 나오게 3 | 4 | # ctrl + tab 5 | def pikachu_level2(p_skill_num): 6 | print("전기공격") 7 | 8 | 9 | pikachu_level2(1) # 전기공격 10 | pikachu_level2(2) # 피뢰침 11 | pikachu_level2(3) # 레벨업을 더 하세요 -------------------------------------------------------------------------------- /lecture/taling/week1_function/09_get_score.py: -------------------------------------------------------------------------------- 1 | # 학생들 점수 2 | # score에 있는 점수의 등급을 매기는 if문을 짜보세요 3 | # >=90 A, >=80 B, >=70 C, < 70 F 4 | 5 | def get_grade(p_score): 6 | if(p_score >= 90): 7 | print("A") 8 | elif(p_score >= 80): 9 | print("B") 10 | else: 11 | print("F") 12 | 13 | get_grade(95) 14 | get_grade(90) 15 | get_grade(85) 16 | get_grade(80) 17 | get_grade(70) 18 | get_grade(60) -------------------------------------------------------------------------------- /lecture/taling/week1_function/11_get_grade_return.py: -------------------------------------------------------------------------------- 1 | # 학생들 점수 2 | # 점수(score)를 입력하면 등급(grade)이 아래와 같이 나오는 3 | # function을 짜보세요 4 | # >=90 A, >=80 B, >=70 C, < 70 F 5 | # alt + j 6 | def get_grade(p_score): 7 | if (p_score >= 90): 8 | return "A" 9 | elif (p_score >= 80): 10 | return "B" 11 | else: 12 | return "F" 13 | 14 | grade = get_grade(90) 15 | print(grade) 16 | print(get_grade(90)) 17 | print(get_grade(80)) 18 | -------------------------------------------------------------------------------- /lecture/taling/week1_function/12_variable.py: -------------------------------------------------------------------------------- 1 | # 변수 variable 2 | # 변하는 수(값), 상수 항상 같은 수(값) 3 | result = 10 4 | print(result) 5 | result = 20 6 | print(result) 7 | 8 | # 상수 "" 따옴표 9 | 10 | grade = "A" 11 | print(grade) 12 | grade = "B" 13 | print(grade) 14 | -------------------------------------------------------------------------------- /lecture/taling/week1_function/13_pirie_return.py: -------------------------------------------------------------------------------- 1 | # 1파이어 볼, 2 파이어 월 그밖에 레벨업을 하세요 2 | # 를 return하는 파이리를 만들어보세요 3 | def pirie(p_skill_number): 4 | if(p_skill_number == 1): 5 | return "파이어볼" 6 | elif(p_skill_number == 2): 7 | return "파이어월" 8 | else: 9 | return "레벨업을 하세요" 10 | 11 | print(pirie(1)) 12 | -------------------------------------------------------------------------------- /lecture/taling/week1_function/14_recursive.py: -------------------------------------------------------------------------------- 1 | def multiple(n, a): 2 | if(n == 1): return a 3 | return multiple(n-1, a) + a 4 | 5 | result = multiple(4, 5) 6 | print(result) 7 | -------------------------------------------------------------------------------- /lecture/taling/week1_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/lecture/taling/week1_function/__init__.py -------------------------------------------------------------------------------- /lecture/week4/04_replace.py: -------------------------------------------------------------------------------- 1 | text = "강남 맛집 11번출구" 2 | print(text.replace("", "")) -------------------------------------------------------------------------------- /libs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/__init__.py -------------------------------------------------------------------------------- /libs/agrion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/agrion/__init__.py -------------------------------------------------------------------------------- /libs/bible/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/bible/__init__.py -------------------------------------------------------------------------------- /libs/bin/chromedriver: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/bin/chromedriver -------------------------------------------------------------------------------- /libs/coupang/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/coupang/__init__.py -------------------------------------------------------------------------------- /libs/coupang/coupangCrawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | def getString(url): 4 | data = requests.get(url) 5 | print(data.status_code, url) 6 | return data.content -------------------------------------------------------------------------------- /libs/coupang/crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | def crawl(categoryNo, pageNo): 4 | url = "https://www.coupang.com/np/categories/{}?page={}".format(categoryNo, pageNo) 5 | data = requests.get(url) 6 | print(url, data) 7 | return data.content 8 | 9 | 10 | -------------------------------------------------------------------------------- /libs/coupang/parser.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | 3 | def getProductInfo(li): 4 | name = li.find("img")['alt'] 5 | price = li.find("strong", {"class":"price-value"}).text 6 | return {"name":name, "price":price} 7 | 8 | def parse(string): 9 | bsObj = BeautifulSoup(string, "html.parser") 10 | ul = bsObj.find("ul", {"id":"productList"}) 11 | lis = ul.findAll("li") 12 | return [getProductInfo(li) for li in lis] 13 | -------------------------------------------------------------------------------- /libs/crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | def crawl(url): 4 | session = requests.Session() 5 | print(session.cookies) 6 | result = requests.get(url) 7 | print(result.status_code, url) 8 | return result.content -------------------------------------------------------------------------------- /libs/daily_sam/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/daily_sam/__init__.py -------------------------------------------------------------------------------- /libs/dorm_libs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/dorm_libs/__init__.py -------------------------------------------------------------------------------- /libs/dorm_libs/fileSaver.py: -------------------------------------------------------------------------------- 1 | def saveHello(): 2 | file = open("./1.html", "w+", encoding="utf-8") 3 | file.write("hello") 4 | file.close() 5 | 6 | # "./1.html" 7 | def save(fileLocationName, contents): 8 | file = open(fileLocationName, "w+") 9 | file.write(contents) 10 | file.close() -------------------------------------------------------------------------------- /libs/excelSaver.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | def save(dicts, filename): 3 | writer = pd.ExcelWriter(filename) 4 | for dict in dicts: 5 | dict['df'].to_excel(writer, dict['sheetName']) 6 | writer.save() 7 | 8 | -------------------------------------------------------------------------------- /libs/jolse/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/jolse/__init__.py -------------------------------------------------------------------------------- /libs/jolse/jolse_crawl.py: -------------------------------------------------------------------------------- 1 | import requests 2 | def crawl(url, fileLocationName): 3 | # url을 받아서 4 | # request를 하고 str 5 | result = requests.get(url) 6 | # 결과를 파일로 저장 7 | print(result.content) 8 | saveToFile(result.content, fileLocationName) 9 | 10 | def saveToFile(str1, fileLocationName): 11 | # file에 저장 12 | file = open(fileLocationName, "w+") 13 | file.write(str(str1)) 14 | -------------------------------------------------------------------------------- /libs/jsonFileSaver.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | def save(content, fileName): 4 | file = open(fileName, "w+") 5 | file.write(json.dumps(content)) -------------------------------------------------------------------------------- /libs/naver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/naver/__init__.py -------------------------------------------------------------------------------- /libs/naver/apiCrawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import quote 3 | 4 | def crawl(keyword, start): 5 | # 검색했을때 검색결과 6 | encText = quote(keyword) # unicode encoding 7 | url = "https://openapi.naver.com/v1/search/blog?query=" + encText \ 8 | + "&display=100" + "&start={}".format(start) # json 결과 9 | data = requests.get(url, headers={ 10 | "X-Naver-Client-Id":"l6_H64cWBZnHLGhZ7zKO", 11 | "X-Naver-Client-Secret":"dlNVIJaLeI" 12 | }) 13 | print(data.status_code, keyword) 14 | return data.json() -------------------------------------------------------------------------------- /libs/naver/crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | def crawl(code): 4 | url = "https://finance.naver.com/item/main.nhn?code={}".format(code) 5 | data = requests.get(url) 6 | print(data.status_code) 7 | return data.content -------------------------------------------------------------------------------- /libs/naver/parser.py: -------------------------------------------------------------------------------- 1 | def parse(string): 2 | print("=>", string) 3 | return {} -------------------------------------------------------------------------------- /libs/naver_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/naver_api/__init__.py -------------------------------------------------------------------------------- /libs/naver_api/api_call.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlparse 3 | # keyword를 받아서 검색 결과를 return해줌 4 | 5 | def naverApiCall(keyword): 6 | url = "https://openapi.naver.com/v1/search/blog?query=" + keyword # json 결과 7 | result = requests.get(urlparse(url).geturl(), 8 | headers={"X-Naver-Client-Id":"bmMn9JNtSywmpiVmsqkO", 9 | "X-Naver-Client-Secret":"e2u6Dw4w0c"}) 10 | return result.json() 11 | -------------------------------------------------------------------------------- /libs/naver_finance/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/naver_finance/__init__.py -------------------------------------------------------------------------------- /libs/naver_finance/crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | def crawl(code): 3 | url = "https://finance.naver.com/item/main.nhn?code={}".format(code) 4 | data = requests.get(url) 5 | print(data) 6 | return data.content 7 | -------------------------------------------------------------------------------- /libs/naver_shopping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/naver_shopping/__init__.py -------------------------------------------------------------------------------- /libs/naver_shopping/crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | def crawl(keyword): 4 | url = "https://search.shopping.naver.com/search/all.nhn?query={}&cat_id=&frm=NVSHATC".format(keyword) 5 | data = requests.get(url) 6 | print(data) 7 | return data.content -------------------------------------------------------------------------------- /libs/naver_shopping2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/naver_shopping2/__init__.py -------------------------------------------------------------------------------- /libs/naver_shopping2/crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | def crawl(keyword): 4 | url = f'https://search.shopping.naver.com/search/all?query={keyword}&cat_id=&frm=NVSHATC' 5 | data = requests.get(url) 6 | print(data.status_code, url) 7 | return data.content -------------------------------------------------------------------------------- /libs/ncbi/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/ncbi/__init__.py -------------------------------------------------------------------------------- /libs/netd/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/libs/netd/__init__.py -------------------------------------------------------------------------------- /libs/patternMatcher.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | # python에서 정규식 쓰기(regular expression) 4 | 5 | def findMatchedTexts(text, regexp): 6 | matched = re.findall(regexp, text) 7 | return matched -------------------------------------------------------------------------------- /libs/singleExcelSaver.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | def save(df, filename): 4 | writer = pd.ExcelWriter(filename) 5 | df.to_excel(writer, "sheet1") 6 | writer.save() 7 | -------------------------------------------------------------------------------- /naver_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/naver_api/__init__.py -------------------------------------------------------------------------------- /naver_api/shopping/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/naver_api/shopping/__init__.py -------------------------------------------------------------------------------- /py_auto_gui/01_start.py: -------------------------------------------------------------------------------- 1 | import pyautogui 2 | current = pyautogui.position() 3 | print(current) 4 | 5 | #762, 872 6 | x = current[0] 7 | y = current[1] 8 | print(x) 9 | pyautogui.moveTo(771, 872, duration=1) 10 | # pyautogui.click(x=771, y=872, clicks=1, interval=1, button='left') 11 | pyautogui.typewrite('Hello world!\n', interval=1) -------------------------------------------------------------------------------- /qna/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/qna/__init__.py -------------------------------------------------------------------------------- /question/01_naver_finance_sise_market_sum/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/question/01_naver_finance_sise_market_sum/__init__.py -------------------------------------------------------------------------------- /question/02_naver_shopping_zzim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/question/02_naver_shopping_zzim/__init__.py -------------------------------------------------------------------------------- /question/03_youtube/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/question/03_youtube/__init__.py -------------------------------------------------------------------------------- /question/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/question/__init__.py -------------------------------------------------------------------------------- /sites/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/__init__.py -------------------------------------------------------------------------------- /sites/bible/02_parse_index.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | file = open("./gae.json") 4 | bible = json.loads(file.read()) 5 | print(len(bible)) 6 | 7 | for ee in bible: 8 | print(ee) 9 | -------------------------------------------------------------------------------- /sites/bible/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/bible/__init__.py -------------------------------------------------------------------------------- /sites/bible/books/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/bible/books/__init__.py -------------------------------------------------------------------------------- /sites/dart/01_crawl_dart.py: -------------------------------------------------------------------------------- 1 | 2 | from libs.crawler import crawl 3 | from bs4 import BeautifulSoup 4 | 5 | def parse(pageString): 6 | soup = BeautifulSoup(pageString, "html.parser") 7 | print(soup) 8 | 9 | url = "http://dart.fss.or.kr/dsab002/search.ax?reportName=%EB%B0%B0%EB%8B%B9&&maxResults=100&&textCrpNm=%EC%82%BC%EC%84%B1%EC%A0%84%EC%9E%90" 10 | pageString = crawl(url) 11 | infos = parse(pageString) 12 | print(infos) 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /sites/dart/02_specific.py: -------------------------------------------------------------------------------- 1 | from libs.crawler import crawl 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://dart.fss.or.kr/report/viewer.do?rcpNo=20181031800040&dcmNo=6364473&eleId=0&offset=0&length=0&dtd=HTML" 5 | 6 | pageString = crawl(url) 7 | 8 | def parse(pageString): 9 | bsObj = BeautifulSoup(pageString, "html.parser") 10 | print(bsObj) 11 | 12 | result = parse(pageString) 13 | 14 | 15 | -------------------------------------------------------------------------------- /sites/ecolife/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/ecolife/__init__.py -------------------------------------------------------------------------------- /sites/holy_bible/04_analyze.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | # df = pd.read_json("./bible.json") 4 | # print(df.count()) 5 | 6 | import json 7 | 8 | file = open("./bible.json") 9 | jsonObj = json.loads(file.read) 10 | 11 | print(jsonObj) -------------------------------------------------------------------------------- /sites/holy_bible/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/holy_bible/__init__.py -------------------------------------------------------------------------------- /sites/instagram/01_tag.py: -------------------------------------------------------------------------------- 1 | from libs.crawler import crawl 2 | from libs.browser import Browser 3 | 4 | url = "https://www.instagram.com/explore/tags/댕스타그램" 5 | 6 | 7 | browser = Browser() 8 | browser.move(url) 9 | ele_a_datetime = browser.find_one('.e02As .c-Yi7', 5) 10 | print(ele_a_datetime) 11 | -------------------------------------------------------------------------------- /sites/instagram/02_driver.py: -------------------------------------------------------------------------------- 1 | import os 2 | from selenium import webdriver 3 | 4 | dir_path = os.path.dirname(os.path.abspath(__file__)) 5 | print(dir_path) 6 | driver = webdriver.Chrome( 7 | executable_path="{}/bin/chromedriver".format(dir_path), 8 | ) 9 | 10 | url = "https://coupang.co.kr/" 11 | driver.get(url) 12 | 13 | -------------------------------------------------------------------------------- /sites/lotte/02_http.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | url = "https://display.ellotte.com/display-fo/categoryShop?dshopNo=10023" 4 | 5 | result = requests.get(url) 6 | bs_obj = BeautifulSoup(result.content) 7 | print(bs_obj) -------------------------------------------------------------------------------- /sites/lotte/03_style_landa.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | url = "http://goods.ellotte.com/goods-fo/goodsDetail/1200261484?dshopNo=10023" 4 | 5 | result = requests.get(url) 6 | print(BeautifulSoup(result.content, "html.parser")) 7 | -------------------------------------------------------------------------------- /sites/lotte/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/lotte/__init__.py -------------------------------------------------------------------------------- /sites/nalsam/01_crawl_nalsam.py: -------------------------------------------------------------------------------- 1 | from libs.daily_sam.dailySamCrawler import runDailySam 2 | 3 | runDailySam(7953) 4 | -------------------------------------------------------------------------------- /sites/nalsam/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/nalsam/__init__.py -------------------------------------------------------------------------------- /sites/naver_finance/01_call_samsung.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | def get_bs_obj(): 5 | url = "https://finance.naver.com/item/main.nhn?code=005930" 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | return bs_obj 9 | 10 | bs_obj = get_bs_obj() 11 | print(bs_obj) -------------------------------------------------------------------------------- /sites/naver_finance/02_get_price.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | def get_bs_obj(): 5 | url = "https://finance.naver.com/item/main.nhn?code=005930" 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | return bs_obj 9 | 10 | bs_obj = get_bs_obj() 11 | no_today = bs_obj.find("p", {"class":"no_today"}) 12 | blind_now = no_today.find("span", {"class":"blind"}) 13 | print(blind_now.text) -------------------------------------------------------------------------------- /sites/naver_finance/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/naver_finance/__init__.py -------------------------------------------------------------------------------- /sites/ncbi/02_separater.py: -------------------------------------------------------------------------------- 1 | import json 2 | file = open("./links.txt") 3 | 4 | lines = file.readlines() 5 | 6 | list = [] 7 | for line in lines: 8 | list.append(line.replace("\n", "")) 9 | 10 | list.sort() 11 | 12 | file2 = open("./links2.json", "w+") 13 | file2.write(json.dumps(list)) 14 | -------------------------------------------------------------------------------- /sites/ncbi/09_reduce.py: -------------------------------------------------------------------------------- 1 | from libs.json_reducer import collectToOneFile 2 | 3 | location = "/Users/kyeongrok/Desktop/" 4 | collectToOneFile(location+"result2.json", location+"json_results/") -------------------------------------------------------------------------------- /sites/ncbi/10_json_to_xml.py: -------------------------------------------------------------------------------- 1 | from json import loads 2 | from dicttoxml import dicttoxml 3 | 4 | def convertJsonToXml(fromFileName, toFileName): 5 | json_obj = loads(str(open(fromFileName).read())) 6 | print(type(json_obj)) 7 | 8 | xml = dicttoxml(json_obj) 9 | 10 | file = open(toFileName, "w+") 11 | file.write(str(xml)) 12 | file.close() 13 | 14 | location = "/Users/kyeongrok/Desktop/" 15 | convertJsonToXml(location +"result.json",location+"result_xml.json") -------------------------------------------------------------------------------- /sites/ncbi/11_sub.py: -------------------------------------------------------------------------------- 1 | from libs.ncbi import ncbi_crawler as crawler 2 | 3 | file = open("/Users/kyeongrok/Desktop/sub_pages2/" + str(int(8180)) +".html") 4 | text = file.read() 5 | res = crawler.getSubPageContent(text) 6 | print(res['authors']) 7 | -------------------------------------------------------------------------------- /sites/ncbi/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/ncbi/__init__.py -------------------------------------------------------------------------------- /sites/ncbi/analyze/03_count_year.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | location = "/Users/kyeongrok/Desktop/" 4 | df = pd.read_json(location+"result3.json") 5 | 6 | institutions = df['institutions'] 7 | 8 | for institution in institutions: 9 | print(len(institution)) 10 | 11 | df1000 = df.head(1000) 12 | print(df1000.count()) 13 | 14 | df1000.to_json(location + "result1000.json") 15 | -------------------------------------------------------------------------------- /sites/ncbi/analyze/04_count_keywords.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("/Users/kyeongrok/Desktop/result3.json") 4 | print(df.count()) 5 | 6 | print(df['authors']) 7 | 8 | count = 0 9 | for author in df['year']: 10 | print(author) 11 | print(count) -------------------------------------------------------------------------------- /sites/ncbi/analyze/04_to_excel.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | location = "/Users/kyeongrok/Desktop/" 4 | df = pd.read_json(location + "result_year_authors_nation.json") 5 | 6 | writer = pd.ExcelWriter(location +'output.xlsx') 7 | df.to_excel(writer,'Sheet1') 8 | writer.save() -------------------------------------------------------------------------------- /sites/ncbi/analyze/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/ncbi/analyze/__init__.py -------------------------------------------------------------------------------- /sites/psyworld/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/psyworld/__init__.py -------------------------------------------------------------------------------- /sites/public_data/01_public.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | path_code = "https://raw.githubusercontent.com/WooilJeong/PublicDataReader/f14e4de3410cc0f798a83ee5934070d651cbd67b/docs/%EB%B2%95%EC%A0%95%EB%8F%99%EC%BD%94%EB%93%9C%20%EC%A0%84%EC%B2%B4%EC%9E%90%EB%A3%8C.txt" 4 | code = pd.read_csv(path_code, encoding='cp949', sep='\t') 5 | print(code) 6 | -------------------------------------------------------------------------------- /sites/public_data/02_pd_read_csv.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | path_code = "https://raw.githubusercontent.com/WooilJeong/PublicDataReader/f14e4de3410cc0f798a83ee5934070d651cbd67b/docs/%EB%B2%95%EC%A0%95%EB%8F%99%EC%BD%94%EB%93%9C%20%EC%A0%84%EC%B2%B4%EC%9E%90%EB%A3%8C.txt" 4 | code = pd.read_csv(path_code, encoding='cp949', sep='\t') 5 | print(code) 6 | -------------------------------------------------------------------------------- /sites/public_data/03_requests.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | path_code = "https://raw.githubusercontent.com/WooilJeong/PublicDataReader/f14e4de3410cc0f798a83ee5934070d651cbd67b/docs/%EB%B2%95%EC%A0%95%EB%8F%99%EC%BD%94%EB%93%9C%20%EC%A0%84%EC%B2%B4%EC%9E%90%EB%A3%8C.txt" 4 | data = requests.get(path_code) 5 | print(data.content) -------------------------------------------------------------------------------- /sites/public_data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/public_data/__init__.py -------------------------------------------------------------------------------- /sites/seoul_data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/seoul_data/__init__.py -------------------------------------------------------------------------------- /sites/unipass/01_unipass.py: -------------------------------------------------------------------------------- 1 | from sites.unipass import parse 2 | from libs.crawler import crawl 3 | 4 | 5 | url = "https://unipass.customs.go.kr/csp/index.do" 6 | 7 | result = crawl(url) 8 | parse(result) 9 | -------------------------------------------------------------------------------- /sites/unipass/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/sites/unipass/__init__.py -------------------------------------------------------------------------------- /sites/unipass/parser.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | 3 | def parse(string): 4 | bsObj = BeautifulSoup(string, "html.parser") 5 | section = bsObj.find("section", {"class":"M18_potlete_cont1"}) 6 | print(section) 7 | -------------------------------------------------------------------------------- /sports/01_hello.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | df = pd.read_json("./bet365_prematch_odds.json") 4 | 5 | print(df) -------------------------------------------------------------------------------- /sports/02_import_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | file = open("./bet365_prematch_odds.json") 4 | jsonObj = json.loads(file.read()) 5 | 6 | 7 | result = jsonObj['results'][0] 8 | print(result) 9 | 10 | 11 | -------------------------------------------------------------------------------- /test/01_largest.py: -------------------------------------------------------------------------------- 1 | A = [3, 2, -2, 5, -3] 2 | 3 | def solution(A): 4 | positive = [] 5 | negative = [] 6 | result = [] 7 | 8 | for n in A: 9 | if n > 0: 10 | positive.append(n) 11 | elif n < 0: 12 | negative.append(n) 13 | 14 | for ne in negative: 15 | if abs(ne) in positive: 16 | result.append(ne) 17 | 18 | if len(result) == 0: 19 | return 0 20 | else: 21 | result.sort() 22 | return abs(result[0]) 23 | 24 | -------------------------------------------------------------------------------- /test/01_naver.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = 'https://finance.naver.com/item/board.nhn?code=096240' 5 | headers = { 6 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) 26.109 Safari/537.36" 7 | } 8 | req = requests.get(url, headers) 9 | 10 | contents = req.content 11 | soup = BeautifulSoup(contents,'html.parser') 12 | print(soup) -------------------------------------------------------------------------------- /test/04_turtle_key_press.py: -------------------------------------------------------------------------------- 1 | import turtle 2 | tt=turtle.Turtle() 3 | 4 | def up(): 5 | tt.setheading(90) 6 | tt.forward(100) 7 | 8 | def left(): 9 | tt.setheading(180) 10 | tt.forward(100) 11 | 12 | def right(): 13 | tt.setheading(0) 14 | tt.forward(100) 15 | 16 | def down(): 17 | tt.setheading(270) 18 | tt.forward(100) 19 | 20 | 21 | turtle.listen() 22 | turtle.onkey(up,"Up") 23 | turtle.onkey(down,"Down") 24 | turtle.onkey(left,"Left") 25 | turtle.onkey(right,"Right") 26 | turtle.mainloop() -------------------------------------------------------------------------------- /test/08_ddd.py: -------------------------------------------------------------------------------- 1 | room_width = 365 2 | room_height = 326 3 | 4 | smaller = 20 / 365 5 | print(room_height * smaller) 6 | print(73 * smaller) 7 | print(155, 155 * smaller, 88, 88 * smaller) 8 | print(145 * smaller, 200 * smaller) 9 | print(160, 160 * smaller, 60, 60 * smaller) 10 | print(80 * smaller, 28 * smaller) 11 | print(175 * smaller) -------------------------------------------------------------------------------- /test/09_agrion_parser_test.py: -------------------------------------------------------------------------------- 1 | from libs.agrion.parser import parse 2 | 3 | file = open('hello.html') 4 | file_content = file.read() 5 | res = parse(file_content) 6 | # 7 | print(res) -------------------------------------------------------------------------------- /test/10_http_request_with_header.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | url = 'http://localhost:8080/api/v1/order/distribute?id=1111&name=krk' 4 | res = requests.get(url, headers={ 5 | 'X-USER-ID':'oceanfog' 6 | }) 7 | 8 | print(res.content) 9 | 10 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/test/__init__.py -------------------------------------------------------------------------------- /test/agrion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyeongrok/python_crawler/5a5da8af7bb080f752a9a066741ac8adab136a3a/test/agrion/__init__.py -------------------------------------------------------------------------------- /test_data/01_ppompu.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import requests 3 | from bs4 import BeautifulSoup 4 | import html5lib 5 | 6 | url = 'http://m.ppomppu.co.kr/new/bbs_view.php?id=tour&no=89857&page=1' 7 | html = urllib.request.urlopen(url) 8 | result = requests.get(url) 9 | 10 | bs_obj = BeautifulSoup(result.content, "html5lib") 11 | 12 | info = bs_obj.find("div", {"class":"info"}) 13 | print(info) 14 | 15 | -------------------------------------------------------------------------------- /test_data/01_shoppingmal_jolse/01_jolse_mist.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | print(bs_obj) -------------------------------------------------------------------------------- /test_data/01_shoppingmal_jolse/02_box.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | box = bs_obj.find("div", {"class":"box"}) # 가장 처음 나오는걸 return 10 | print(box) -------------------------------------------------------------------------------- /test_data/01_shoppingmal_jolse/03_boxes_findall.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | boxes = bs_obj.findAll("div", {"class":"box"}) # 전체다 []에 넣어줌 10 | 11 | for box in boxes: 12 | print(box) 13 | 14 | -------------------------------------------------------------------------------- /test_data/01_shoppingmal_jolse/04_box_name.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | boxes = bs_obj.findAll("div", {"class":"box"}) # 전체다 []에 넣어줌 10 | 11 | for box in boxes[0:1]: 12 | p_name = box.find("p", {"class":"name"}) 13 | a_tag = p_name.find("a") 14 | print(a_tag) 15 | 16 | -------------------------------------------------------------------------------- /test_data/01_shoppingmal_jolse/05_box_name_a.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | boxes = bs_obj.findAll("div", {"class":"box"}) # 전체다 []에 넣어줌 10 | 11 | for box in boxes[0:1]: 12 | p_name = box.find("p", {"class":"name"}) 13 | a_tag = p_name.find("a") 14 | print(a_tag) 15 | 16 | -------------------------------------------------------------------------------- /test_data/01_shoppingmal_jolse/06_box_name_a_title.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | boxes = bs_obj.findAll("div", {"class":"box"}) # 전체다 []에 넣어줌 10 | 11 | for box in boxes[0:1]: 12 | p_name = box.find("p", {"class":"name"}) 13 | a_tag = p_name.find("a") 14 | product_name = a_tag['title'] 15 | print(product_name) 16 | 17 | -------------------------------------------------------------------------------- /test_data/01_shoppingmal_jolse/07_box_price_ul.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | url = "http://jolse.com/category/tonermist/43/" 5 | result = requests.get(url) 6 | 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | 9 | boxes = bs_obj.findAll("div", {"class":"box"}) # 전체다 []에 넣어줌 10 | 11 | for box in boxes[0:1]: 12 | p_name = box.find("p", {"class":"name"}) 13 | a_tag = p_name.find("a") 14 | product_name = a_tag['title'] 15 | ul = box.find("ul") 16 | print(ul) 17 | 18 | -------------------------------------------------------------------------------- /test_data/01_slice.py: -------------------------------------------------------------------------------- 1 | splitted = "hello world bye".split(" ") 2 | 3 | print(splitted[1]) 4 | 5 | targetString = "aa=19" 6 | start = len(targetString) - 3 7 | end = len(targetString) 8 | result = targetString[start:end] 9 | 10 | print(result.replace("=", "")) 11 | -------------------------------------------------------------------------------- /test_data/02_hello.py: -------------------------------------------------------------------------------- 1 | users = [ 2 | {"name": "kyeongrok", "age": 33, "salary": 80000000}, 3 | {"name": "yeonghwan", "age": 30, "salary": 50000000} 4 | ] 5 | 6 | for user in users: 7 | print(user["age"], user["salary"]) 8 | 9 | # 10 | def plus(val1, val2): 11 | return val1 + val2 12 | 13 | def multiple(val1, val2): 14 | return val1 * val2 15 | 16 | result = multiple(plus(10, 20), plus(20, 30)) 17 | print(result) -------------------------------------------------------------------------------- /test_data/06_naver_finance/01_naver_exchange.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | def get_bs_obj(): 5 | url = "https://finance.naver.com/item/main.nhn?code=035420" 6 | result = requests.get(url) 7 | bs_obj = BeautifulSoup(result.content, "html.parser") 8 | return bs_obj 9 | 10 | bs_obj = get_bs_obj() 11 | today = bs_obj.find("div",{"class":"today"}) 12 | 13 | 14 | -------------------------------------------------------------------------------- /test_data/chapter12_selenium/01_login_bank.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | 3 | driver = webdriver.Chrome("../../chrome/mac/chromedriver") 4 | 5 | url = "" 6 | driver.get("http://www.kofiabond.or.kr/websquare/websquare.html?w2xPath=/xml/Com/Common_TabMnuDsp.xml&divisionId=MBIS01070010000000&serviceId=BIS0100100280&topMenuIndex=6&w2xHome=/xml/&w2xDocumentRoot=") 7 | 8 | table = driver.title 9 | 10 | print(table) 11 | print(driver.find_element_by_tag_name("div").text) 12 | -------------------------------------------------------------------------------- /test_data/chapter12_selenium/01_login_naver.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | 3 | driver = webdriver.Chrome("../../chrome/mac/chromedriver") 4 | 5 | driver.get("https://nid.naver.com/nidlogin.login") 6 | 7 | driver.implicitly_wait(3) 8 | login_id = driver.find_element_by_id("id") 9 | login_passwd = driver.find_element_by_id("pw") 10 | login_id.send_keys("oceanfog") 11 | login_passwd.send_keys("") 12 | 13 | login_button = driver.find_element_by_class_name('w2image') 14 | login_button.click() 15 | -------------------------------------------------------------------------------- /test_data/chapter12_selenium/05_hello.py: -------------------------------------------------------------------------------- 1 | data = ["상품1", "상품2"]# array(배열) 2 | print(data) 3 | print(data[2]) 4 | 5 | -------------------------------------------------------------------------------- /test_data/chapter14_file/01_file_hello.py: -------------------------------------------------------------------------------- 1 | with open("./hello.txt", mode="w+") as f: 2 | f.write("hello \n") -------------------------------------------------------------------------------- /test_data/chapter14_file/02_open_file.py: -------------------------------------------------------------------------------- 1 | f1 = open("./hello.txt", mode='r') 2 | lines = f1.readlines() 3 | 4 | print(lines) 5 | -------------------------------------------------------------------------------- /test_data/chapter14_file/03_save_line_to_file.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | file_name = "./save_line.txt" 4 | f1 = open(file_name, mode='w+') 5 | for item in range(0, 10): 6 | f1.write(str(random.randint(1, 100)) + "\n") 7 | f1.close() -------------------------------------------------------------------------------- /test_data/chapter14_file/04_replace.py: -------------------------------------------------------------------------------- 1 | message = "hello" 2 | replaced = message.replace("el", "") 3 | 4 | print(replaced) -------------------------------------------------------------------------------- /test_data/chapter14_file/05_random_number_to_file.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | file_name = "./random_number.txt" 4 | f1 = open(file_name, mode='w+') 5 | for item in range(1, 1000+1): 6 | f1.write(str(random.randint(1, 100)) + "\n") 7 | f1.close() 8 | print("파일 저장이 완료 되었습니다. 같은 폴더에 {}를 찾아보세요.".format(file_name)) 9 | 10 | -------------------------------------------------------------------------------- /test_data/chapter14_file/06_draw_chart_using_file.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pylab as plt 3 | 4 | f1 = open("./save_line.txt", mode='r') 5 | lines = f1.readlines() 6 | 7 | datas = [line.replace("\n", "") for line in lines] 8 | 9 | plt.plot(np.arange(1, len(datas)+1, 1), datas) 10 | plt.show() --------------------------------------------------------------------------------