├── Chapter1 ├── Chapter1.md ├── Chapter1.py └── ch1_3.txt ├── Chapter2 ├── Chapter2.md └── Chapter2.py ├── README.md ├── src ├── 1 │ ├── calculating_with_dictionaries │ │ └── example.py │ ├── determine_the_top_n_items_occurring_in_a_list │ │ └── example.py │ ├── extracting_a_subset_of_a_dictionary │ │ └── example.py │ ├── filtering_list_elements │ │ └── example.py │ ├── finding_out_what_two_dictionaries_have_in_common │ │ └── example.py │ ├── finding_the_largest_or_smallest_n_items │ │ └── example.py │ ├── grouping-records-together-based-on-a-field │ │ └── grouping.py │ ├── implementing_a_priority_queue │ │ └── example.py │ ├── keeping_the_last_n_items │ │ ├── example.py │ │ └── somefile.txt │ ├── mapping_names_to_sequence_elements │ │ └── example1.py │ ├── removing_duplicates_from_a_sequence_while_maintaining_order │ │ ├── example.py │ │ └── example2.py │ ├── sort_a_list_of_dictionaries_by_a_common_key │ │ └── example.py │ ├── sort_objects_without_native_comparison_support │ │ └── example.py │ ├── transforming_and_reducing_data_at_the_same_time │ │ └── example.py │ ├── unpack_a_fixed_number_of_elements_from_iterables_of_arbitrary_length │ │ └── example.py │ └── working_with_multiple_mappings_as_a_single_mapping │ │ └── example.py ├── 2 │ ├── combining_and_concatenating_strings │ │ └── example.py │ ├── matching_and_searching_for_text_patterns_using_regular_expressions │ │ └── example.py │ ├── matching_strings_using_shell_wildcard_patterns │ │ └── example.py │ ├── normalizing_unicode_text_to_a_standard_representation │ │ └── example.py │ ├── reformatting_text_to_fixed_number_of_columns │ │ └── example.py │ ├── sanitizing_and_cleaning_up_text │ │ └── example.py │ ├── searching_and_replacing_text │ │ └── example.py │ ├── specifying_a_regular_expression_for_the_shortest_match │ │ └── example.py │ ├── splitting_strings_on_any_of_multiple_delimiters │ │ └── example.py │ ├── tokenizing_text │ │ └── example.py │ ├── variable_interpolation_in_strings │ │ └── example.py │ ├── writing_a_regular_expression_for_multiline_patterns │ │ └── example.py │ └── writing_a_simple_recursive_descent_parser │ │ ├── example.py │ │ └── plyexample.py ├── 3 │ ├── determining_last_fridays_date │ │ └── example.py │ └── finding_the_date_range_for_the_current_month │ │ └── example.py ├── 4 │ ├── creating_data_processing_pipelines │ │ ├── example.py │ │ └── www │ │ │ ├── bar │ │ │ ├── access-log │ │ │ ├── access-log-0108.bz2 │ │ │ └── access-log-0208.bz2 │ │ │ └── foo │ │ │ ├── access-log │ │ │ ├── access-log-0108.gz │ │ │ └── access-log-0208.gz │ ├── creating_new_iteration_patterns_with_generators │ │ └── example.py │ ├── delegating-iteration │ │ └── example.py │ ├── easy_implementation_of_the_iterator_protocol │ │ ├── example.py │ │ └── hardexample.py │ ├── generators_with_state │ │ ├── example.py │ │ └── somefile.txt │ ├── how_to_flatten_a_nested_sequence │ │ └── example.py │ ├── iterate_over_the_index-value_pairs_of_a_list │ │ ├── example.py │ │ └── sample.dat │ ├── iterating_in_reverse │ │ └── example.py │ ├── iterating_in_sorted_order_over_merged_sorted_iterables │ │ └── example.py │ └── iterating_on_items_in_separate_containers │ │ └── example.py ├── 5 │ ├── adding_or_changing_the_encoding_of_an_already_open_file │ │ └── example.py │ ├── getting_a_directory_listing │ │ └── example.py │ ├── iterating_over_fixed-sized_records │ │ ├── data.bin │ │ └── example.py │ ├── reading_and_writing_text_data │ │ ├── example.py │ │ └── sample.txt │ ├── wrapping_an_existing_file_descriptor_as_a_file_object │ │ └── echo.py │ └── writing_bytes_to_a_text_file │ │ └── example.py ├── 6 │ ├── incremental_parsing_of_huge_xml_files │ │ ├── example.py │ │ └── potholes.xml │ ├── parsing_modifying_and_rewriting_xml │ │ ├── example.py │ │ └── pred.xml │ ├── parsing_simple_xml_data │ │ └── example.py │ ├── parsing_xml_documents_with_namespaces │ │ ├── example.py │ │ └── sample.xml │ ├── reading_and_writing_binary_arrays_of_structures │ │ ├── readrecords.py │ │ ├── unpackrecords.py │ │ └── writerecords.py │ ├── reading_and_writing_csv_data │ │ ├── example.py │ │ ├── stocks.csv │ │ └── stocks.tsv │ ├── reading_and_writing_json_data │ │ └── example.py │ └── reading_nested_and_variable_sized_binary_structures │ │ ├── example1.py │ │ ├── example2.py │ │ ├── example3.py │ │ ├── example4.py │ │ └── writepolys.py ├── 7 │ ├── accessing_variables_defined_inside_a_closure │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py │ ├── carrying_extra_state_with_callback_functions │ │ └── example.py │ ├── functions_that_accept_any_number_of_arguments │ │ └── example.py │ ├── functions_that_only_accept_keyword_arguments │ │ └── example.py │ ├── functions_with_default_arguments │ │ └── example.py │ ├── inlining_callback_functions │ │ └── example.py │ └── making_an_n-argument_callable_work_as_a_callable_with_fewer_arguments │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py ├── 8 │ ├── calling_a_method_on_a_parent_class │ │ ├── example1.py │ │ ├── example2.py │ │ ├── example3.py │ │ ├── example4.py │ │ └── example5.py │ ├── calling_a_method_on_an_object_given_the_name_as_a_string │ │ └── example.py │ ├── changing_the_string_representation_of_instances │ │ └── example.py │ ├── creating_a_new_kind_of_class_or_instance_attribute │ │ ├── example1.py │ │ └── example2.py │ ├── creating_an_instance_without_invoking_init │ │ └── example.py │ ├── creating_cached_instances │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py │ ├── creating_managed_attributes │ │ └── example.py │ ├── customized_formatting │ │ └── example1.py │ ├── delegation_and_proxies │ │ ├── example1.py │ │ ├── example2.py │ │ ├── example3.py │ │ └── example4.py │ ├── extending_a_property_in_a_subclass │ │ ├── example.py │ │ └── example2.py │ ├── extending_classes_with_mixins │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py │ ├── how_to_define_an_interface_or_abstract_base_class │ │ ├── example.py │ │ └── example2.py │ ├── how_to_define_more_than_one_constructor_in_a_class │ │ ├── example.py │ │ └── example2.py │ ├── how_to_encapsulate_names_in_a_class │ │ └── example.py │ ├── implementing_a_data_model_or_type_system │ │ ├── example.py │ │ └── example_clsdec.py │ ├── implementing_custom_containers │ │ ├── example1.py │ │ └── example2.py │ ├── implementing_stateful_objects_or_state_machines │ │ ├── example1.py │ │ └── example2.py │ ├── implementing_the_visitor_pattern │ │ └── example.py │ ├── implementing_the_visitor_pattern_without_recursion │ │ ├── example1.py │ │ ├── example2.py │ │ ├── example3.py │ │ └── node.py │ ├── lazily_computed_attributes │ │ ├── example1.py │ │ └── example2.py │ ├── making_classes_support_comparison_operations │ │ └── example.py │ ├── making_objects_support_the_context_manager_protocol │ │ ├── example1.py │ │ └── example2.py │ ├── managing_memory_in_cyclic_data_structures │ │ └── example.py │ └── simplified_initialization_of_data_structures │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py ├── 9 │ ├── applying_decorators_to_class_and_static_methods │ │ └── example.py │ ├── avoiding_repetitive_property_methods │ │ └── example1.py │ ├── capturing_class_attribute_definition_order │ │ ├── example1.py │ │ └── example2.py │ ├── defining_a_decorator_that_takes_an_optional_argument │ │ └── example.py │ ├── defining_a_decorator_that_takes_arguments │ │ └── example.py │ ├── defining_a_decorator_with_user_adjustable_attributes │ │ ├── example1.py │ │ └── example2.py │ ├── defining_a_metaclass_that_takes_optional_arguments │ │ └── example.py │ ├── defining_classes_programmatically │ │ ├── example1.py │ │ └── example2.py │ ├── defining_context_managers_the_easy_way │ │ ├── example1.py │ │ └── example2.py │ ├── defining_decorators_as_classes │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py │ ├── defining_decorators_as_part_of_a_class │ │ ├── example1.py │ │ └── example2.py │ ├── disassembling_python_byte_code │ │ └── example.py │ ├── enforcing_an_argument_signature │ │ ├── example1.py │ │ └── example2.py │ ├── enforcing_coding_conventions_in_classes │ │ ├── example1.py │ │ └── example2.py │ ├── enforcing_type_checking_on_a_function_using_a_decorator │ │ └── example.py │ ├── executing_code_with_local_side_effects │ │ └── example.py │ ├── initializing_class_members_at_definition_time │ │ └── example.py │ ├── monkeypatching_class_definitions │ │ └── example.py │ ├── multiple_dispatch_with_function_annotations │ │ ├── example1.py │ │ └── example2.py │ ├── parsing_and_analyzing_python_source │ │ ├── example1.py │ │ └── namelower.py │ ├── preserving_function_metadata_when_writing_decorators │ │ └── example.py │ ├── unwrapping_a_decorator │ │ └── example.py │ └── using_metaclasses_to_control_instance_creation │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py ├── 10 │ ├── loading_modules_from_a_remote_machine_using_import_hooks │ │ ├── explicit_load.py │ │ ├── metaexample.py │ │ ├── pathexample.py │ │ ├── testcode │ │ │ ├── fib.py │ │ │ ├── grok │ │ │ │ ├── __init__.py │ │ │ │ └── blah.py │ │ │ └── spam.py │ │ └── urlimport.py │ ├── making_separate_directories_import_under_a_common_namespace │ │ ├── bar-package │ │ │ └── spam │ │ │ │ └── grok.py │ │ ├── example.py │ │ └── foo-package │ │ │ └── spam │ │ │ └── blah.py │ ├── monkeypatching_modules_on_import │ │ ├── example1.py │ │ ├── example2.py │ │ └── postimport.py │ └── splitting_a_module_into_multiple_files │ │ ├── example.py │ │ └── mymodule │ │ ├── __init__.py │ │ ├── a.py │ │ └── b.py ├── 11 │ ├── adding_ssl_to_network_servers │ │ ├── echoclient.py │ │ ├── echoserv.py │ │ ├── makecerts.sh │ │ ├── ssl_xmlrpc_client.py │ │ ├── ssl_xmlrpc_server.py │ │ └── sslmixin.py │ ├── creating_a_simple_rest_based_interface │ │ ├── client1.py │ │ ├── example1.py │ │ └── resty.py │ ├── creating_a_tcp_server │ │ ├── echoclient.py │ │ ├── echoserv.py │ │ ├── echoserv1.py │ │ ├── echoserv2.py │ │ ├── echoserv3.py │ │ ├── echoserv4.py │ │ ├── echoserv5.py │ │ └── threadedserv.py │ ├── creating_a_udp_server │ │ ├── client.py │ │ ├── timeserv1.py │ │ └── timeserv2.py │ ├── event_driven_io_explained │ │ ├── eventhandler.py │ │ ├── tcpclient.py │ │ ├── tcpserver.py │ │ ├── threadpool.py │ │ ├── thrpoolclient.py │ │ ├── udpclient.py │ │ └── udpserver.py │ ├── implementing_remote_procedure_call │ │ ├── jsonrpclient.py │ │ ├── jsonrpcserver.py │ │ ├── rpcclient.py │ │ └── rpcserver.py │ ├── interacting_with_http_services_as_a_client │ │ ├── example1.py │ │ ├── example2.py │ │ ├── example3.py │ │ └── example4.py │ ├── passing_a_socket_file_descriptor_between_processes │ │ ├── client1.py │ │ ├── server.py │ │ ├── server1.py │ │ ├── servermp.py │ │ ├── worker.py │ │ └── workermp.py │ ├── simple_authentication_of_clients │ │ ├── auth.py │ │ ├── client.py │ │ └── server.py │ ├── simple_communication_between_interpreters │ │ ├── echoclient.py │ │ └── echoserv.py │ ├── simple_remote_procedure_call_with_xmlrpc │ │ ├── client.py │ │ └── keyserv.py │ └── zero_copy_sending_and_receiving_of_large_arrays │ │ ├── client.py │ │ ├── server.py │ │ └── zerocopy.py ├── 12 │ ├── defining_an_actor_task │ │ ├── actor.py │ │ ├── tagged.py │ │ └── worker.py │ ├── how_to_communicate_between_threads │ │ ├── example1.py │ │ └── example2.py │ ├── how_to_create_a_thread_pool │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py │ ├── how_to_determine_if_a_thread_has_started │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py │ ├── how_to_lock_critical_sections │ │ └── example1.py │ ├── how_to_start_and_stop_threads │ │ └── example.py │ ├── implementing_publish_subscribe_messaging │ │ ├── exchange1.py │ │ └── exchange2.py │ ├── launching_a_daemon_process_on_unix │ │ └── daemon.py │ ├── locking_with_deadlock_avoidance │ │ ├── deadlock.py │ │ ├── example1.py │ │ ├── example2.py │ │ └── example3.py │ ├── polling_multiple_thread_queues │ │ └── pqueue.py │ ├── simple_parallel_programming │ │ ├── findrobots.py │ │ ├── findrobots_par.py │ │ └── logs │ │ │ ├── 20121217.log.gz │ │ │ ├── 20121218.log.gz │ │ │ ├── 20121219.log.gz │ │ │ ├── 20121220.log.gz │ │ │ ├── 20121221.log.gz │ │ │ ├── 20121222.log.gz │ │ │ ├── 20121223.log.gz │ │ │ ├── 20121224.log.gz │ │ │ ├── 20121225.log.gz │ │ │ ├── 20121226.log.gz │ │ │ ├── 20121227.log.gz │ │ │ ├── 20121228.log.gz │ │ │ ├── 20121229.log.gz │ │ │ └── 20121230.log.gz │ ├── storing_thread_specific_state │ │ ├── example1.py │ │ └── example2.py │ └── using_generators_as_an_alternative_to_threads │ │ ├── actorsched.py │ │ ├── netsched.py │ │ └── simple.py ├── 13 │ ├── adding_logging_to_libraries │ │ └── somelib.py │ ├── executing_an_external_command_and_getting_its_output │ │ ├── example1.py │ │ └── example2.py │ ├── finding_files │ │ └── modified_within.py │ ├── generating_a_range_of_ip_addresses_from_a_cidr_address │ │ └── example.py │ ├── getting_the_terminal_size │ │ └── example.py │ ├── making_a_stopwatch │ │ └── stopwatch.py │ ├── parsing_command_line_options │ │ └── search.py │ ├── prompting_for_a_password_at_runtime │ │ └── example.py │ ├── putting_limits_on_memory_and_cpu_usage │ │ └── example.py │ ├── reading_configuration_files │ │ ├── config.ini │ │ └── example1.py │ └── simple_logging_for_scripts │ │ ├── example1.py │ │ ├── example2.py │ │ └── logconfig.ini ├── 14 │ ├── logging_test_output_to_a_file │ │ └── test.py │ ├── make_your_programs_run_faster │ │ └── example.py │ ├── profiling_and_timing_your_program │ │ └── timethis.py │ ├── raising_an_exception_in_response_to_another_exception │ │ └── example.py │ ├── skipping_or_anticipating_test_failures │ │ └── test.py │ ├── testing_for_exceptional_conditions_in_unit_tests │ │ └── test.py │ └── testing_output_sent_to_stdout │ │ ├── mymodule.py │ │ └── testmymodule.py └── 15 │ ├── Makefile │ ├── accessing_c_code_using_ctypes │ ├── example.py │ └── sample.py │ ├── calling_python_from_c │ ├── Makefile │ └── embed.c │ ├── consuming_an_iterable_from_c │ ├── example.py │ ├── sample.c │ └── setup.py │ ├── defining_and_exporting_c_apis_from_extension_modules │ ├── README.txt │ ├── example.py │ ├── ptexample.c │ ├── ptsetup.py │ ├── pysample.c │ ├── pysample.h │ └── setup.py │ ├── diagnosing_segmentation_faults │ ├── example.py │ ├── sample.c │ └── setup.py │ ├── managing_opaque_pointers_in_c_extension_modules │ ├── example.py │ ├── pysample.c │ └── setup.py │ ├── passing_null_terminated_strings_to_c_libraries │ ├── example.py │ ├── sample.c │ └── setup.py │ ├── passing_unicode_strings_to_c_libraries │ ├── example.py │ ├── sample.c │ └── setup.py │ ├── reading_file_like_objects_from_c │ ├── example.py │ ├── sample.c │ └── setup.py │ ├── sample.c │ ├── sample.h │ ├── turning_a_function_pointer_into_a_callable │ └── example.py │ ├── using_cython_to_write_high_performance_array_operations │ ├── example.py │ ├── sample.pyx │ └── setup.py │ ├── working_with_c_strings_of_dubious_encoding │ ├── example.py │ ├── sample.c │ └── setup.py │ ├── wrapping_c_code_with_swig │ ├── example.py │ ├── sample.i │ └── setup.py │ ├── wrapping_existing_c_code_with_cython │ ├── csample.pxd │ ├── example.py │ ├── sample.pyx │ ├── sample_alt.pyx │ ├── setup.py │ └── setup_alt.py │ ├── writing_a_simple_c_extension_module │ ├── example.py │ ├── pysample.c │ └── setup.py │ └── writing_an_extension_function_that_operates_on_arrays │ ├── example.py │ ├── pysample.c │ └── setup.py └── 《Python Cookbook》第三版中文v2.0.0.pdf /Chapter1/Chapter1.md: -------------------------------------------------------------------------------- 1 | ##第1章 数据结构和算法 2 | 3 | ##问题 4 | - 1.1 将序列分解为单独的变量 5 | - 1.2 从任意长度的可迭代对象中分解元素 6 | - 1.3 保存最后N个元素 7 | - 1.4 找到最大或最小的N个元素 8 | - 1.5 实现优先级队列 9 | - 1.6 在字典中将键映射到多个值上 10 | - 1.7 让字典保持有序 11 | - 1.8 与字典有关的计算问题 12 | - 1.9 在两个字典中寻找相同点 13 | - 1.10 从序列中移除重复项并保持元素间顺序不变 14 | - 1.11 对切片命名 15 | - 1.12 找出序列中出现次数最多的元素 16 | - 1.13 通过公共键对字典列表排序 17 | - 1.14 对不原生支持比较操作的对象排序 18 | - 1.15 根据字段将记录分组 19 | - 1.16 筛选序列中的元素 20 | - 1.17 从字典中提权子集 21 | - 1.18 将名称映射到序列的元素中 22 | - 1.19 同时对数据做转换和换算 23 | - 1.20 将对个映射合并为单个映射 24 | 25 | -------------------------------------------------------------------------------- /Chapter1/ch1_3.txt: -------------------------------------------------------------------------------- 1 | Perl 2 | Python 3 | PHP 4 | C++ 5 | Java 6 | R 7 | JavaScript 8 | HTML 9 | CSS 10 | -------------------------------------------------------------------------------- /Chapter2/Chapter2.md: -------------------------------------------------------------------------------- 1 | ##第2章 字符串和文本 2 | 3 | 4 | ##问题 5 | - 2.1 使用多个界定符分割字符串 6 | - 2.2 字符串开头或结尾匹配 7 | - 2.3 用Shell通配符匹配字符串 8 | - 2.4 字符串匹配和搜索 9 | - 2.5 字符串搜索和替换 10 | - 2.6 字符串忽略大小写的搜索替换 11 | - 2.7 最短匹配模式 12 | - 2.8 多行匹配模式 13 | - 2.9 将Unicode文本标准化 14 | - 2.10 在正则式中使用Unicode 15 | - 2.11 删除字符串中不需要的字符 16 | - 2.12 审查清理文本字符串 17 | - 2.13 字符串对齐 18 | - 2.14 合并拼接字符串 19 | - 2.15 字符串中插入变量 20 | - 2.16 以指定列宽格式化字符串 21 | - 2.17 在字符串中处理html和xml 22 | - 2.18 字符串令牌解析 23 | - 2.19 实现一个简单的递归下降分析器 24 | - 2.20 字节字符串上的字符串操作 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## python-cookbook 第三版 2 | 3 | 4 | ## 目录 5 | - [第1章 数据结构和算法](Chapter1/) 6 | - [第2章 字符串和文本](Chapter2/) 7 | - 第3章 数字，日期和时间 8 | - 第4章 迭代器和生成器 9 | - 第5章 文件和I/O 10 | - 第6章 数据编码与处理 11 | - 第7章 函数 12 | - 第8章 类与对象 13 | - 第9章 元编程 14 | - 第10章 模块与包 15 | - 第11章 网络和Web编程 16 | - 第12章 并发 17 | - 第13章 实用脚本和系统管理 18 | - 第14章 测试，调试以及异常 19 | - 第15章 C语言扩展 20 | 21 | 22 | 23 | 24 | ## 注意 25 | 代码使用Python3 26 | -------------------------------------------------------------------------------- /src/1/calculating_with_dictionaries/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of calculating with dictionaries 4 | 5 | prices = { 6 | 'ACME': 45.23, 7 | 'AAPL': 612.78, 8 | 'IBM': 205.55, 9 | 'HPQ': 37.20, 10 | 'FB': 10.75 11 | } 12 | 13 | # Find min and max price 14 | min_price = min(zip(prices.values(), prices.keys())) 15 | max_price = max(zip(prices.values(), prices.keys())) 16 | 17 | print('min price:', min_price) 18 | print('max price:', max_price) 19 | 20 | print('sorted prices:') 21 | prices_sorted = sorted(zip(prices.values(), prices.keys())) 22 | for price, name in prices_sorted: 23 | print(' ', name, price) 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/1/determine_the_top_n_items_occurring_in_a_list/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Determine the most common words in a list 4 | 5 | words = [ 6 | 'look', 'into', 'my', 'eyes', 'look', 'into', 'my', 'eyes', 7 | 'the', 'eyes', 'the', 'eyes', 'the', 'eyes', 'not', 'around', 'the', 8 | 'eyes', "don't", 'look', 'around', 'the', 'eyes', 'look', 'into', 9 | 'my', 'eyes', "you're", 'under' 10 | ] 11 | 12 | from collections import Counter 13 | word_counts = Counter(words) 14 | top_three = word_counts.most_common(3) 15 | print(top_three) 16 | # outputs [('eyes', 8), ('the', 5), ('look', 4)] 17 | 18 | # Example of merging in more words 19 | 20 | morewords = ['why','are','you','not','looking','in','my','eyes'] 21 | word_counts.update(morewords) 22 | print(word_counts.most_common(3)) 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/1/extracting_a_subset_of_a_dictionary/example.py: -------------------------------------------------------------------------------- 1 | # example of extracting a subset from a dictionary 2 | from pprint import pprint 3 | 4 | prices = { 5 | 'ACME': 45.23, 6 | 'AAPL': 612.78, 7 | 'IBM': 205.55, 8 | 'HPQ': 37.20, 9 | 'FB': 10.75 10 | } 11 | 12 | # Make a dictionary of all prices over 200 13 | p1 = { key:value for key, value in prices.items() if value > 200 } 14 | 15 | print("All prices over 200") 16 | pprint(p1) 17 | 18 | # Make a dictionary of tech stocks 19 | tech_names = { 'AAPL', 'IBM', 'HPQ', 'MSFT' } 20 | p2 = { key:value for key,value in prices.items() if key in tech_names } 21 | 22 | print("All techs") 23 | pprint(p2) 24 | -------------------------------------------------------------------------------- /src/1/filtering_list_elements/example.py: -------------------------------------------------------------------------------- 1 | # Examples of different ways to filter data 2 | 3 | mylist = [1, 4, -5, 10, -7, 2, 3, -1] 4 | 5 | # All positive values 6 | pos = [n for n in mylist if n > 0] 7 | print(pos) 8 | 9 | # All negative values 10 | neg = [n for n in mylist if n < 0] 11 | print(neg) 12 | 13 | # Negative values clipped to 0 14 | neg_clip = [n if n > 0 else 0 for n in mylist] 15 | print(neg_clip) 16 | 17 | # Positive values clipped to 0 18 | pos_clip = [n if n < 0 else 0 for n in mylist] 19 | print(pos_clip) 20 | 21 | # Compressing example 22 | 23 | addresses = [ 24 | '5412 N CLARK', 25 | '5148 N CLARK', 26 | '5800 E 58TH', 27 | '2122 N CLARK', 28 | '5645 N RAVENSWOOD', 29 | '1060 W ADDISON', 30 | '4801 N BROADWAY', 31 | '1039 W GRANVILLE', 32 | ] 33 | 34 | counts = [ 0, 3, 10, 4, 1, 7, 6, 1] 35 | 36 | from itertools import compress 37 | 38 | more5 = [ n > 5 for n in counts ] 39 | a = list(compress(addresses, more5)) 40 | print(a) 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /src/1/finding_out_what_two_dictionaries_have_in_common/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Find out what two dictionaries have in common 4 | 5 | a = { 6 | 'x' : 1, 7 | 'y' : 2, 8 | 'z' : 3 9 | } 10 | 11 | b = { 12 | 'w' : 10, 13 | 'x' : 11, 14 | 'y' : 2 15 | } 16 | 17 | print('Common keys:', a.keys() & b.keys()) 18 | print('Keys in a not in b:', a.keys() - b.keys()) 19 | print('(key,value) pairs in common:', a.items() & b.items()) 20 | 21 | -------------------------------------------------------------------------------- /src/1/finding_the_largest_or_smallest_n_items/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of using heapq to find the N smallest or largest items 4 | 5 | import heapq 6 | 7 | portfolio = [ 8 | {'name': 'IBM', 'shares': 100, 'price': 91.1}, 9 | {'name': 'AAPL', 'shares': 50, 'price': 543.22}, 10 | {'name': 'FB', 'shares': 200, 'price': 21.09}, 11 | {'name': 'HPQ', 'shares': 35, 'price': 31.75}, 12 | {'name': 'YHOO', 'shares': 45, 'price': 16.35}, 13 | {'name': 'ACME', 'shares': 75, 'price': 115.65} 14 | ] 15 | 16 | cheap = heapq.nsmallest(3, portfolio, key=lambda s: s['price']) 17 | expensive = heapq.nlargest(3, portfolio, key=lambda s: s['price']) 18 | 19 | print(cheap) 20 | print(expensive) 21 | -------------------------------------------------------------------------------- /src/1/grouping-records-together-based-on-a-field/grouping.py: -------------------------------------------------------------------------------- 1 | rows = [ 2 | {'address': '5412 N CLARK', 'date': '07/01/2012'}, 3 | {'address': '5148 N CLARK', 'date': '07/04/2012'}, 4 | {'address': '5800 E 58TH', 'date': '07/02/2012'}, 5 | {'address': '2122 N CLARK', 'date': '07/03/2012'}, 6 | {'address': '5645 N RAVENSWOOD', 'date': '07/02/2012'}, 7 | {'address': '1060 W ADDISON', 'date': '07/02/2012'}, 8 | {'address': '4801 N BROADWAY', 'date': '07/01/2012'}, 9 | {'address': '1039 W GRANVILLE', 'date': '07/04/2012'}, 10 | ] 11 | 12 | from itertools import groupby 13 | 14 | rows.sort(key=lambda r: r['date']) 15 | for date, items in groupby(rows, key=lambda r: r['date']): 16 | print(date) 17 | for i in items: 18 | print(' ', i) 19 | 20 | # Example of building a multidict 21 | from collections import defaultdict 22 | rows_by_date = defaultdict(list) 23 | for row in rows: 24 | rows_by_date[row['date']].append(row) 25 | 26 | for r in rows_by_date['07/01/2012']: 27 | print(r) 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/1/implementing_a_priority_queue/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of a priority queue 4 | 5 | import heapq 6 | 7 | class PriorityQueue: 8 | def __init__(self): 9 | self._queue = [] 10 | self._index = 0 11 | 12 | def push(self, item, priority): 13 | heapq.heappush(self._queue, (-priority, self._index, item)) 14 | self._index += 1 15 | 16 | def pop(self): 17 | return heapq.heappop(self._queue)[-1] 18 | 19 | # Example use 20 | class Item: 21 | def __init__(self, name): 22 | self.name = name 23 | def __repr__(self): 24 | return 'Item({!r})'.format(self.name) 25 | 26 | q = PriorityQueue() 27 | q.push(Item('foo'), 1) 28 | q.push(Item('bar'), 5) 29 | q.push(Item('spam'), 4) 30 | q.push(Item('grok'), 1) 31 | 32 | print("Should be bar:", q.pop()) 33 | print("Should be spam:", q.pop()) 34 | print("Should be foo:", q.pop()) 35 | print("Should be grok:", q.pop()) 36 | -------------------------------------------------------------------------------- /src/1/keeping_the_last_n_items/example.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | 3 | def search(lines, pattern, history=5): 4 | previous_lines = deque(maxlen=history) 5 | for line in lines: 6 | if pattern in line: 7 | yield line, previous_lines 8 | previous_lines.append(line) 9 | 10 | # Example use on a file 11 | if __name__ == '__main__': 12 | with open('somefile.txt') as f: 13 | for line, prevlines in search(f, 'python', 5): 14 | for pline in prevlines: 15 | print(pline, end='') 16 | print(line, end='') 17 | print('-'*20) 18 | -------------------------------------------------------------------------------- /src/1/mapping_names_to_sequence_elements/example1.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | 3 | from collections import namedtuple 4 | 5 | Stock = namedtuple('Stock', ['name', 'shares', 'price']) 6 | 7 | def compute_cost(records): 8 | total = 0.0 9 | for rec in records: 10 | s = Stock(*rec) 11 | total += s.shares * s.price 12 | return total 13 | 14 | # Some Data 15 | records = [ 16 | ('GOOG', 100, 490.1), 17 | ('ACME', 100, 123.45), 18 | ('IBM', 50, 91.15) 19 | ] 20 | 21 | print(compute_cost(records)) 22 | 23 | -------------------------------------------------------------------------------- /src/1/removing_duplicates_from_a_sequence_while_maintaining_order/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Remove duplicate entries from a sequence while keeping order 4 | 5 | def dedupe(items): 6 | seen = set() 7 | for item in items: 8 | if item not in seen: 9 | yield item 10 | seen.add(item) 11 | 12 | if __name__ == '__main__': 13 | a = [1, 5, 2, 1, 9, 1, 5, 10] 14 | print(a) 15 | print(list(dedupe(a))) 16 | -------------------------------------------------------------------------------- /src/1/removing_duplicates_from_a_sequence_while_maintaining_order/example2.py: -------------------------------------------------------------------------------- 1 | # example2.py 2 | # 3 | # Remove duplicate entries from a sequence while keeping order 4 | 5 | def dedupe(items, key=None): 6 | seen = set() 7 | for item in items: 8 | val = item if key is None else key(item) 9 | if val not in seen: 10 | yield item 11 | seen.add(val) 12 | 13 | if __name__ == '__main__': 14 | a = [ 15 | {'x': 2, 'y': 3}, 16 | {'x': 1, 'y': 4}, 17 | {'x': 2, 'y': 3}, 18 | {'x': 2, 'y': 3}, 19 | {'x': 10, 'y': 15} 20 | ] 21 | print(a) 22 | print(list(dedupe(a, key=lambda a: (a['x'],a['y'])))) 23 | 24 | -------------------------------------------------------------------------------- /src/1/sort_a_list_of_dictionaries_by_a_common_key/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Sort a list of a dicts on a common key 4 | 5 | rows = [ 6 | {'fname': 'Brian', 'lname': 'Jones', 'uid': 1003}, 7 | {'fname': 'David', 'lname': 'Beazley', 'uid': 1002}, 8 | {'fname': 'John', 'lname': 'Cleese', 'uid': 1001}, 9 | {'fname': 'Big', 'lname': 'Jones', 'uid': 1004} 10 | ] 11 | 12 | from operator import itemgetter 13 | 14 | rows_by_fname = sorted(rows, key=itemgetter('fname')) 15 | rows_by_uid = sorted(rows, key=itemgetter('uid')) 16 | 17 | from pprint import pprint 18 | 19 | print("Sorted by fname:") 20 | pprint(rows_by_fname) 21 | 22 | print("Sorted by uid:") 23 | pprint(rows_by_uid) 24 | 25 | rows_by_lfname = sorted(rows, key=itemgetter('lname','fname')) 26 | print("Sorted by lname,fname:") 27 | pprint(rows_by_lfname) 28 | -------------------------------------------------------------------------------- /src/1/sort_objects_without_native_comparison_support/example.py: -------------------------------------------------------------------------------- 1 | from operator import attrgetter 2 | 3 | class User: 4 | def __init__(self, user_id): 5 | self.user_id = user_id 6 | def __repr__(self): 7 | return 'User({})'.format(self.user_id) 8 | 9 | # Example 10 | users = [User(23), User(3), User(99)] 11 | print(users) 12 | 13 | # Sort it by user-id 14 | print(sorted(users, key=attrgetter('user_id'))) 15 | -------------------------------------------------------------------------------- /src/1/transforming_and_reducing_data_at_the_same_time/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Some examples of using generators in arguments 4 | 5 | import os 6 | files = os.listdir(os.path.expanduser('~')) 7 | if any(name.endswith('.py') for name in files): 8 | print('There be python!') 9 | else: 10 | print('Sorry, no python.') 11 | 12 | # Output a tuple as CSV 13 | s = ('ACME', 50, 123.45) 14 | print(','.join(str(x) for x in s)) 15 | 16 | # Data reduction across fields of a data structure 17 | portfolio = [ 18 | {'name':'GOOG', 'shares': 50}, 19 | {'name':'YHOO', 'shares': 75}, 20 | {'name':'AOL', 'shares': 20}, 21 | {'name':'SCOX', 'shares': 65} 22 | ] 23 | min_shares = min(s['shares'] for s in portfolio) 24 | print(min_shares) 25 | -------------------------------------------------------------------------------- /src/1/unpack_a_fixed_number_of_elements_from_iterables_of_arbitrary_length/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Unpacking of tagged tuples of varying sizes 4 | 5 | records = [ 6 | ('foo', 1, 2), 7 | ('bar', 'hello'), 8 | ('foo', 3, 4), 9 | ] 10 | 11 | def do_foo(x,y): 12 | print('foo', x, y) 13 | 14 | def do_bar(s): 15 | print('bar', s) 16 | 17 | for tag, *args in records: 18 | if tag == 'foo': 19 | do_foo(*args) 20 | elif tag == 'bar': 21 | do_bar(*args) 22 | -------------------------------------------------------------------------------- /src/1/working_with_multiple_mappings_as_a_single_mapping/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of combining dicts into a chainmap 4 | 5 | a = {'x': 1, 'z': 3 } 6 | b = {'y': 2, 'z': 4 } 7 | 8 | # (a) Simple example of combining 9 | from collections import ChainMap 10 | c = ChainMap(a,b) 11 | print(c['x']) # Outputs 1 (from a) 12 | print(c['y']) # Outputs 2 (from b) 13 | print(c['z']) # Outputs 3 (from a) 14 | 15 | # Output some common values 16 | print('len(c):', len(c)) 17 | print('c.keys():', list(c.keys())) 18 | print('c.values():', list(c.values())) 19 | 20 | # Modify some values 21 | c['z'] = 10 22 | c['w'] = 40 23 | del c['x'] 24 | print("a:", a) 25 | 26 | 27 | # Example of stacking mappings (like scopes) 28 | values = ChainMap() 29 | values['x'] = 1 30 | 31 | # Add a new mapping 32 | values = values.new_child() 33 | values['x'] = 2 34 | 35 | # Add a new mapping 36 | values = values.new_child() 37 | values['x'] = 3 38 | 39 | print(values) 40 | print(values['x']) 41 | 42 | # Discard last mapping 43 | values = values.parents 44 | print(values) 45 | print(values['x']) 46 | 47 | # Discard last mapping 48 | values = values.parents 49 | print(values) 50 | print(values['x']) 51 | 52 | -------------------------------------------------------------------------------- /src/10/loading_modules_from_a_remote_machine_using_import_hooks/explicit_load.py: -------------------------------------------------------------------------------- 1 | # Example of explicit module loading using imp library 2 | 3 | import imp 4 | import urllib.request 5 | import sys 6 | 7 | def load_module(url): 8 | u = urllib.request.urlopen(url) 9 | source = u.read().decode('utf-8') 10 | mod = sys.modules.setdefault(url, imp.new_module(url)) 11 | code = compile(source, url, 'exec') 12 | mod.__file__ = url 13 | mod.__package__ = '' 14 | exec(code, mod.__dict__) 15 | return mod 16 | 17 | if __name__ == '__main__': 18 | fib = load_module('http://localhost:15000/fib.py') 19 | print(fib.fib(10)) 20 | spam = load_module('http://localhost:15000/spam.py') 21 | spam.hello('Guido') 22 | print(fib) 23 | print(spam) 24 | -------------------------------------------------------------------------------- /src/10/loading_modules_from_a_remote_machine_using_import_hooks/metaexample.py: -------------------------------------------------------------------------------- 1 | # metaexample.py 2 | # 3 | # Example of using a meta-path importer 4 | 5 | # Enable for debugging 6 | if False: 7 | import logging 8 | logging.basicConfig(level=logging.DEBUG) 9 | 10 | import urlimport 11 | urlimport.install_meta('http://localhost:15000') 12 | 13 | import fib 14 | import spam 15 | import grok.blah 16 | print(grok.blah.__file__) 17 | -------------------------------------------------------------------------------- /src/10/loading_modules_from_a_remote_machine_using_import_hooks/pathexample.py: -------------------------------------------------------------------------------- 1 | # Example of path-path import hook 2 | 3 | # Enable for debugging 4 | if False: 5 | import logging 6 | logging.basicConfig(level=logging.DEBUG) 7 | 8 | import urlimport 9 | urlimport.install_path_hook() 10 | 11 | import sys 12 | sys.path.append('http://localhost:15000') 13 | 14 | import fib 15 | import spam 16 | import grok.blah 17 | print(grok.blah.__file__) 18 | 19 | -------------------------------------------------------------------------------- /src/10/loading_modules_from_a_remote_machine_using_import_hooks/testcode/fib.py: -------------------------------------------------------------------------------- 1 | print("I'm fib") 2 | 3 | def fib(n): 4 | if n < 2: 5 | return 1 6 | else: 7 | return fib(n-1) + fib(n-2) 8 | -------------------------------------------------------------------------------- /src/10/loading_modules_from_a_remote_machine_using_import_hooks/testcode/grok/__init__.py: -------------------------------------------------------------------------------- 1 | print("I'm grok.__init__") 2 | -------------------------------------------------------------------------------- /src/10/loading_modules_from_a_remote_machine_using_import_hooks/testcode/grok/blah.py: -------------------------------------------------------------------------------- 1 | print("I'm grok.blah") 2 | -------------------------------------------------------------------------------- /src/10/loading_modules_from_a_remote_machine_using_import_hooks/testcode/spam.py: -------------------------------------------------------------------------------- 1 | print("I'm spam") 2 | 3 | def hello(name): 4 | print('Hello %s' % name) 5 | -------------------------------------------------------------------------------- /src/10/making_separate_directories_import_under_a_common_namespace/bar-package/spam/grok.py: -------------------------------------------------------------------------------- 1 | print('bar-package grok!') 2 | -------------------------------------------------------------------------------- /src/10/making_separate_directories_import_under_a_common_namespace/example.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.extend(['foo-package', 'bar-package']) 3 | import spam.blah 4 | import spam.grok 5 | -------------------------------------------------------------------------------- /src/10/making_separate_directories_import_under_a_common_namespace/foo-package/spam/blah.py: -------------------------------------------------------------------------------- 1 | print('foo-package blah!') 2 | -------------------------------------------------------------------------------- /src/10/monkeypatching_modules_on_import/example1.py: -------------------------------------------------------------------------------- 1 | from postimport import when_imported 2 | 3 | @when_imported('threading') 4 | def warn_threads(mod): 5 | print('Threads? Are you crazy?') 6 | 7 | if __name__ == '__main__': 8 | import threading 9 | -------------------------------------------------------------------------------- /src/10/monkeypatching_modules_on_import/example2.py: -------------------------------------------------------------------------------- 1 | from postimport import when_imported 2 | from functools import wraps 3 | 4 | 5 | def logged(func): 6 | @wraps(func) 7 | def wrapper(*args, **kwargs): 8 | print('Calling', func.__name__, args, kwargs) 9 | return func(*args, **kwargs) 10 | return wrapper 11 | 12 | # Example 13 | @when_imported('math') 14 | def add_logging(mod): 15 | mod.cos = logged(mod.cos) 16 | mod.sin = logged(mod.sin) 17 | 18 | if __name__ == '__main__': 19 | import math 20 | print(math.cos(2)) 21 | print(math.sin(2)) 22 | 23 | -------------------------------------------------------------------------------- /src/10/monkeypatching_modules_on_import/postimport.py: -------------------------------------------------------------------------------- 1 | # postimport.py 2 | 3 | import importlib 4 | import sys 5 | from collections import defaultdict 6 | 7 | _post_import_hooks = defaultdict(list) 8 | 9 | class PostImportFinder: 10 | def __init__(self): 11 | self._skip = set() 12 | 13 | def find_module(self, fullname, path=None): 14 | if fullname in self._skip: 15 | return None 16 | self._skip.add(fullname) 17 | return PostImportLoader(self) 18 | 19 | class PostImportLoader: 20 | def __init__(self, finder): 21 | self._finder = finder 22 | 23 | def load_module(self, fullname): 24 | importlib.import_module(fullname) 25 | module = sys.modules[fullname] 26 | for func in _post_import_hooks[fullname]: 27 | func(module) 28 | self._finder._skip.remove(fullname) 29 | return module 30 | 31 | def when_imported(fullname): 32 | def decorate(func): 33 | if fullname in sys.modules: 34 | func(sys.modules[fullname]) 35 | else: 36 | _post_import_hooks[fullname].append(func) 37 | return func 38 | return decorate 39 | 40 | sys.meta_path.insert(0, PostImportFinder()) 41 | -------------------------------------------------------------------------------- /src/10/splitting_a_module_into_multiple_files/example.py: -------------------------------------------------------------------------------- 1 | import mymodule 2 | a = mymodule.A() 3 | a.spam() 4 | 5 | b = mymodule.B() 6 | b.bar() 7 | -------------------------------------------------------------------------------- /src/10/splitting_a_module_into_multiple_files/mymodule/__init__.py: -------------------------------------------------------------------------------- 1 | # __init__.py 2 | 3 | from .a import A 4 | from .b import B 5 | 6 | -------------------------------------------------------------------------------- /src/10/splitting_a_module_into_multiple_files/mymodule/a.py: -------------------------------------------------------------------------------- 1 | # a.py 2 | 3 | class A: 4 | def spam(self): 5 | print('A.spam') 6 | 7 | -------------------------------------------------------------------------------- /src/10/splitting_a_module_into_multiple_files/mymodule/b.py: -------------------------------------------------------------------------------- 1 | # b.py 2 | 3 | from .a import A 4 | 5 | class B(A): 6 | def bar(self): 7 | print('B.bar') 8 | 9 | -------------------------------------------------------------------------------- /src/11/adding_ssl_to_network_servers/echoclient.py: -------------------------------------------------------------------------------- 1 | # echoclient.py 2 | # 3 | # An example of a client that connects to an SSL server 4 | # and verifies its certificate 5 | 6 | from socket import socket, AF_INET, SOCK_STREAM 7 | import ssl 8 | 9 | s = socket(AF_INET, SOCK_STREAM) 10 | 11 | # Wrap with an SSL layer and require the server to present its certificate 12 | ssl_s = ssl.wrap_socket(s, 13 | cert_reqs=ssl.CERT_REQUIRED, 14 | ca_certs='server_cert.pem', 15 | ) 16 | 17 | ssl_s.connect(('localhost', 20000)) 18 | 19 | # Communicate with the server 20 | ssl_s.send(b'Hello World!') 21 | resp = ssl_s.recv(8192) 22 | print('Got:', resp) 23 | 24 | # Done 25 | ssl_s.close() 26 | -------------------------------------------------------------------------------- /src/11/adding_ssl_to_network_servers/echoserv.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | from socket import SOL_SOCKET, SO_REUSEADDR 3 | import ssl 4 | 5 | KEYFILE = 'server_key.pem' # Private key of the server 6 | CERTFILE = 'server_cert.pem' # Server certificate (given to client) 7 | 8 | def echo_client(s): 9 | while True: 10 | data = s.recv(8192) 11 | if data == b'': 12 | break 13 | s.send(data) 14 | s.close() 15 | print('Connection closed') 16 | 17 | def echo_server(address): 18 | s = socket(AF_INET, SOCK_STREAM) 19 | s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) 20 | s.bind(address) 21 | s.listen(1) 22 | 23 | # Wrap with an SSL layer requiring client certs 24 | s_ssl = ssl.wrap_socket(s, 25 | keyfile=KEYFILE, 26 | certfile=CERTFILE, 27 | server_side=True 28 | ) 29 | # Wait for connections 30 | while True: 31 | try: 32 | c,a = s_ssl.accept() 33 | print('Got connection', c, a) 34 | echo_client(c) 35 | except Exception as e: 36 | print('{}: {}'.format(e.__class__.__name__, e)) 37 | 38 | echo_server(('', 20000)) 39 | -------------------------------------------------------------------------------- /src/11/adding_ssl_to_network_servers/makecerts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | openssl req -new -x509 -days 365 -nodes -out server_cert.pem -keyout server_key.pem 4 | openssl req -new -x509 -days 365 -nodes -out client_cert.pem -keyout client_key.pem 5 | -------------------------------------------------------------------------------- /src/11/adding_ssl_to_network_servers/ssl_xmlrpc_client.py: -------------------------------------------------------------------------------- 1 | # ssl_xmlrpc_client.py 2 | # 3 | # An XML-RPC client that verifies the server certificate 4 | 5 | from xmlrpc.client import SafeTransport, ServerProxy 6 | import ssl 7 | 8 | class VerifyCertSafeTransport(SafeTransport): 9 | def __init__(self, cafile, certfile=None, keyfile=None): 10 | super().__init__() 11 | self._ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) 12 | self._ssl_context.load_verify_locations(cafile) 13 | if certfile: 14 | self._ssl_context.load_cert_chain(certfile, keyfile) 15 | self._ssl_context.verify_mode = ssl.CERT_REQUIRED 16 | 17 | def make_connection(self, host): 18 | s = super().make_connection((host, {'context': self._ssl_context})) 19 | 20 | return s 21 | 22 | # Create the client proxy 23 | s = ServerProxy('https://localhost:15000', 24 | transport=VerifyCertSafeTransport('server_cert.pem', 'client_cert.pem', 'client_key.pem'), 25 | # transport=VerifyCertSafeTransport('server_cert.pem'), 26 | allow_none=True) 27 | 28 | s.set('foo', 'bar') 29 | s.set('spam', [1, 2, 3]) 30 | print(s.keys()) 31 | print(s.get('foo')) 32 | print(s.get('spam')) 33 | s.delete('spam') 34 | print(s.exists('spam')) 35 | 36 | -------------------------------------------------------------------------------- /src/11/adding_ssl_to_network_servers/sslmixin.py: -------------------------------------------------------------------------------- 1 | import ssl 2 | 3 | class SSLMixin: 4 | def __init__(self, *args, 5 | keyfile=None, certfile=None, ca_certs=None, cert_reqs=ssl.CERT_NONE, 6 | **kwargs): 7 | self._keyfile = keyfile 8 | self._certfile = certfile 9 | self._ca_certs = ca_certs 10 | self._cert_reqs = cert_reqs 11 | super().__init__(*args, **kwargs) 12 | 13 | def get_request(self): 14 | client, addr = super().get_request() 15 | client_ssl = ssl.wrap_socket(client, 16 | keyfile = self._keyfile, 17 | certfile = self._certfile, 18 | ca_certs = self._ca_certs, 19 | cert_reqs = self._cert_reqs, 20 | server_side = True) 21 | return client_ssl, addr 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/11/creating_a_simple_rest_based_interface/client1.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | 3 | u = urlopen('http://localhost:8080/hello?name=Guido') 4 | print(u.read().decode('utf-8')) 5 | 6 | u = urlopen('http://localhost:8080/localtime') 7 | print(u.read().decode('utf-8')) 8 | -------------------------------------------------------------------------------- /src/11/creating_a_simple_rest_based_interface/resty.py: -------------------------------------------------------------------------------- 1 | # resty.py 2 | 3 | import cgi 4 | 5 | def notfound_404(environ, start_response): 6 | start_response('404 Not Found', [ ('Content-type', 'text/plain') ]) 7 | return [b'Not Found'] 8 | 9 | class PathDispatcher: 10 | def __init__(self): 11 | self.pathmap = { } 12 | 13 | def __call__(self, environ, start_response): 14 | path = environ['PATH_INFO'] 15 | params = cgi.FieldStorage(environ['wsgi.input'], 16 | environ=environ) 17 | method = environ['REQUEST_METHOD'].lower() 18 | environ['params'] = { key: params.getvalue(key) for key in params } 19 | handler = self.pathmap.get((method,path), notfound_404) 20 | return handler(environ, start_response) 21 | 22 | def register(self, method, path, function): 23 | self.pathmap[method.lower(), path] = function 24 | return function 25 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/echoclient.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | s = socket(AF_INET, SOCK_STREAM) 3 | s.connect(('localhost', 20000)) 4 | 5 | s.send(b'Hello\n') 6 | resp = s.recv(8192) 7 | print('Response:', resp) 8 | s.close() 9 | 10 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/echoserv.py: -------------------------------------------------------------------------------- 1 | from socketserver import BaseRequestHandler, TCPServer 2 | 3 | class EchoHandler(BaseRequestHandler): 4 | def handle(self): 5 | print('Got connection from', self.client_address) 6 | while True: 7 | msg = self.request.recv(8192) 8 | if not msg: 9 | break 10 | self.request.send(msg) 11 | 12 | if __name__ == '__main__': 13 | serv = TCPServer(('', 20000), EchoHandler) 14 | print('Echo server running on port 20000') 15 | serv.serve_forever() 16 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/echoserv1.py: -------------------------------------------------------------------------------- 1 | from socketserver import BaseRequestHandler, TCPServer 2 | 3 | class EchoHandler(BaseRequestHandler): 4 | def handle(self): 5 | print('Got connection from', self.client_address) 6 | while True: 7 | msg = self.request.recv(8192) 8 | if not msg: 9 | break 10 | self.request.send(msg) 11 | 12 | if __name__ == '__main__': 13 | serv = TCPServer(('', 20000), EchoHandler) 14 | print('Echo server running on port 20000') 15 | serv.serve_forever() 16 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/echoserv2.py: -------------------------------------------------------------------------------- 1 | from socketserver import StreamRequestHandler, TCPServer 2 | 3 | class EchoHandler(StreamRequestHandler): 4 | def handle(self): 5 | print('Got connection from', self.client_address) 6 | # self.rfile is a file-like object for reading 7 | for line in self.rfile: 8 | # self.wfile is a file-like object for writing 9 | self.wfile.write(line) 10 | 11 | if __name__ == '__main__': 12 | serv = TCPServer(('', 20000), EchoHandler) 13 | print('Echo server running on port 20000') 14 | serv.serve_forever() 15 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/echoserv3.py: -------------------------------------------------------------------------------- 1 | from socketserver import StreamRequestHandler, TCPServer 2 | 3 | class EchoHandler(StreamRequestHandler): 4 | def handle(self): 5 | print('Got connection from', self.client_address) 6 | # self.rfile is a file-like object for reading 7 | for line in self.rfile: 8 | # self.wfile is a file-like object for writing 9 | self.wfile.write(line) 10 | 11 | if __name__ == '__main__': 12 | import socket 13 | 14 | serv = TCPServer(('', 20000), EchoHandler, bind_and_activate=False) 15 | # Set up various socket options 16 | serv.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) 17 | # Bind and activate 18 | serv.server_bind() 19 | serv.server_activate() 20 | print('Echo server running on port 20000') 21 | serv.serve_forever() 22 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/echoserv4.py: -------------------------------------------------------------------------------- 1 | from socketserver import StreamRequestHandler, TCPServer 2 | import socket 3 | 4 | class EchoHandler(StreamRequestHandler): 5 | timeout = 5 6 | rbufsize = -1 7 | wbufsize = 0 8 | disable_nagle_algorithm = False 9 | def handle(self): 10 | print('Got connection from', self.client_address) 11 | # self.rfile is a file-like object for reading 12 | try: 13 | for line in self.rfile: 14 | # self.wfile is a file-like object for writing 15 | self.wfile.write(line) 16 | except socket.timeout: 17 | print('Timed out!') 18 | 19 | if __name__ == '__main__': 20 | serv = TCPServer(('', 20000), EchoHandler) 21 | print('Echo server running on port 20000') 22 | serv.serve_forever() 23 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/echoserv5.py: -------------------------------------------------------------------------------- 1 | # Echo server using sockets directly 2 | 3 | from socket import socket, AF_INET, SOCK_STREAM 4 | 5 | def echo_handler(address, client_sock): 6 | print('Got connection from {}'.format(address)) 7 | while True: 8 | msg = client_sock.recv(8192) 9 | if not msg: 10 | break 11 | client_sock.sendall(msg) 12 | client_sock.close() 13 | 14 | def echo_server(address, backlog=5): 15 | sock = socket(AF_INET, SOCK_STREAM) 16 | sock.bind(address) 17 | sock.listen(backlog) 18 | while True: 19 | client_sock, client_addr = sock.accept() 20 | echo_handler(client_addr, client_sock) 21 | 22 | if __name__ == '__main__': 23 | echo_server(('', 20000)) 24 | -------------------------------------------------------------------------------- /src/11/creating_a_tcp_server/threadedserv.py: -------------------------------------------------------------------------------- 1 | from socketserver import StreamRequestHandler, TCPServer 2 | 3 | class EchoHandler(StreamRequestHandler): 4 | def handle(self): 5 | print('Got connection from', self.client_address) 6 | # self.rfile is a file-like object for reading 7 | for line in self.rfile: 8 | # self.wfile is a file-like object for writing 9 | self.wfile.write(line) 10 | 11 | if __name__ == '__main__': 12 | from threading import Thread 13 | NWORKERS = 16 14 | serv = TCPServer(('', 20000), EchoHandler) 15 | for n in range(NWORKERS): 16 | t = Thread(target=serv.serve_forever) 17 | t.daemon = True 18 | t.start() 19 | print('Multithreaded server running on port 20000') 20 | serv.serve_forever() 21 | -------------------------------------------------------------------------------- /src/11/creating_a_udp_server/client.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_DGRAM 2 | 3 | s = socket(AF_INET, SOCK_DGRAM) 4 | s.sendto(b'', ('localhost', 20000)) 5 | print(s.recvfrom(8192)) 6 | -------------------------------------------------------------------------------- /src/11/creating_a_udp_server/timeserv1.py: -------------------------------------------------------------------------------- 1 | from socketserver import BaseRequestHandler, UDPServer 2 | import time 3 | 4 | class TimeHandler(BaseRequestHandler): 5 | def handle(self): 6 | print('Got connection from', self.client_address) 7 | # Get message and client socket 8 | msg, sock = self.request 9 | resp = time.ctime() 10 | sock.sendto(resp.encode('ascii'), self.client_address) 11 | 12 | if __name__ == '__main__': 13 | serv = UDPServer(('', 20000), TimeHandler) 14 | serv.serve_forever() 15 | -------------------------------------------------------------------------------- /src/11/creating_a_udp_server/timeserv2.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_DGRAM 2 | import time 3 | 4 | def time_server(address): 5 | sock = socket(AF_INET, SOCK_DGRAM) 6 | sock.bind(address) 7 | while True: 8 | msg, addr = sock.recvfrom(8192) 9 | print('Got message from', addr) 10 | resp = time.ctime() 11 | sock.sendto(resp.encode('ascii'), addr) 12 | 13 | if __name__ == '__main__': 14 | time_server(('', 20000)) 15 | -------------------------------------------------------------------------------- /src/11/event_driven_io_explained/eventhandler.py: -------------------------------------------------------------------------------- 1 | class EventHandler: 2 | def fileno(self): 3 | 'Return the associated file descriptor' 4 | raise NotImplemented('must implement') 5 | 6 | def wants_to_receive(self): 7 | 'Return True if receiving is allowed' 8 | return False 9 | 10 | def handle_receive(self): 11 | 'Perform the receive operation' 12 | pass 13 | 14 | def wants_to_send(self): 15 | 'Return True if sending is requested' 16 | return False 17 | 18 | def handle_send(self): 19 | 'Send outgoing data' 20 | pass 21 | 22 | import select 23 | 24 | def event_loop(handlers): 25 | while True: 26 | wants_recv = [h for h in handlers if h.wants_to_receive()] 27 | wants_send = [h for h in handlers if h.wants_to_send()] 28 | can_recv, can_send, _ = select.select(wants_recv, wants_send, []) 29 | for h in can_recv: 30 | h.handle_receive() 31 | for h in can_send: 32 | h.handle_send() 33 | -------------------------------------------------------------------------------- /src/11/event_driven_io_explained/tcpclient.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | 3 | s = socket(AF_INET, SOCK_STREAM) 4 | s.connect(('localhost', 16000)) 5 | s.send(b'Hello\n') 6 | print('Got:', s.recv(8192)) 7 | s.close() 8 | -------------------------------------------------------------------------------- /src/11/event_driven_io_explained/thrpoolclient.py: -------------------------------------------------------------------------------- 1 | from socket import * 2 | sock = socket(AF_INET, SOCK_DGRAM) 3 | for x in range(40): 4 | sock.sendto(str(x).encode('ascii'), ('localhost', 16000)) 5 | resp = sock.recvfrom(8192) 6 | print(resp[0]) 7 | -------------------------------------------------------------------------------- /src/11/event_driven_io_explained/udpclient.py: -------------------------------------------------------------------------------- 1 | from socket import * 2 | s = socket(AF_INET, SOCK_DGRAM) 3 | s.sendto(b'', ('localhost', 14000)) 4 | print(s.recvfrom(128)) 5 | 6 | s.sendto(b'Hello', ('localhost', 15000)) 7 | print(s.recvfrom(128)) 8 | -------------------------------------------------------------------------------- /src/11/event_driven_io_explained/udpserver.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import time 3 | 4 | from eventhandler import EventHandler, event_loop 5 | 6 | class UDPServer(EventHandler): 7 | def __init__(self, address): 8 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 9 | self.sock.bind(address) 10 | 11 | def fileno(self): 12 | return self.sock.fileno() 13 | 14 | def wants_to_receive(self): 15 | return True 16 | 17 | class UDPTimeServer(UDPServer): 18 | def handle_receive(self): 19 | msg, addr = self.sock.recvfrom(1) 20 | self.sock.sendto(time.ctime().encode('ascii'), addr) 21 | 22 | class UDPEchoServer(UDPServer): 23 | def handle_receive(self): 24 | msg, addr = self.sock.recvfrom(8192) 25 | self.sock.sendto(msg, addr) 26 | 27 | if __name__ == '__main__': 28 | handlers = [ UDPTimeServer(('',14000)), UDPEchoServer(('',15000)) ] 29 | event_loop(handlers) 30 | -------------------------------------------------------------------------------- /src/11/implementing_remote_procedure_call/jsonrpclient.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | class RPCProxy: 4 | def __init__(self, connection): 5 | self._connection = connection 6 | def __getattr__(self, name): 7 | def do_rpc(*args, **kwargs): 8 | self._connection.send(json.dumps((name, args, kwargs))) 9 | result = json.loads(self._connection.recv()) 10 | return result 11 | return do_rpc 12 | 13 | # Example use 14 | from multiprocessing.connection import Client 15 | c = Client(('localhost', 17000), authkey=b'peekaboo') 16 | proxy = RPCProxy(c) 17 | print(proxy.add(2, 3)) 18 | print(proxy.sub(2, 3)) 19 | try: 20 | print(proxy.sub([1, 2], 4)) 21 | except Exception as e: 22 | print(e) 23 | -------------------------------------------------------------------------------- /src/11/implementing_remote_procedure_call/rpcclient.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | class RPCProxy: 4 | def __init__(self, connection): 5 | self._connection = connection 6 | def __getattr__(self, name): 7 | def do_rpc(*args, **kwargs): 8 | self._connection.send(pickle.dumps((name, args, kwargs))) 9 | result = pickle.loads(self._connection.recv()) 10 | if isinstance(result, Exception): 11 | raise result 12 | return result 13 | return do_rpc 14 | 15 | # Example use 16 | from multiprocessing.connection import Client 17 | c = Client(('localhost', 17000), authkey=b'peekaboo') 18 | proxy = RPCProxy(c) 19 | print(proxy.add(2, 3)) 20 | print(proxy.sub(2, 3)) 21 | try: 22 | proxy.sub([1, 2], 4) 23 | except Exception as e: 24 | print(e) 25 | -------------------------------------------------------------------------------- /src/11/interacting_with_http_services_as_a_client/example1.py: -------------------------------------------------------------------------------- 1 | # A basic GET request 2 | 3 | from urllib import request, parse 4 | 5 | # Base URL being accessed 6 | url = 'http://httpbin.org/get' 7 | 8 | # Dictionary of query parameters (if any) 9 | parms = { 10 | 'name1' : 'value1', 11 | 'name2' : 'value2' 12 | } 13 | 14 | # Encode the query string 15 | querystring = parse.urlencode(parms) 16 | 17 | # Make a GET request and read the response 18 | u = request.urlopen(url+'?' + querystring) 19 | resp = u.read() 20 | 21 | import json 22 | from pprint import pprint 23 | 24 | json_resp = json.loads(resp.decode('utf-8')) 25 | pprint(json_resp) 26 | 27 | -------------------------------------------------------------------------------- /src/11/interacting_with_http_services_as_a_client/example2.py: -------------------------------------------------------------------------------- 1 | # A basic POST request 2 | 3 | from urllib import request, parse 4 | 5 | # Base URL being accessed 6 | url = 'http://httpbin.org/post' 7 | 8 | # Dictionary of query parameters (if any) 9 | parms = { 10 | 'name1' : 'value1', 11 | 'name2' : 'value2' 12 | } 13 | 14 | # Encode the query string 15 | querystring = parse.urlencode(parms) 16 | 17 | # Make a POST request and read the response 18 | u = request.urlopen(url, querystring.encode('ascii')) 19 | resp = u.read() 20 | 21 | import json 22 | from pprint import pprint 23 | 24 | json_resp = json.loads(resp.decode('utf-8')) 25 | pprint(json_resp) 26 | 27 | -------------------------------------------------------------------------------- /src/11/interacting_with_http_services_as_a_client/example3.py: -------------------------------------------------------------------------------- 1 | # A POST request using requests library 2 | import requests 3 | 4 | # Base URL being accessed 5 | url = 'http://httpbin.org/post' 6 | 7 | # Dictionary of query parameters (if any) 8 | parms = { 9 | 'name1' : 'value1', 10 | 'name2' : 'value2' 11 | } 12 | 13 | # Extra headers 14 | headers = { 15 | 'User-agent' : 'none/ofyourbusiness', 16 | 'Spam' : 'Eggs' 17 | } 18 | 19 | resp = requests.post(url, data=parms, headers=headers) 20 | 21 | # Decoded text returned by the request 22 | text = resp.text 23 | 24 | from pprint import pprint 25 | pprint(resp.json) 26 | -------------------------------------------------------------------------------- /src/11/interacting_with_http_services_as_a_client/example4.py: -------------------------------------------------------------------------------- 1 | # Example of a HEAD request 2 | 3 | import requests 4 | 5 | resp = requests.head('http://www.python.org/index.html') 6 | 7 | status = resp.status_code 8 | last_modified = resp.headers['last-modified'] 9 | content_type = resp.headers['content-type'] 10 | content_length = resp.headers['content-length'] 11 | 12 | print(status) 13 | print(last_modified) 14 | print(content_type) 15 | print(content_length) 16 | -------------------------------------------------------------------------------- /src/11/passing_a_socket_file_descriptor_between_processes/client1.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | 3 | s = socket(AF_INET, SOCK_STREAM) 4 | s.connect(('localhost', 15000)) 5 | s.send(b'Hello\n') 6 | print('Got:', s.recv(8192)) 7 | s.send(b'World\n') 8 | print('Got:', s.recv(8192)) 9 | s.close() 10 | -------------------------------------------------------------------------------- /src/11/passing_a_socket_file_descriptor_between_processes/server.py: -------------------------------------------------------------------------------- 1 | # server.py 2 | import socket 3 | import struct 4 | 5 | def send_fd(sock, fd): 6 | ''' 7 | Send a single file descriptor. 8 | ''' 9 | sock.sendmsg([b'x'], 10 | [(socket.SOL_SOCKET, socket.SCM_RIGHTS, struct.pack('i', fd))]) 11 | ack = sock.recv(2) 12 | assert ack == b'OK' 13 | 14 | def server(work_address, port): 15 | # Wait for the worker to connect 16 | work_serv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 17 | work_serv.bind(work_address) 18 | work_serv.listen(1) 19 | worker, addr = work_serv.accept() 20 | 21 | # Now run a TCP/IP server and send clients to worker 22 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 23 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) 24 | s.bind(('',port)) 25 | s.listen(1) 26 | while True: 27 | client, addr = s.accept() 28 | print('SERVER: Got connection from', addr) 29 | send_fd(worker, client.fileno()) 30 | client.close() 31 | 32 | if __name__ == '__main__': 33 | import sys 34 | if len(sys.argv) != 3: 35 | print('Usage: server.py server_address port', file=sys.stderr) 36 | raise SystemExit(1) 37 | 38 | server(sys.argv[1], int(sys.argv[2])) 39 | -------------------------------------------------------------------------------- /src/11/passing_a_socket_file_descriptor_between_processes/servermp.py: -------------------------------------------------------------------------------- 1 | # servermp.py 2 | from multiprocessing.connection import Listener 3 | from multiprocessing.reduction import send_handle 4 | import socket 5 | 6 | def server(work_address, port): 7 | # Wait for the worker to connect 8 | work_serv = Listener(work_address, authkey=b'peekaboo') 9 | worker = work_serv.accept() 10 | worker_pid = worker.recv() 11 | 12 | # Now run a TCP/IP server and send clients to worker 13 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) 15 | s.bind(('', port)) 16 | s.listen(1) 17 | while True: 18 | client, addr = s.accept() 19 | print('SERVER: Got connection from', addr) 20 | send_handle(worker, client.fileno(), worker_pid) 21 | client.close() 22 | 23 | if __name__ == '__main__': 24 | import sys 25 | if len(sys.argv) != 3: 26 | print('Usage: server.py server_address port', file=sys.stderr) 27 | raise SystemExit(1) 28 | 29 | server(sys.argv[1], int(sys.argv[2])) 30 | -------------------------------------------------------------------------------- /src/11/passing_a_socket_file_descriptor_between_processes/worker.py: -------------------------------------------------------------------------------- 1 | # worker.py 2 | import socket 3 | import struct 4 | 5 | def recv_fd(sock): 6 | ''' 7 | Receive a single file descriptor 8 | ''' 9 | msg, ancdata, flags, addr = sock.recvmsg(1, 10 | socket.CMSG_LEN(struct.calcsize('i'))) 11 | 12 | cmsg_level, cmsg_type, cmsg_data = ancdata[0] 13 | assert cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS 14 | sock.sendall(b'OK') 15 | return struct.unpack('i', cmsg_data)[0] 16 | 17 | def worker(server_address): 18 | serv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 19 | serv.connect(server_address) 20 | while True: 21 | fd = recv_fd(serv) 22 | print('WORKER: GOT FD', fd) 23 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as client: 24 | while True: 25 | msg = client.recv(1024) 26 | if not msg: 27 | break 28 | print('WORKER: RECV {!r}'.format(msg)) 29 | client.send(msg) 30 | 31 | if __name__ == '__main__': 32 | import sys 33 | if len(sys.argv) != 2: 34 | print('Usage: worker.py server_address', file=sys.stderr) 35 | raise SystemExit(1) 36 | 37 | worker(sys.argv[1]) 38 | -------------------------------------------------------------------------------- /src/11/passing_a_socket_file_descriptor_between_processes/workermp.py: -------------------------------------------------------------------------------- 1 | # workermp.py 2 | 3 | from multiprocessing.connection import Client 4 | from multiprocessing.reduction import recv_handle 5 | import os 6 | import socket 7 | 8 | def worker(server_address): 9 | serv = Client(server_address, authkey=b'peekaboo') 10 | serv.send(os.getpid()) 11 | while True: 12 | fd = recv_handle(serv) 13 | print('WORKER: GOT FD', fd) 14 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as client: 15 | while True: 16 | msg = client.recv(1024) 17 | if not msg: 18 | break 19 | print('WORKER: RECV {!r}'.format(msg)) 20 | client.send(msg) 21 | 22 | if __name__ == '__main__': 23 | import sys 24 | if len(sys.argv) != 2: 25 | print('Usage: worker.py server_address', file=sys.stderr) 26 | raise SystemExit(1) 27 | 28 | worker(sys.argv[1]) 29 | -------------------------------------------------------------------------------- /src/11/simple_authentication_of_clients/auth.py: -------------------------------------------------------------------------------- 1 | # auth.py 2 | 3 | import hmac 4 | import os 5 | 6 | def client_authenticate(connection, secret_key): 7 | ''' 8 | Authenticate client to a remote service. 9 | connection represents a network connection. 10 | secret_key is a key known only to both client/server. 11 | ''' 12 | message = connection.recv(32) 13 | hash = hmac.new(secret_key, message) 14 | digest = hash.digest() 15 | connection.send(digest) 16 | 17 | def server_authenticate(connection, secret_key): 18 | ''' 19 | Request client authentication. 20 | ''' 21 | message = os.urandom(32) 22 | connection.send(message) 23 | hash = hmac.new(secret_key, message) 24 | digest = hash.digest() 25 | response = connection.recv(len(digest)) 26 | return hmac.compare_digest(digest,response) 27 | -------------------------------------------------------------------------------- /src/11/simple_authentication_of_clients/client.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | from auth import client_authenticate 3 | 4 | secret_key = b'peekaboo' 5 | 6 | s = socket(AF_INET, SOCK_STREAM) 7 | s.connect(('localhost', 18000)) 8 | client_authenticate(s, secret_key) 9 | s.send(b'Hello World') 10 | resp = s.recv(1024) 11 | print('Got:', resp) 12 | -------------------------------------------------------------------------------- /src/11/simple_authentication_of_clients/server.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | from auth import server_authenticate 3 | 4 | secret_key = b'peekaboo' 5 | 6 | def echo_handler(client_sock): 7 | if not server_authenticate(client_sock, secret_key): 8 | client_sock.close() 9 | return 10 | while True: 11 | msg = client_sock.recv(8192) 12 | if not msg: 13 | break 14 | client_sock.sendall(msg) 15 | 16 | def echo_server(address): 17 | s = socket(AF_INET, SOCK_STREAM) 18 | s.bind(address) 19 | s.listen(5) 20 | while True: 21 | c,a = s.accept() 22 | echo_handler(c) 23 | 24 | print('Echo server running on port 18000') 25 | 26 | echo_server(('', 18000)) 27 | -------------------------------------------------------------------------------- /src/11/simple_communication_between_interpreters/echoclient.py: -------------------------------------------------------------------------------- 1 | from multiprocessing.connection import Client 2 | c = Client(('localhost', 25000), authkey=b'peekaboo') 3 | c.send('hello') 4 | print('Got:', c.recv()) 5 | c.send(42) 6 | print('Got:', c.recv()) 7 | c.send([1, 2, 3, 4, 5]) 8 | print('Got:', c.recv()) 9 | 10 | -------------------------------------------------------------------------------- /src/11/simple_communication_between_interpreters/echoserv.py: -------------------------------------------------------------------------------- 1 | from multiprocessing.connection import Listener 2 | import traceback 3 | 4 | def echo_client(conn): 5 | try: 6 | while True: 7 | msg = conn.recv() 8 | conn.send(msg) 9 | except EOFError: 10 | print('Connection closed') 11 | 12 | def echo_server(address, authkey): 13 | serv = Listener(address, authkey=authkey) 14 | while True: 15 | try: 16 | client = serv.accept() 17 | echo_client(client) 18 | except Exception: 19 | traceback.print_exc() 20 | 21 | echo_server(('', 25000), authkey=b'peekaboo') 22 | -------------------------------------------------------------------------------- /src/11/simple_remote_procedure_call_with_xmlrpc/client.py: -------------------------------------------------------------------------------- 1 | from xmlrpc.client import ServerProxy 2 | s = ServerProxy('http://localhost:15000', allow_none=True) 3 | s.set('foo', 'bar') 4 | s.set('spam', [1, 2, 3]) 5 | print(s.keys()) 6 | print(s.get('foo')) 7 | print(s.get('spam')) 8 | s.delete('spam') 9 | print(s.exists('spam')) 10 | -------------------------------------------------------------------------------- /src/11/simple_remote_procedure_call_with_xmlrpc/keyserv.py: -------------------------------------------------------------------------------- 1 | from xmlrpc.server import SimpleXMLRPCServer 2 | 3 | class KeyValueServer: 4 | _rpc_methods_ = ['get', 'set', 'delete', 'exists', 'keys'] 5 | def __init__(self, address): 6 | self._data = {} 7 | self._serv = SimpleXMLRPCServer(address, allow_none=True) 8 | for name in self._rpc_methods_: 9 | self._serv.register_function(getattr(self, name)) 10 | 11 | def get(self, name): 12 | return self._data[name] 13 | 14 | def set(self, name, value): 15 | self._data[name] = value 16 | 17 | def delete(self, name): 18 | del self._data[name] 19 | 20 | def exists(self, name): 21 | return name in self._data 22 | 23 | def keys(self): 24 | return list(self._data) 25 | 26 | def serve_forever(self): 27 | self._serv.serve_forever() 28 | 29 | # Example 30 | if __name__ == '__main__': 31 | kvserv = KeyValueServer(('', 15000)) 32 | kvserv.serve_forever() 33 | -------------------------------------------------------------------------------- /src/11/zero_copy_sending_and_receiving_of_large_arrays/client.py: -------------------------------------------------------------------------------- 1 | from zerocopy import recv_into 2 | from socket import * 3 | 4 | c = socket(AF_INET, SOCK_STREAM) 5 | c.connect(('localhost', 25000)) 6 | 7 | import numpy 8 | a = numpy.zeros(shape=50000000, dtype=float) 9 | print(a[0:10]) 10 | recv_into(a, c) 11 | print(a[0:10]) 12 | print(a[-10:]) 13 | -------------------------------------------------------------------------------- /src/11/zero_copy_sending_and_receiving_of_large_arrays/server.py: -------------------------------------------------------------------------------- 1 | from zerocopy import send_from 2 | from socket import * 3 | 4 | s = socket(AF_INET, SOCK_STREAM) 5 | s.bind(('', 25000)) 6 | s.listen(1) 7 | c,a = s.accept() 8 | 9 | import numpy 10 | a = numpy.arange(0.0, 50000000.0) 11 | send_from(a, c) 12 | c.close() 13 | 14 | -------------------------------------------------------------------------------- /src/11/zero_copy_sending_and_receiving_of_large_arrays/zerocopy.py: -------------------------------------------------------------------------------- 1 | # zerocopy.py 2 | 3 | def send_from(arr, dest): 4 | view = memoryview(arr).cast('B') 5 | while len(view): 6 | nsent = dest.send(view) 7 | view = view[nsent:] 8 | 9 | def recv_into(arr, source): 10 | view = memoryview(arr).cast('B') 11 | while len(view): 12 | nrecv = source.recv_into(view) 13 | view = view[nrecv:] 14 | -------------------------------------------------------------------------------- /src/12/defining_an_actor_task/tagged.py: -------------------------------------------------------------------------------- 1 | from actor import Actor 2 | 3 | class TaggedActor(Actor): 4 | def run(self): 5 | while True: 6 | tag, *payload = self.recv() 7 | getattr(self,"do_"+tag)(*payload) 8 | 9 | # Methods correponding to different message tags 10 | def do_A(self, x): 11 | print("Running A", x) 12 | 13 | def do_B(self, x, y): 14 | print("Running B", x, y) 15 | 16 | # Example 17 | if __name__ == '__main__': 18 | a = TaggedActor() 19 | a.start() 20 | a.send(('A', 1)) # Invokes do_A(1) 21 | a.send(('B', 2, 3)) # Invokes do_B(2,3) 22 | a.close() 23 | a.join() 24 | 25 | -------------------------------------------------------------------------------- /src/12/defining_an_actor_task/worker.py: -------------------------------------------------------------------------------- 1 | from actor import Actor 2 | from threading import Event 3 | 4 | class Result: 5 | def __init__(self): 6 | self._evt = Event() 7 | self._result = None 8 | 9 | def set_result(self, value): 10 | self._result = value 11 | self._evt.set() 12 | 13 | def result(self): 14 | self._evt.wait() 15 | return self._result 16 | 17 | class Worker(Actor): 18 | def submit(self, func, *args, **kwargs): 19 | r = Result() 20 | self.send((func, args, kwargs, r)) 21 | return r 22 | 23 | def run(self): 24 | while True: 25 | func, args, kwargs, r = self.recv() 26 | r.set_result(func(*args, **kwargs)) 27 | 28 | # Example use 29 | if __name__ == '__main__': 30 | worker = Worker() 31 | worker.start() 32 | r = worker.submit(pow, 2, 3) 33 | print(r.result()) 34 | worker.close() 35 | worker.join() 36 | 37 | -------------------------------------------------------------------------------- /src/12/how_to_communicate_between_threads/example1.py: -------------------------------------------------------------------------------- 1 | from queue import Queue 2 | from threading import Thread 3 | import time 4 | 5 | _sentinel = object() 6 | 7 | # A thread that produces data 8 | def producer(out_q): 9 | n = 10 10 | while n > 0: 11 | # Produce some data 12 | out_q.put(n) 13 | time.sleep(2) 14 | n -= 1 15 | 16 | 17 | # Put the sentinel on the queue to indicate completion 18 | out_q.put(_sentinel) 19 | 20 | # A thread that consumes data 21 | def consumer(in_q): 22 | while True: 23 | # Get some data 24 | data = in_q.get() 25 | 26 | # Check for termination 27 | if data is _sentinel: 28 | in_q.put(_sentinel) 29 | break 30 | 31 | # Process the data 32 | print('Got:', data) 33 | print('Consumer shutting down') 34 | 35 | if __name__ == '__main__': 36 | q = Queue() 37 | t1 = Thread(target=consumer, args=(q,)) 38 | t2 = Thread(target=producer, args=(q,)) 39 | t1.start() 40 | t2.start() 41 | t1.join() 42 | t2.join() 43 | 44 | -------------------------------------------------------------------------------- /src/12/how_to_communicate_between_threads/example2.py: -------------------------------------------------------------------------------- 1 | import heapq 2 | import threading 3 | import time 4 | 5 | class PriorityQueue: 6 | def __init__(self): 7 | self._queue = [] 8 | self._count = 0 9 | self._cv = threading.Condition() 10 | def put(self, item, priority): 11 | with self._cv: 12 | heapq.heappush(self._queue, (-priority, self._count, item)) 13 | self._count += 1 14 | self._cv.notify() 15 | 16 | def get(self): 17 | with self._cv: 18 | while len(self._queue) == 0: 19 | self._cv.wait() 20 | return heapq.heappop(self._queue)[-1] 21 | 22 | def producer(q): 23 | print('Producing items') 24 | q.put('C', 5) 25 | q.put('A', 15) 26 | q.put('B', 10) 27 | q.put('D', 0) 28 | q.put(None, -100) 29 | 30 | def consumer(q): 31 | time.sleep(5) 32 | print('Getting items') 33 | while True: 34 | item = q.get() 35 | if item is None: 36 | break 37 | print('Got:', item) 38 | print('Consumer done') 39 | 40 | if __name__ == '__main__': 41 | q = PriorityQueue() 42 | t1 = threading.Thread(target=producer, args=(q,)) 43 | t2 = threading.Thread(target=consumer, args=(q,)) 44 | t1.start() 45 | t2.start() 46 | t1.join() 47 | t2.join() 48 | 49 | 50 | -------------------------------------------------------------------------------- /src/12/how_to_create_a_thread_pool/example1.py: -------------------------------------------------------------------------------- 1 | from socket import AF_INET, SOCK_STREAM, socket 2 | from concurrent.futures import ThreadPoolExecutor 3 | 4 | def echo_client(sock, client_addr): 5 | ''' 6 | Handle a client connection 7 | ''' 8 | print('Got connection from', client_addr) 9 | while True: 10 | msg = sock.recv(65536) 11 | if not msg: 12 | break 13 | sock.sendall(msg) 14 | print('Client closed connection') 15 | sock.close() 16 | 17 | def echo_server(addr): 18 | print('Echo server running at', addr) 19 | pool = ThreadPoolExecutor(128) 20 | sock = socket(AF_INET, SOCK_STREAM) 21 | sock.bind(addr) 22 | sock.listen(5) 23 | while True: 24 | client_sock, client_addr = sock.accept() 25 | pool.submit(echo_client, client_sock, client_addr) 26 | 27 | echo_server(('',15000)) 28 | -------------------------------------------------------------------------------- /src/12/how_to_create_a_thread_pool/example2.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | from threading import Thread 3 | from queue import Queue 4 | 5 | def echo_client(q): 6 | ''' 7 | Handle a client connection 8 | ''' 9 | sock, client_addr = q.get() 10 | print('Got connection from', client_addr) 11 | while True: 12 | msg = sock.recv(65536) 13 | if not msg: 14 | break 15 | sock.sendall(msg) 16 | print('Client closed connection') 17 | sock.close() 18 | 19 | def echo_server(addr, nworkers): 20 | print('Echo server running at', addr) 21 | # Launch the client workers 22 | q = Queue() 23 | for n in range(nworkers): 24 | t = Thread(target=echo_client, args=(q,)) 25 | t.daemon = True 26 | t.start() 27 | 28 | # Run the server 29 | sock = socket(AF_INET, SOCK_STREAM) 30 | sock.bind(addr) 31 | sock.listen(5) 32 | while True: 33 | client_sock, client_addr = sock.accept() 34 | q.put((client_sock, client_addr)) 35 | 36 | echo_server(('',15000), 128) 37 | -------------------------------------------------------------------------------- /src/12/how_to_create_a_thread_pool/example3.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures import ThreadPoolExecutor 2 | import urllib.request 3 | 4 | def fetch_url(url): 5 | u = urllib.request.urlopen(url) 6 | data = u.read() 7 | return data 8 | 9 | pool = ThreadPoolExecutor(10) 10 | # Submit work to the pool 11 | a = pool.submit(fetch_url, 'http://www.python.org') 12 | b = pool.submit(fetch_url, 'http://www.pypy.org') 13 | 14 | # Get the results back 15 | x = a.result() 16 | y = b.result() 17 | -------------------------------------------------------------------------------- /src/12/how_to_determine_if_a_thread_has_started/example1.py: -------------------------------------------------------------------------------- 1 | from threading import Thread, Event 2 | import time 3 | 4 | # Code to execute in an independent thread 5 | def countdown(n, started_evt): 6 | print("countdown starting") 7 | started_evt.set() 8 | while n > 0: 9 | print("T-minus", n) 10 | n -= 1 11 | time.sleep(5) 12 | 13 | # Create the event object that will be used to signal startup 14 | started_evt = Event() 15 | 16 | # Launch the thread and pass the startup event 17 | print("Launching countdown") 18 | t = Thread(target=countdown, args=(10,started_evt)) 19 | t.start() 20 | 21 | # Wait for the thread to start 22 | started_evt.wait() 23 | print("countdown is running") 24 | -------------------------------------------------------------------------------- /src/12/how_to_determine_if_a_thread_has_started/example3.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | 4 | # Worker thread 5 | def worker(n, sema): 6 | # Wait to be signalled 7 | sema.acquire() 8 | # Do some work 9 | print("Working", n) 10 | 11 | # Create some threads 12 | sema = threading.Semaphore(0) 13 | nworkers = 10 14 | for n in range(nworkers): 15 | t = threading.Thread(target=worker, args=(n, sema,)) 16 | t.daemon=True 17 | t.start() 18 | 19 | print('About to release first worker') 20 | time.sleep(5) 21 | sema.release() 22 | time.sleep(1) 23 | print('About to release second worker') 24 | time.sleep(5) 25 | sema.release() 26 | time.sleep(1) 27 | print('Goodbye') 28 | -------------------------------------------------------------------------------- /src/12/how_to_lock_critical_sections/example1.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | class SharedCounter: 4 | ''' 5 | A counter object that can be shared by multiple threads. 6 | ''' 7 | def __init__(self, initial_value = 0): 8 | self._value = initial_value 9 | self._value_lock = threading.Lock() 10 | 11 | def incr(self,delta=1): 12 | ''' 13 | Increment the counter with locking 14 | ''' 15 | with self._value_lock: 16 | self._value += delta 17 | 18 | def decr(self,delta=1): 19 | ''' 20 | Decrement the counter with locking 21 | ''' 22 | with self._value_lock: 23 | self._value -= delta 24 | 25 | def test(c): 26 | for n in range(1000000): 27 | c.incr() 28 | for n in range(1000000): 29 | c.decr() 30 | 31 | if __name__ == '__main__': 32 | c = SharedCounter() 33 | t1 = threading.Thread(target=test, args=(c,)) 34 | t2 = threading.Thread(target=test, args=(c,)) 35 | t3 = threading.Thread(target=test, args=(c,)) 36 | t1.start() 37 | t2.start() 38 | t3.start() 39 | print('Running test') 40 | t1.join() 41 | t2.join() 42 | t3.join() 43 | 44 | assert c._value == 0 45 | print('Looks good!') 46 | -------------------------------------------------------------------------------- /src/12/how_to_start_and_stop_threads/example.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | import time 3 | 4 | class CountdownTask: 5 | def __init__(self): 6 | self._running = True 7 | 8 | def terminate(self): 9 | self._running = False 10 | 11 | def run(self, n): 12 | while self._running and n > 0: 13 | print("T-minus", n) 14 | n -= 1 15 | time.sleep(5) 16 | 17 | c = CountdownTask() 18 | t = Thread(target=c.run, args=(10,)) 19 | t.start() 20 | 21 | time.sleep(20) 22 | print('About to terminate') 23 | c.terminate() 24 | t.join() 25 | print('Terminated') 26 | 27 | -------------------------------------------------------------------------------- /src/12/implementing_publish_subscribe_messaging/exchange1.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | class Exchange: 4 | def __init__(self): 5 | self._subscribers = set() 6 | 7 | def attach(self, task): 8 | self._subscribers.add(task) 9 | 10 | def detach(self, task): 11 | self._subscribers.remove(task) 12 | 13 | def send(self, msg): 14 | for subscriber in self._subscribers: 15 | subscriber.send(msg) 16 | 17 | # Dictionary of all created exchanges 18 | _exchanges = defaultdict(Exchange) 19 | 20 | # Return the Exchange instance associated with a given name 21 | def get_exchange(name): 22 | return _exchanges[name] 23 | 24 | if __name__ == '__main__': 25 | # Example task (just for testing) 26 | class Task: 27 | def __init__(self, name): 28 | self.name = name 29 | def send(self, msg): 30 | print('{} got: {!r}'.format(self.name, msg)) 31 | 32 | task_a = Task('A') 33 | task_b = Task('B') 34 | 35 | exc = get_exchange('spam') 36 | exc.attach(task_a) 37 | exc.attach(task_b) 38 | exc.send('msg1') 39 | exc.send('msg2') 40 | 41 | exc.detach(task_a) 42 | exc.detach(task_b) 43 | exc.send('msg3') 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/12/locking_with_deadlock_avoidance/deadlock.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from contextlib import contextmanager 3 | 4 | # Thread-local state to stored information on locks already acquired 5 | _local = threading.local() 6 | 7 | @contextmanager 8 | def acquire(*locks): 9 | # Sort locks by object identifier 10 | locks = sorted(locks, key=lambda x: id(x)) 11 | 12 | # Make sure lock order of previously acquired locks is not violated 13 | acquired = getattr(_local, 'acquired',[]) 14 | if acquired and max(id(lock) for lock in acquired) >= id(locks[0]): 15 | raise RuntimeError('Lock Order Violation') 16 | 17 | # Acquire all of the locks 18 | acquired.extend(locks) 19 | _local.acquired = acquired 20 | try: 21 | for lock in locks: 22 | lock.acquire() 23 | yield 24 | finally: 25 | # Release locks in reverse order of acquisition 26 | for lock in reversed(locks): 27 | lock.release() 28 | del acquired[-len(locks):] 29 | -------------------------------------------------------------------------------- /src/12/locking_with_deadlock_avoidance/example1.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from deadlock import acquire 3 | 4 | x_lock = threading.Lock() 5 | y_lock = threading.Lock() 6 | 7 | def thread_1(): 8 | while True: 9 | with acquire(x_lock, y_lock): 10 | print("Thread-1") 11 | 12 | def thread_2(): 13 | while True: 14 | with acquire(y_lock, x_lock): 15 | print("Thread-2") 16 | 17 | input('This program runs forever. Press [return] to start, Ctrl-C to exit') 18 | 19 | t1 = threading.Thread(target=thread_1) 20 | t1.daemon = True 21 | t1.start() 22 | 23 | t2 = threading.Thread(target=thread_2) 24 | t2.daemon = True 25 | t2.start() 26 | 27 | import time 28 | while True: 29 | time.sleep(1) 30 | -------------------------------------------------------------------------------- /src/12/locking_with_deadlock_avoidance/example2.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | from deadlock import acquire 4 | 5 | 6 | x_lock = threading.Lock() 7 | y_lock = threading.Lock() 8 | 9 | def thread_1(): 10 | while True: 11 | with acquire(x_lock): 12 | with acquire(y_lock): 13 | print("Thread-1") 14 | time.sleep(1) 15 | 16 | def thread_2(): 17 | while True: 18 | with acquire(y_lock): 19 | with acquire(x_lock): 20 | print("Thread-2") 21 | time.sleep(1) 22 | 23 | input('This program crashes with an exception. Press [return] to start') 24 | 25 | t1 = threading.Thread(target=thread_1) 26 | t1.daemon = True 27 | t1.start() 28 | 29 | t2 = threading.Thread(target=thread_2) 30 | t2.daemon = True 31 | t2.start() 32 | 33 | time.sleep(5) 34 | 35 | -------------------------------------------------------------------------------- /src/12/locking_with_deadlock_avoidance/example3.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from deadlock import acquire 3 | 4 | # The philosopher thread 5 | def philosopher(left, right): 6 | while True: 7 | with acquire(left,right): 8 | print(threading.currentThread(), 'eating') 9 | 10 | # The chopsticks (represented by locks) 11 | NSTICKS = 5 12 | chopsticks = [threading.Lock() for n in range(NSTICKS)] 13 | 14 | # Create all of the philosophers 15 | for n in range(NSTICKS): 16 | t = threading.Thread(target=philosopher, 17 | args=(chopsticks[n],chopsticks[(n+1) % NSTICKS])) 18 | t.daemon = True 19 | t.start() 20 | 21 | import time 22 | while True: 23 | time.sleep(1) 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/findrobots.py: -------------------------------------------------------------------------------- 1 | # findrobots.py 2 | 3 | import gzip 4 | import io 5 | import glob 6 | 7 | def find_robots(filename): 8 | ''' 9 | Find all of the hosts that access robots.txt in a single log file 10 | ''' 11 | robots = set() 12 | with gzip.open(filename) as f: 13 | for line in io.TextIOWrapper(f,encoding='ascii'): 14 | fields = line.split() 15 | if fields[6] == '/robots.txt': 16 | robots.add(fields[0]) 17 | return robots 18 | 19 | def find_all_robots(logdir): 20 | ''' 21 | Find all hosts across and entire sequence of files 22 | ''' 23 | files = glob.glob(logdir+"/*.log.gz") 24 | all_robots = set() 25 | for robots in map(find_robots, files): 26 | all_robots.update(robots) 27 | return all_robots 28 | 29 | if __name__ == '__main__': 30 | import time 31 | start = time.time() 32 | robots = find_all_robots("logs") 33 | end = time.time() 34 | for ipaddr in robots: 35 | print(ipaddr) 36 | print('Took {:f} seconds'.format(end-start)) 37 | 38 | -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/findrobots_par.py: -------------------------------------------------------------------------------- 1 | # findrobots.py 2 | 3 | import gzip 4 | import io 5 | import glob 6 | from concurrent import futures 7 | 8 | def find_robots(filename): 9 | ''' 10 | Find all of the hosts that access robots.txt in a single log file 11 | ''' 12 | robots = set() 13 | with gzip.open(filename) as f: 14 | for line in io.TextIOWrapper(f,encoding='ascii'): 15 | fields = line.split() 16 | if fields[6] == '/robots.txt': 17 | robots.add(fields[0]) 18 | return robots 19 | 20 | def find_all_robots(logdir): 21 | ''' 22 | Find all hosts across and entire sequence of files 23 | ''' 24 | files = glob.glob(logdir+"/*.log.gz") 25 | all_robots = set() 26 | with futures.ProcessPoolExecutor() as pool: 27 | for robots in pool.map(find_robots, files): 28 | all_robots.update(robots) 29 | return all_robots 30 | 31 | if __name__ == '__main__': 32 | import time 33 | start = time.time() 34 | robots = find_all_robots("logs") 35 | end = time.time() 36 | for ipaddr in robots: 37 | print(ipaddr) 38 | print('Took {:f} seconds'.format(end-start)) 39 | -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121217.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121217.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121218.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121218.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121219.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121219.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121220.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121220.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121221.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121221.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121222.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121222.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121223.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121223.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121224.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121224.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121225.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121225.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121226.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121226.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121227.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121227.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121228.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121228.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121229.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121229.log.gz -------------------------------------------------------------------------------- /src/12/simple_parallel_programming/logs/20121230.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/12/simple_parallel_programming/logs/20121230.log.gz -------------------------------------------------------------------------------- /src/12/using_generators_as_an_alternative_to_threads/simple.py: -------------------------------------------------------------------------------- 1 | # A very simple example of a coroutine/generator scheduler 2 | 3 | # Two simple generator functions 4 | def countdown(n): 5 | while n > 0: 6 | print("T-minus", n) 7 | yield 8 | n -= 1 9 | print("Blastoff!") 10 | 11 | def countup(n): 12 | x = 0 13 | while x < n: 14 | print("Counting up", x) 15 | yield 16 | x += 1 17 | 18 | from collections import deque 19 | 20 | class TaskScheduler: 21 | def __init__(self): 22 | self._task_queue = deque() 23 | 24 | def new_task(self, task): 25 | ''' 26 | Admit a newly started task to the scheduler 27 | ''' 28 | self._task_queue.append(task) 29 | 30 | def run(self): 31 | ''' 32 | Run until there are no more tasks 33 | ''' 34 | while self._task_queue: 35 | task = self._task_queue.popleft() 36 | try: 37 | # Run until the next yield statement 38 | next(task) 39 | self._task_queue.append(task) 40 | except StopIteration: 41 | # Generator is no longer executing 42 | pass 43 | 44 | # Example use 45 | sched = TaskScheduler() 46 | sched.new_task(countdown(10)) 47 | sched.new_task(countdown(5)) 48 | sched.new_task(countup(15)) 49 | sched.run() 50 | -------------------------------------------------------------------------------- /src/13/adding_logging_to_libraries/somelib.py: -------------------------------------------------------------------------------- 1 | # somelib.py 2 | 3 | import logging 4 | log = logging.getLogger(__name__) 5 | log.addHandler(logging.NullHandler()) 6 | 7 | # Example function (for testing) 8 | def func(): 9 | log.critical("A Critical Error!") 10 | log.debug("A debug message") 11 | -------------------------------------------------------------------------------- /src/13/executing_an_external_command_and_getting_its_output/example1.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | try: 3 | out_bytes = subprocess.check_output(['netstat', '-a']) 4 | out_text = out_bytes.decode('utf-8') 5 | print(out_text) 6 | except subprocess.CalledProcessError as e: 7 | print('It did not work. Reason:', e) 8 | print('Exitcode:', e.returncode) 9 | 10 | -------------------------------------------------------------------------------- /src/13/executing_an_external_command_and_getting_its_output/example2.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | # Some text to send 4 | text = b''' 5 | hello world 6 | this is a test 7 | goodbye 8 | ''' 9 | 10 | # Launch a command with pipes 11 | p = subprocess.Popen(['wc'], 12 | stdout = subprocess.PIPE, 13 | stdin = subprocess.PIPE) 14 | 15 | # Send the data and get the output 16 | stdout, stderr = p.communicate(text) 17 | 18 | text = stdout.decode('utf-8') 19 | print(text) 20 | 21 | -------------------------------------------------------------------------------- /src/13/finding_files/modified_within.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.3 2 | 3 | import os 4 | import time 5 | 6 | def modified_within(top, seconds): 7 | now = time.time() 8 | for path, dirs, files in os.walk(top): 9 | for name in files: 10 | fullpath = os.path.join(path, name) 11 | if os.path.exists(fullpath): 12 | mtime = os.path.getmtime(fullpath) 13 | if mtime > (now - seconds): 14 | print(fullpath) 15 | 16 | 17 | if __name__ == '__main__': 18 | import sys 19 | if len(sys.argv) != 3: 20 | print('Usage: {} dir seconds'.format(sys.argv[0])) 21 | raise SystemExit(1) 22 | 23 | modified_within(sys.argv[1], float(sys.argv[2])) 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/13/generating_a_range_of_ip_addresses_from_a_cidr_address/example.py: -------------------------------------------------------------------------------- 1 | from socket import AF_INET, AF_INET6, inet_pton, inet_ntop 2 | 3 | def cidr_range(cidr_address): 4 | family = AF_INET6 if ':' in cidr_address else AF_INET 5 | address, maskstr = cidr_address.split('/') 6 | maskbits = int(maskstr) 7 | 8 | # Parse the supplied address into bytes 9 | addr_bytes = inet_pton(family, address) 10 | 11 | # Calculate number of address bytes and mask bits 12 | addr_len = len(addr_bytes) 13 | numaddrs = 2**(addr_len*8 - maskbits) 14 | mask = -numaddrs 15 | 16 | # Generate addresses 17 | addr = int.from_bytes(addr_bytes, 'big') & mask 18 | for n in range(numaddrs): 19 | yield inet_ntop(family, (addr+n).to_bytes(addr_len, 'big')) 20 | 21 | if __name__ == '__main__': 22 | for a in cidr_range('123.45.67.89/27'): 23 | print(a) 24 | 25 | for a in cidr_range('12:3456:78:90ab:cd:ef01:23:34/125'): 26 | print(a) 27 | -------------------------------------------------------------------------------- /src/13/getting_the_terminal_size/example.py: -------------------------------------------------------------------------------- 1 | import os 2 | sz = os.get_terminal_size() 3 | print(sz.columns, 'columns') 4 | print(sz.lines, 'lines') 5 | -------------------------------------------------------------------------------- /src/13/making_a_stopwatch/stopwatch.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | class Timer: 4 | def __init__(self, func=time.perf_counter): 5 | self.elapsed = 0.0 6 | self._func = func 7 | self._start = None 8 | 9 | def start(self): 10 | if self._start is not None: 11 | raise RuntimeError('Already started') 12 | self._start = self._func() 13 | 14 | def end(self): 15 | if self._start is None: 16 | raise RuntimeError('Not started') 17 | end = self._func() 18 | self.elapsed += end - self._start 19 | self._start = None 20 | 21 | def reset(self): 22 | self.elapsed = 0.0 23 | 24 | @property 25 | def running(self): 26 | return self._start is not None 27 | 28 | def __enter__(self): 29 | self.start() 30 | return self 31 | 32 | def __exit__(self, *args): 33 | self.end() 34 | 35 | 36 | if __name__ == '__main__': 37 | def countdown(n): 38 | while n > 0: 39 | n -= 1 40 | 41 | t = Timer() 42 | t.start() 43 | countdown(1000000) 44 | t.end() 45 | print(t.elapsed) 46 | 47 | with t: 48 | countdown(1000000) 49 | print(t.elapsed) 50 | 51 | -------------------------------------------------------------------------------- /src/13/parsing_command_line_options/search.py: -------------------------------------------------------------------------------- 1 | # search.py 2 | ''' 3 | Hypothetical command line tool for searching a collection of 4 | files for one or more text patterns. 5 | ''' 6 | import argparse 7 | parser = argparse.ArgumentParser(description='Search some files') 8 | 9 | parser.add_argument(dest='filenames',metavar='filename', nargs='*') 10 | 11 | parser.add_argument('-p', '--pat',metavar='pattern', required=True, 12 | dest='patterns', action='append', 13 | help='text pattern to search for') 14 | 15 | parser.add_argument('-v', dest='verbose', action='store_true', 16 | help='verbose mode') 17 | 18 | parser.add_argument('-o', dest='outfile', action='store', 19 | help='output file') 20 | 21 | parser.add_argument('--speed', dest='speed', action='store', 22 | choices={'slow','fast'}, default='slow', 23 | help='search speed') 24 | 25 | args = parser.parse_args() 26 | 27 | # Output the collected arguments 28 | print(args.filenames) 29 | print(args.patterns) 30 | print(args.verbose) 31 | print(args.outfile) 32 | print(args.speed) 33 | -------------------------------------------------------------------------------- /src/13/prompting_for_a_password_at_runtime/example.py: -------------------------------------------------------------------------------- 1 | import getpass 2 | 3 | user = getpass.getuser() 4 | passwd = getpass.getpass() 5 | 6 | print('User:', user) 7 | print('Passwd:', passwd) 8 | -------------------------------------------------------------------------------- /src/13/putting_limits_on_memory_and_cpu_usage/example.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import resource 3 | import os 4 | 5 | def time_exceeded(signo, frame): 6 | print("Time's up!") 7 | raise SystemExit(1) 8 | 9 | def set_max_runtime(seconds): 10 | # Install the signal handler and set a resource limit 11 | soft, hard = resource.getrlimit(resource.RLIMIT_CPU) 12 | resource.setrlimit(resource.RLIMIT_CPU, (seconds, hard)) 13 | signal.signal(signal.SIGXCPU, time_exceeded) 14 | 15 | if __name__ == '__main__': 16 | set_max_runtime(15) 17 | while True: 18 | pass 19 | -------------------------------------------------------------------------------- /src/13/reading_configuration_files/config.ini: -------------------------------------------------------------------------------- 1 | 2 | ; config.ini 3 | ; Sample configuration file 4 | 5 | [installation] 6 | library=%(prefix)s/lib 7 | include=%(prefix)s/include 8 | bin=%(prefix)s/bin 9 | prefix=/usr/local 10 | 11 | # Setting related to debug configuration 12 | [debug] 13 | log_errors=true 14 | show_warnings=False 15 | 16 | [server] 17 | port: 8080 18 | nworkers: 32 19 | pid-file=/tmp/spam.pid 20 | root=/www/root 21 | signature: 22 | ================================= 23 | Brought to by the Python Cookbook 24 | ================================= 25 | -------------------------------------------------------------------------------- /src/13/reading_configuration_files/example1.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser 2 | cfg = ConfigParser() 3 | cfg.read('config.ini') 4 | print('sections:', cfg.sections()) 5 | print('installation:library', cfg.get('installation','library')) 6 | print('debug:log_errors', cfg.getboolean('debug','log_errors')) 7 | print('server:port', cfg.getint('server','port')) 8 | print('server:nworkers', cfg.getint('server','nworkers')) 9 | print('server:signature', cfg.get('server','signature')) 10 | -------------------------------------------------------------------------------- /src/13/simple_logging_for_scripts/example1.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | def main(): 4 | # Configure the logging system 5 | logging.basicConfig( 6 | filename='app.log', 7 | level=logging.ERROR 8 | ) 9 | 10 | # Variables (to make the calls that follow work) 11 | hostname = 'www.python.org' 12 | item = 'spam' 13 | filename = 'data.csv' 14 | mode = 'r' 15 | 16 | # Example logging calls (insert into your program) 17 | logging.critical('Host %s unknown', hostname) 18 | logging.error("Couldn't find %r", item) 19 | logging.warning('Feature is deprecated') 20 | logging.info('Opening file %r, mode=%r', filename, mode) 21 | logging.debug('Got here') 22 | 23 | if __name__ == '__main__': 24 | main() 25 | -------------------------------------------------------------------------------- /src/13/simple_logging_for_scripts/example2.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging.config 3 | 4 | def main(): 5 | # Configure the logging system 6 | logging.config.fileConfig('logconfig.ini') 7 | 8 | # Variables (to make the calls that follow work) 9 | hostname = 'www.python.org' 10 | item = 'spam' 11 | filename = 'data.csv' 12 | mode = 'r' 13 | 14 | # Example logging calls (insert into your program) 15 | logging.critical('Host %s unknown', hostname) 16 | logging.error("Couldn't find %r", item) 17 | logging.warning('Feature is deprecated') 18 | logging.info('Opening file %r, mode=%r', filename, mode) 19 | logging.debug('Got here') 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /src/13/simple_logging_for_scripts/logconfig.ini: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=defaultHandler 6 | 7 | [formatters] 8 | keys=defaultFormatter 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=defaultHandler 13 | qualname=root 14 | 15 | [handler_defaultHandler] 16 | class=FileHandler 17 | formatter=defaultFormatter 18 | args=('app.log', 'a') 19 | 20 | [formatter_defaultFormatter] 21 | format=%(levelname)s:%(name)s:%(message)s 22 | -------------------------------------------------------------------------------- /src/14/logging_test_output_to_a_file/test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | # A simple function to illustrate 4 | def parse_int(s): 5 | return int(s) 6 | 7 | class TestConversion(unittest.TestCase): 8 | # Testing that an exception gets raised 9 | def test_bad_int(self): 10 | self.assertRaises(ValueError, parse_int, "N/A") 11 | 12 | # Testing an exception plus regex on exception message 13 | def test_bad_int_msg(self): 14 | self.assertRaisesRegex(ValueError, 'invalid literal .*', parse_int, 'N/A') 15 | 16 | # Example of testing an exception along with inspection of exception instance 17 | import errno 18 | 19 | class TestIO(unittest.TestCase): 20 | def test_file_not_found(self): 21 | try: 22 | f = open('/file/not/found') 23 | except IOError as e: 24 | self.assertEqual(e.errno, errno.ENOENT) 25 | else: 26 | self.fail("IOError not raised") 27 | 28 | import sys 29 | def main(out=sys.stderr, verbosity=2): 30 | loader = unittest.TestLoader() 31 | suite = loader.loadTestsFromModule(sys.modules[__name__]) 32 | unittest.TextTestRunner(out, verbosity=verbosity).run(suite) 33 | 34 | if __name__ == '__main__': 35 | with open('testing.out', 'w') as f: 36 | main(f) 37 | -------------------------------------------------------------------------------- /src/14/make_your_programs_run_faster/example.py: -------------------------------------------------------------------------------- 1 | import time 2 | def test(func): 3 | start = time.time() 4 | nums = range(1000000) 5 | for n in range(100): 6 | r = func(nums) 7 | end = time.time() 8 | print(func.__name__, ':', end-start) 9 | 10 | import math 11 | def compute_roots_1(nums): 12 | result = [] 13 | for n in nums: 14 | result.append(math.sqrt(n)) 15 | return result 16 | 17 | from math import sqrt 18 | def compute_roots_2(nums): 19 | result = [] 20 | result_append = result.append 21 | for n in nums: 22 | result_append(sqrt(n)) 23 | return result 24 | 25 | def compute_roots_3(nums): 26 | sqrt = math.sqrt 27 | result = [] 28 | result_append = result.append 29 | for n in nums: 30 | result_append(sqrt(n)) 31 | return result 32 | 33 | tests = [compute_roots_1, compute_roots_2, compute_roots_3] 34 | for func in tests: 35 | test(func) 36 | -------------------------------------------------------------------------------- /src/14/profiling_and_timing_your_program/timethis.py: -------------------------------------------------------------------------------- 1 | # timethis.py 2 | 3 | import time 4 | from functools import wraps 5 | 6 | def timethis(func): 7 | @wraps(func) 8 | def wrapper(*args, **kwargs): 9 | start = time.perf_counter() 10 | r = func(*args, **kwargs) 11 | end = time.perf_counter() 12 | print('{}.{} : {}'.format(func.__module__, func.__name__, end-start)) 13 | return r 14 | return wrapper 15 | 16 | if __name__ == '__main__': 17 | @timethis 18 | def countdown(n): 19 | while n > 0: 20 | n -= 1 21 | 22 | 23 | countdown(10000000) 24 | -------------------------------------------------------------------------------- /src/14/skipping_or_anticipating_test_failures/test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import platform 4 | 5 | class Tests(unittest.TestCase): 6 | def test_0(self): 7 | self.assertTrue(True) 8 | 9 | @unittest.skip('skipped test') 10 | def test_1(self): 11 | self.fail("should have failed!") 12 | 13 | @unittest.skipIf(os.name=='posix', 'Not supported on Unix') 14 | def test_2(self): 15 | import winreg 16 | 17 | @unittest.skipUnless(platform.system() == 'Darwin', 'Mac specific test') 18 | def test_3(self): 19 | self.assertTrue(True) 20 | 21 | @unittest.expectedFailure 22 | def test_4(self): 23 | self.assertEqual(2+2, 5) 24 | 25 | if __name__ == '__main__': 26 | unittest.main(verbosity=2) 27 | -------------------------------------------------------------------------------- /src/14/testing_for_exceptional_conditions_in_unit_tests/test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | # A simple function to illustrate 4 | def parse_int(s): 5 | return int(s) 6 | 7 | class TestConversion(unittest.TestCase): 8 | # Testing that an exception gets raised 9 | def test_bad_int(self): 10 | self.assertRaises(ValueError, parse_int, "N/A") 11 | 12 | # Testing an exception plus regex on exception message 13 | def test_bad_int_msg(self): 14 | self.assertRaisesRegex(ValueError, 'invalid literal .*', parse_int, 'N/A') 15 | 16 | # Example of testing an exception along with inspection of exception instance 17 | import errno 18 | 19 | class TestIO(unittest.TestCase): 20 | def test_file_not_found(self): 21 | try: 22 | f = open('/file/not/found') 23 | except IOError as e: 24 | self.assertEqual(e.errno, errno.ENOENT) 25 | else: 26 | self.fail("IOError not raised") 27 | 28 | if __name__ == '__main__': 29 | unittest.main() 30 | -------------------------------------------------------------------------------- /src/14/testing_output_sent_to_stdout/mymodule.py: -------------------------------------------------------------------------------- 1 | # mymodule.py 2 | 3 | def urlprint(protocol, host, domain): 4 | url = '{}://{}.{}'.format(protocol, host, domain) 5 | print(url) 6 | -------------------------------------------------------------------------------- /src/14/testing_output_sent_to_stdout/testmymodule.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | from unittest import TestCase 3 | from unittest.mock import patch 4 | import mymodule 5 | 6 | class TestURLPrint(TestCase): 7 | def test_url_gets_to_stdout(self): 8 | protocol = 'http' 9 | host = 'www' 10 | domain = 'example.com' 11 | expected_url = '{}://{}.{}\n'.format(protocol, host, domain) 12 | 13 | with patch('sys.stdout', new=StringIO()) as fake_out: 14 | mymodule.urlprint(protocol, host, domain) 15 | self.assertEqual(fake_out.getvalue(), expected_url) 16 | 17 | if __name__ == '__main__': 18 | import unittest 19 | unittest.main() 20 | -------------------------------------------------------------------------------- /src/15/Makefile: -------------------------------------------------------------------------------- 1 | osx:: 2 | gcc -shared -undefined dynamic_lookup sample.c -o libsample.so 3 | 4 | linux:: 5 | gcc -shared -fPIC sample.c -o libsample.so 6 | -------------------------------------------------------------------------------- /src/15/accessing_c_code_using_ctypes/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | print(sample.gcd(35,42)) 3 | print(sample.in_mandel(0,0,500)) 4 | print(sample.in_mandel(2.0,1.0,500)) 5 | print(sample.divide(42,8)) 6 | print(sample.avg([1,2,3])) 7 | p1 = sample.Point(1,2) 8 | p2 = sample.Point(4,5) 9 | print(sample.distance(p1,p2)) 10 | -------------------------------------------------------------------------------- /src/15/calling_python_from_c/Makefile: -------------------------------------------------------------------------------- 1 | all:: 2 | cc -g embed.c -I/usr/local/include/python3.3m \ 3 | -L/usr/local/lib/python3.3/config-3.3m -lpython3.3m 4 | -------------------------------------------------------------------------------- /src/15/consuming_an_iterable_from_c/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | 3 | sample.consume_iterable([1,2,3,4]) 4 | 5 | def countdown(n): 6 | while n > 0: 7 | yield n 8 | n -= 1 9 | 10 | sample.consume_iterable(countdown(10)) 11 | -------------------------------------------------------------------------------- /src/15/consuming_an_iterable_from_c/sample.c: -------------------------------------------------------------------------------- 1 | #include "Python.h" 2 | 3 | static PyObject *py_consume_iterable(PyObject *self, PyObject *args) { 4 | PyObject *obj; 5 | PyObject *iter; 6 | PyObject *item; 7 | 8 | if (!PyArg_ParseTuple(args, "O", &obj)) { 9 | return NULL; 10 | } 11 | if ((iter = PyObject_GetIter(obj)) == NULL) { 12 | return NULL; 13 | } 14 | while ((item = PyIter_Next(iter)) != NULL) { 15 | /* Use item */ 16 | PyObject_Print(item, stdout, 0); 17 | printf("\n"); 18 | Py_DECREF(item); 19 | } 20 | Py_DECREF(iter); 21 | return Py_BuildValue(""); 22 | } 23 | 24 | /* Module method table */ 25 | static PyMethodDef SampleMethods[] = { 26 | {"consume_iterable", py_consume_iterable, METH_VARARGS}, 27 | { NULL, NULL, 0, NULL} 28 | }; 29 | 30 | /* Module structure */ 31 | static struct PyModuleDef samplemodule = { 32 | PyModuleDef_HEAD_INIT, 33 | "sample", /* name of module */ 34 | "A sample module", /* Doc string (may be NULL) */ 35 | -1, /* Size of per-interpreter state or -1 */ 36 | SampleMethods /* Method table */ 37 | }; 38 | 39 | /* Module initialization function */ 40 | PyMODINIT_FUNC 41 | PyInit_sample(void) { 42 | return PyModule_Create(&samplemodule); 43 | } 44 | -------------------------------------------------------------------------------- /src/15/consuming_an_iterable_from_c/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["sample.c"], 8 | ) 9 | ] 10 | ) 11 | -------------------------------------------------------------------------------- /src/15/defining_and_exporting_c_apis_from_extension_modules/README.txt: -------------------------------------------------------------------------------- 1 | To build, you need to perform these two steps: 2 | 3 | % python3 setup.py build_ext --inplace 4 | % python3 ptsetup.py build_ext --inplace 5 | -------------------------------------------------------------------------------- /src/15/defining_and_exporting_c_apis_from_extension_modules/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | import ptexample 3 | p1 = sample.Point(2,3) 4 | ptexample.print_point(p1) 5 | -------------------------------------------------------------------------------- /src/15/defining_and_exporting_c_apis_from_extension_modules/ptsetup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="ptexample", 5 | ext_modules=[ 6 | Extension("ptexample", 7 | ["ptexample.c"], 8 | include_dirs = ['..','.'], # May need pysample.h directory 9 | ) 10 | ] 11 | ) 12 | -------------------------------------------------------------------------------- /src/15/defining_and_exporting_c_apis_from_extension_modules/pysample.h: -------------------------------------------------------------------------------- 1 | /* pysample.h */ 2 | #include "Python.h" 3 | #include "sample.h" 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | /* Public API Table */ 9 | typedef struct { 10 | Point *(*aspoint)(PyObject *); 11 | PyObject *(*frompoint)(Point *, int); 12 | } _PointAPIMethods; 13 | 14 | #ifndef PYSAMPLE_MODULE 15 | /* Method table in external module */ 16 | static _PointAPIMethods *_point_api = 0; 17 | 18 | /* Import the API table from sample */ 19 | static int import_sample(void) { 20 | _point_api = (_PointAPIMethods *) PyCapsule_Import("sample._point_api",0); 21 | return (_point_api != NULL) ? 1 : 0; 22 | } 23 | 24 | /* Macros to implement the programming interface */ 25 | #define PyPoint_AsPoint(obj) (_point_api->aspoint)(obj) 26 | #define PyPoint_FromPoint(obj) (_point_api->frompoint)(obj) 27 | #endif 28 | 29 | #ifdef __cplusplus 30 | } 31 | #endif 32 | -------------------------------------------------------------------------------- /src/15/defining_and_exporting_c_apis_from_extension_modules/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["../sample.c", "pysample.c"], 8 | include_dirs = ['..'], 9 | ) 10 | ] 11 | ) 12 | -------------------------------------------------------------------------------- /src/15/diagnosing_segmentation_faults/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | import sample 3 | 4 | def foo(): 5 | print('About to die') 6 | sample.die() 7 | 8 | def bar(): 9 | print('About to call the function that dies') 10 | foo() 11 | 12 | def spam(): 13 | print('About to call the function that calls the function that dies') 14 | bar() 15 | 16 | if __name__ == '__main__': 17 | import faulthandler 18 | faulthandler.enable() 19 | spam() 20 | 21 | -------------------------------------------------------------------------------- /src/15/diagnosing_segmentation_faults/sample.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | static PyObject *py_die(PyObject *self, PyObject *args) { 5 | char *s = 0; 6 | 7 | *s = 'x'; 8 | Py_RETURN_NONE; 9 | } 10 | 11 | 12 | /* Module method table */ 13 | static PyMethodDef SampleMethods[] = { 14 | {"die", py_die, METH_VARARGS}, 15 | { NULL, NULL, 0, NULL} 16 | }; 17 | 18 | /* Module structure */ 19 | static struct PyModuleDef samplemodule = { 20 | PyModuleDef_HEAD_INIT, 21 | "sample", /* name of module */ 22 | "A sample module", /* Doc string (may be NULL) */ 23 | -1, /* Size of per-interpreter state or -1 */ 24 | SampleMethods /* Method table */ 25 | }; 26 | 27 | /* Module initialization function */ 28 | PyMODINIT_FUNC 29 | PyInit_sample(void) { 30 | return PyModule_Create(&samplemodule); 31 | } 32 | -------------------------------------------------------------------------------- /src/15/diagnosing_segmentation_faults/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["sample.c"], 8 | ) 9 | ] 10 | ) 11 | -------------------------------------------------------------------------------- /src/15/managing_opaque_pointers_in_c_extension_modules/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | p1 = sample.Point(2, 3) 3 | p2 = sample.Point(4, 5) 4 | print(p1) 5 | print(p2) 6 | print(sample.distance(p1, p2)) 7 | del p1 8 | del p2 9 | print('Done') 10 | -------------------------------------------------------------------------------- /src/15/managing_opaque_pointers_in_c_extension_modules/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["../sample.c", "pysample.c"], 8 | include_dirs = ['..'], 9 | ) 10 | ] 11 | ) 12 | -------------------------------------------------------------------------------- /src/15/passing_null_terminated_strings_to_c_libraries/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | import sys 3 | 4 | sample.print_chars(b'hello world') 5 | 6 | s = 'Spicy Jalape\u00f1o' 7 | print(sys.getsizeof(s)) 8 | sample.print_chars_str(s) 9 | print(sys.getsizeof(s)) 10 | del s 11 | 12 | s = 'spicy Jalape\u00f1o' 13 | print(sys.getsizeof(s)) 14 | sample.print_chars_str_alt(s) 15 | print(sys.getsizeof(s)) 16 | -------------------------------------------------------------------------------- /src/15/passing_null_terminated_strings_to_c_libraries/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["sample.c"], 8 | ) 9 | ] 10 | ) 11 | -------------------------------------------------------------------------------- /src/15/passing_unicode_strings_to_c_libraries/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | s = "Spicy Jalape\u00f1o" 3 | sample.print_chars(s) 4 | sample.print_wchars(s) 5 | -------------------------------------------------------------------------------- /src/15/passing_unicode_strings_to_c_libraries/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["sample.c"], 8 | ) 9 | ] 10 | ) 11 | -------------------------------------------------------------------------------- /src/15/reading_file_like_objects_from_c/example.py: -------------------------------------------------------------------------------- 1 | f = open('sample.c') 2 | import sample 3 | sample.consume_file(f) 4 | f.close() 5 | 6 | print('**** DONE') 7 | -------------------------------------------------------------------------------- /src/15/reading_file_like_objects_from_c/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["sample.c"], 8 | ) 9 | ] 10 | ) 11 | -------------------------------------------------------------------------------- /src/15/sample.c: -------------------------------------------------------------------------------- 1 | /* sample.c */ 2 | #include 3 | 4 | /* Compute the greatest common divisor */ 5 | int gcd(int x, int y) { 6 | int g = y; 7 | while (x > 0) { 8 | g = x; 9 | x = y % x; 10 | y = g; 11 | } 12 | return g; 13 | } 14 | 15 | /* Test if (x0,y0) is in the Mandelbrot set or not */ 16 | int in_mandel(double x0, double y0, int n) { 17 | double x=0,y=0,xtemp; 18 | while (n > 0) { 19 | xtemp = x*x - y*y + x0; 20 | y = 2*x*y + y0; 21 | x = xtemp; 22 | n -= 1; 23 | if (x*x + y*y > 4) return 0; 24 | } 25 | return 1; 26 | } 27 | 28 | /* Divide two numbers */ 29 | int divide(int a, int b, int *remainder) { 30 | int quot = a / b; 31 | *remainder = a % b; 32 | return quot; 33 | } 34 | 35 | /* Average values in an array */ 36 | double avg(double *a, int n) { 37 | int i; 38 | double total = 0.0; 39 | for (i = 0; i < n; i++) { 40 | total += a[i]; 41 | } 42 | return total / n; 43 | } 44 | 45 | /* A C data structure */ 46 | typedef struct Point { 47 | double x,y; 48 | } Point; 49 | 50 | /* Function involving a C data structure */ 51 | double distance(Point *p1, Point *p2) { 52 | return hypot(p1->x - p2->x, p1->y - p2->y); 53 | } 54 | -------------------------------------------------------------------------------- /src/15/sample.h: -------------------------------------------------------------------------------- 1 | /* sample.h */ 2 | 3 | extern int gcd(int x, int y); 4 | extern int in_mandel(double x0, double y0, int n); 5 | extern int divide(int a, int b, int *remainder); 6 | extern double avg(double *a, int n); 7 | 8 | typedef struct Point { 9 | double x,y; 10 | } Point; 11 | 12 | extern double distance(Point *p1, Point *p2); 13 | -------------------------------------------------------------------------------- /src/15/turning_a_function_pointer_into_a_callable/example.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | lib = ctypes.cdll.LoadLibrary(None) 3 | 4 | # Get the address of sin() from the C math library 5 | addr = ctypes.cast(lib.sin, ctypes.c_void_p).value 6 | print(addr) 7 | 140735505915760 8 | 9 | # Turn the address into a callable function 10 | functype = ctypes.CFUNCTYPE(ctypes.c_double, ctypes.c_double) 11 | func = functype(addr) 12 | print(func) 13 | 14 | # Call the resulting function 15 | print(func(2)) 16 | print(func(0)) 17 | -------------------------------------------------------------------------------- /src/15/using_cython_to_write_high_performance_array_operations/example.py: -------------------------------------------------------------------------------- 1 | # array module example 2 | import sample 3 | import array 4 | a = array.array('d',[1,-3,4,7,2,0]) 5 | print(a) 6 | sample.clip(a,1,4,a) 7 | print(a) 8 | 9 | # numpy example 10 | import numpy 11 | b = numpy.random.uniform(-10,10,size=1000000) 12 | print(b) 13 | c = numpy.zeros_like(b) 14 | print(c) 15 | sample.clip(b,-5,5,c) 16 | print(c) 17 | print(min(c)) 18 | print(max(c)) 19 | 20 | # Timing test 21 | from timeit import timeit 22 | print('numpy.clip') 23 | print(timeit('numpy.clip(b,-5,5,c)', 'from __main__ import b,c,numpy', number=1000)) 24 | print('sample.clip') 25 | print(timeit('sample.clip(b,-5,5,c)', 'from __main__ import b,c,sample', number=1000)) 26 | 27 | print('sample.clip_fast') 28 | print(timeit('sample.clip_fast(b,-5,5,c)', 'from __main__ import b,c,sample', number=1000)) 29 | 30 | # 2D test 31 | d = numpy.random.uniform(-10,10,size=(1000,1000)) 32 | print(d) 33 | sample.clip2d(d, -5, 5, d) 34 | print(d) 35 | -------------------------------------------------------------------------------- /src/15/using_cython_to_write_high_performance_array_operations/setup.py: -------------------------------------------------------------------------------- 1 | 2 | from distutils.core import setup 3 | from distutils.extension import Extension 4 | from Cython.Distutils import build_ext 5 | 6 | ext_modules = [ 7 | Extension("sample", 8 | ["sample.pyx"]) 9 | ] 10 | 11 | setup( 12 | name = 'Sample app', 13 | cmdclass = {'build_ext': build_ext}, 14 | ext_modules = ext_modules 15 | ) 16 | -------------------------------------------------------------------------------- /src/15/working_with_c_strings_of_dubious_encoding/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | s = sample.retstr() 3 | print(repr(s)) 4 | sample.print_chars(s) 5 | -------------------------------------------------------------------------------- /src/15/working_with_c_strings_of_dubious_encoding/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["sample.c"], 8 | ) 9 | ] 10 | ) 11 | -------------------------------------------------------------------------------- /src/15/wrapping_c_code_with_swig/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | print(sample.gcd(42, 8)) 3 | print(sample.divide(42, 8)) 4 | p1 = sample.Point(2, 3) 5 | p2 = sample.Point(4, 5) 6 | print(sample.distance(p1, p2)) 7 | print(p1.x) 8 | print(p1.y) 9 | 10 | import array 11 | a = array.array('d', [1, 2, 3]) 12 | print(sample.avg(a)) 13 | -------------------------------------------------------------------------------- /src/15/wrapping_c_code_with_swig/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name='sample', 5 | py_modules=['sample.py'], 6 | ext_modules=[ 7 | Extension('_sample', 8 | ['../sample.c', 'sample_wrap.c'], 9 | include_dirs = ['..'], 10 | define_macros = [], 11 | undef_macros = [], 12 | library_dirs = [], 13 | libraries = [] 14 | ) 15 | ] 16 | ) 17 | -------------------------------------------------------------------------------- /src/15/wrapping_existing_c_code_with_cython/csample.pxd: -------------------------------------------------------------------------------- 1 | # csample.pxd 2 | # 3 | # Declarations of "external" C functions and structures 4 | 5 | cdef extern from "sample.h": 6 | int gcd(int, int) 7 | bint in_mandel(double, double, int) 8 | int divide(int, int, int *) 9 | double avg(double *, int) nogil 10 | 11 | ctypedef struct Point: 12 | double x 13 | double y 14 | 15 | double distance(Point *, Point *) 16 | -------------------------------------------------------------------------------- /src/15/wrapping_existing_c_code_with_cython/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | print(sample.gcd(42, 8)) 3 | print(sample.divide(42, 8)) 4 | p1 = sample.Point(2, 3) 5 | p2 = sample.Point(4, 5) 6 | print(p1) 7 | print(p2) 8 | print(sample.distance(p1, p2)) 9 | 10 | import array 11 | a = array.array('d', [1, 2, 3]) 12 | print(sample.avg(a)) 13 | -------------------------------------------------------------------------------- /src/15/wrapping_existing_c_code_with_cython/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | from Cython.Distutils import build_ext 4 | 5 | ext_modules = [ 6 | Extension("sample", 7 | ["sample.pyx"], 8 | include_dirs=['..'], 9 | libraries=['sample'], 10 | library_dirs=['..'], 11 | runtime_library_dirs=['..'])] 12 | setup( 13 | name = 'Sample extension module', 14 | cmdclass = {'build_ext': build_ext}, 15 | ext_modules = ext_modules 16 | ) 17 | -------------------------------------------------------------------------------- /src/15/wrapping_existing_c_code_with_cython/setup_alt.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | from Cython.Distutils import build_ext 4 | 5 | ext_modules = [ 6 | Extension("sample", 7 | ["sample_alt.pyx"], 8 | include_dirs=['..'], 9 | libraries=['sample'], 10 | runtime_library_dirs=['..'], 11 | library_dirs=['..'])] 12 | setup( 13 | name = 'Sample extension module', 14 | cmdclass = {'build_ext': build_ext}, 15 | ext_modules = ext_modules 16 | ) 17 | -------------------------------------------------------------------------------- /src/15/writing_a_simple_c_extension_module/example.py: -------------------------------------------------------------------------------- 1 | import sample 2 | print(sample.gcd(35,42)) 3 | print(sample.in_mandel(0,0,500)) 4 | print(sample.in_mandel(2.0,1.0,500)) 5 | print(sample.divide(42,8)) 6 | -------------------------------------------------------------------------------- /src/15/writing_a_simple_c_extension_module/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["../sample.c", "pysample.c"], 8 | include_dirs = ['..'], 9 | ) 10 | ] 11 | ) 12 | -------------------------------------------------------------------------------- /src/15/writing_an_extension_function_that_operates_on_arrays/example.py: -------------------------------------------------------------------------------- 1 | import array 2 | from sample import avg 3 | 4 | print(avg(array.array('d',[1,2,3]))) 5 | try: 6 | import numpy 7 | print(avg(numpy.array([1., 2., 3.]))) 8 | except ImportError: 9 | pass 10 | 11 | -------------------------------------------------------------------------------- /src/15/writing_an_extension_function_that_operates_on_arrays/setup.py: -------------------------------------------------------------------------------- 1 | # setup.py 2 | from distutils.core import setup, Extension 3 | 4 | setup(name="sample", 5 | ext_modules=[ 6 | Extension("sample", 7 | ["../sample.c", "pysample.c"], 8 | include_dirs = ['..'], 9 | ) 10 | ] 11 | ) 12 | -------------------------------------------------------------------------------- /src/2/combining_and_concatenating_strings/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of combining text via generators 4 | 5 | def sample(): 6 | yield "Is" 7 | yield "Chicago" 8 | yield "Not" 9 | yield "Chicago?" 10 | 11 | # (a) Simple join operator 12 | text = ''.join(sample()) 13 | print(text) 14 | 15 | # (b) Redirection of parts to I/O 16 | import sys 17 | for part in sample(): 18 | sys.stdout.write(part) 19 | sys.stdout.write('\n') 20 | 21 | # (c) Combination of parts into buffers and larger I/O operations 22 | def combine(source, maxsize): 23 | parts = [] 24 | size = 0 25 | for part in source: 26 | parts.append(part) 27 | size += len(part) 28 | if size > maxsize: 29 | yield ''.join(parts) 30 | parts = [] 31 | size = 0 32 | yield ''.join(parts) 33 | 34 | for part in combine(sample(), 32768): 35 | sys.stdout.write(part) 36 | sys.stdout.write('\n') 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/2/matching_and_searching_for_text_patterns_using_regular_expressions/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Examples of simple regular expression matching 4 | 5 | import re 6 | 7 | # Some sample text 8 | text = 'Today is 11/27/2012. PyCon starts 3/13/2013.' 9 | 10 | # (a) Find all matching dates 11 | datepat = re.compile(r'\d+/\d+/\d+') 12 | print(datepat.findall(text)) 13 | 14 | # (b) Find all matching dates with capture groups 15 | datepat = re.compile(r'(\d+)/(\d+)/(\d+)') 16 | for month, day, year in datepat.findall(text): 17 | print('{}-{}-{}'.format(year, month, day)) 18 | 19 | # (c) Iterative search 20 | for m in datepat.finditer(text): 21 | print(m.groups()) 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/2/matching_strings_using_shell_wildcard_patterns/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of using shell-wildcard style matching in list comprehensions 4 | 5 | from fnmatch import fnmatchcase as match 6 | 7 | addresses = [ 8 | '5412 N CLARK ST', 9 | '1060 W ADDISON ST', 10 | '1039 W GRANVILLE AVE', 11 | '2122 N CLARK ST', 12 | '4802 N BROADWAY', 13 | ] 14 | 15 | a = [addr for addr in addresses if match(addr, '* ST')] 16 | print(a) 17 | 18 | b = [addr for addr in addresses if match(addr, '54[0-9][0-9] *CLARK*')] 19 | print(b) 20 | -------------------------------------------------------------------------------- /src/2/normalizing_unicode_text_to_a_standard_representation/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of unicode normalization 4 | 5 | # Two strings 6 | s1 = 'Spicy Jalape\u00f1o' 7 | s2 = 'Spicy Jalapen\u0303o' 8 | 9 | # (a) Print them out (usually looks identical) 10 | print(s1) 11 | print(s2) 12 | 13 | # (b) Examine equality and length 14 | print('s1 == s2', s1 == s2) 15 | print(len(s1), len(s2)) 16 | 17 | # (c) Normalize and try the same experiment 18 | import unicodedata 19 | 20 | n_s1 = unicodedata.normalize('NFC', s1) 21 | n_s2 = unicodedata.normalize('NFC', s2) 22 | 23 | print('n_s1 == n_s2', n_s1 == n_s2) 24 | print(len(n_s1), len(n_s2)) 25 | 26 | # (d) Example of normalizing to a decomposed form and stripping accents 27 | t1 = unicodedata.normalize('NFD', s1) 28 | print(''.join(c for c in t1 if not unicodedata.combining(c))) 29 | -------------------------------------------------------------------------------- /src/2/reformatting_text_to_fixed_number_of_columns/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Examples of reformatting text to different column widths 4 | 5 | # A long string 6 | s = "Look into my eyes, look into my eyes, the eyes, the eyes, \ 7 | the eyes, not around the eyes, don't look around the eyes, \ 8 | look into my eyes, you're under." 9 | 10 | import textwrap 11 | 12 | print(textwrap.fill(s, 70)) 13 | print() 14 | 15 | print(textwrap.fill(s, 40)) 16 | print() 17 | 18 | print(textwrap.fill(s, 40, initial_indent=' ')) 19 | print() 20 | 21 | print(textwrap.fill(s, 40, subsequent_indent=' ')) 22 | print() 23 | 24 | -------------------------------------------------------------------------------- /src/2/sanitizing_and_cleaning_up_text/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of some tricky sanitization problems 4 | 5 | # A tricky string 6 | s = 'p\xfdt\u0125\xf6\xf1\x0cis\tawesome\r\n' 7 | print(s) 8 | 9 | # (a) Remapping whitespace 10 | remap = { 11 | ord('\t') : ' ', 12 | ord('\f') : ' ', 13 | ord('\r') : None # Deleted 14 | } 15 | 16 | a = s.translate(remap) 17 | print('whitespace remapped:', a) 18 | 19 | # (b) Remove all combining characters/marks 20 | import unicodedata 21 | import sys 22 | cmb_chrs = dict.fromkeys(c for c in range(sys.maxunicode) 23 | if unicodedata.combining(chr(c))) 24 | 25 | b = unicodedata.normalize('NFD', a) 26 | c = b.translate(cmb_chrs) 27 | print('accents removed:', c) 28 | 29 | # (c) Accent removal using I/O decoding 30 | d = b.encode('ascii','ignore').decode('ascii') 31 | print('accents removed via I/O:', d) 32 | -------------------------------------------------------------------------------- /src/2/searching_and_replacing_text/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Examples of simple regular expression substitution 4 | 5 | import re 6 | 7 | # Some sample text 8 | text = 'Today is 11/27/2012. PyCon starts 3/13/2013.' 9 | 10 | datepat = re.compile(r'(\d+)/(\d+)/(\d+)') 11 | 12 | # (a) Simple substitution 13 | print(datepat.sub(r'\3-\1-\2', text)) 14 | 15 | # (b) Replacement function 16 | from calendar import month_abbr 17 | 18 | def change_date(m): 19 | mon_name = month_abbr[int(m.group(1))] 20 | return '{} {} {}'.format(m.group(2), mon_name, m.group(3)) 21 | 22 | print(datepat.sub(change_date, text)) 23 | -------------------------------------------------------------------------------- /src/2/specifying_a_regular_expression_for_the_shortest_match/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of a regular expression that finds shortest matches 4 | 5 | import re 6 | 7 | # Sample text 8 | text = 'Computer says "no." Phone says "yes."' 9 | 10 | # (a) Regex that finds quoted strings - longest match 11 | str_pat = re.compile(r'\"(.*)\"') 12 | print(str_pat.findall(text)) 13 | 14 | # (b) Regex that finds quoted strings - shortest match 15 | str_pat = re.compile(r'\"(.*?)\"') 16 | print(str_pat.findall(text)) 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/2/splitting_strings_on_any_of_multiple_delimiters/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of splitting a string on multiple delimiters using a regex 4 | 5 | import re 6 | 7 | line = 'asdf fjdk; afed, fjek,asdf, foo' 8 | 9 | # (a) Splitting on space, comma, and semicolon 10 | parts = re.split(r'[;,\s]\s*', line) 11 | print(parts) 12 | 13 | # (b) Splitting with a capture group 14 | fields = re.split(r'(;|,|\s)\s*', line) 15 | print(fields) 16 | 17 | # (c) Rebuilding a string using fields above 18 | values = fields[::2] 19 | delimiters = fields[1::2] 20 | delimiters.append('') 21 | print('value =', values) 22 | print('delimiters =', delimiters) 23 | newline = ''.join(v+d for v,d in zip(values, delimiters)) 24 | print('newline =', newline) 25 | 26 | # (d) Splitting using a non-capture group 27 | parts = re.split(r'(?:,|;|\s)\s*', line) 28 | print(parts) 29 | 30 | -------------------------------------------------------------------------------- /src/2/tokenizing_text/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of a tokenizer 4 | 5 | import re 6 | from collections import namedtuple 7 | 8 | NAME = r'(?P[a-zA-Z_][a-zA-Z_0-9]*)' 9 | NUM = r'(?P\d+)' 10 | PLUS = r'(?P\+)' 11 | TIMES = r'(?P\*)' 12 | EQ = r'(?P=)' 13 | WS = r'(?P\s+)' 14 | 15 | master_pat = re.compile('|'.join([NAME, NUM, PLUS, TIMES, EQ, WS])) 16 | 17 | Token = namedtuple('Token', ['type','value']) 18 | 19 | def generate_tokens(pat, text): 20 | scanner = pat.scanner(text) 21 | for m in iter(scanner.match, None): 22 | yield Token(m.lastgroup, m.group()) 23 | 24 | for tok in generate_tokens(master_pat, 'foo = 42'): 25 | print(tok) 26 | 27 | -------------------------------------------------------------------------------- /src/2/variable_interpolation_in_strings/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Examples of variable interpolation 4 | 5 | # Class for performing safe substitutions 6 | class safesub(dict): 7 | def __missing__(self, key): 8 | return '{%s}' % key 9 | 10 | s = '{name} has {n} messages.' 11 | 12 | # (a) Simple substitution 13 | name = 'Guido' 14 | n = 37 15 | 16 | print(s.format_map(vars())) 17 | 18 | # (b) Safe substitution with missing values 19 | del n 20 | print(s.format_map(safesub(vars()))) 21 | 22 | # (c) Safe substitution + frame hack 23 | n = 37 24 | import sys 25 | def sub(text): 26 | return text.format_map(safesub(sys._getframe(1).f_locals)) 27 | 28 | print(sub('Hello {name}')) 29 | print(sub('{name} has {n} messages')) 30 | print(sub('Your favorite color is {color}')) 31 | -------------------------------------------------------------------------------- /src/2/writing_a_regular_expression_for_multiline_patterns/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Regular expression that matches multiline patterns 4 | 5 | import re 6 | 7 | text = '''/* this is a 8 | multiline comment */ 9 | ''' 10 | 11 | comment = re.compile(r'/\*((?:.|\n)*?)\*/') 12 | print(comment.findall(text)) 13 | -------------------------------------------------------------------------------- /src/3/determining_last_fridays_date/example.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | weekdays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] 4 | 5 | def get_previous_byday(dayname, start_date=None): 6 | if start_date is None: 7 | start_date = datetime.today() 8 | day_num = start_date.weekday() 9 | day_num_target = weekdays.index(dayname) 10 | days_ago = (7 + day_num - day_num_target) % 7 11 | if days_ago == 0: 12 | days_ago = 7 13 | target_date = start_date - timedelta(days=days_ago) 14 | return target_date 15 | 16 | -------------------------------------------------------------------------------- /src/3/finding_the_date_range_for_the_current_month/example.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, date, timedelta 2 | import calendar 3 | 4 | def get_month_range(start_date=None): 5 | if start_date is None: 6 | start_date = date.today().replace(day=1) 7 | days_in_month = calendar.monthrange(start_date.year, start_date.month)[1] 8 | end_date = start_date + timedelta(days=days_in_month) 9 | return (start_date, end_date) 10 | 11 | first_day, last_day = get_month_range() 12 | a_day = timedelta(days=1) 13 | while first_day < last_day: 14 | print(first_day) 15 | first_day += a_day 16 | 17 | def daterange(start, stop, step): 18 | while start < stop: 19 | yield start 20 | start += step 21 | 22 | for d in daterange(date(2012, 8, 1), date(2012, 8, 11), timedelta(days=1)): 23 | print(d) 24 | 25 | for d in daterange(datetime(2012, 8, 1), datetime(2012, 8, 3), timedelta(minutes=30)): 26 | print(d) 27 | 28 | -------------------------------------------------------------------------------- /src/4/creating_data_processing_pipelines/www/bar/access-log-0108.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/4/creating_data_processing_pipelines/www/bar/access-log-0108.bz2 -------------------------------------------------------------------------------- /src/4/creating_data_processing_pipelines/www/bar/access-log-0208.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/4/creating_data_processing_pipelines/www/bar/access-log-0208.bz2 -------------------------------------------------------------------------------- /src/4/creating_data_processing_pipelines/www/foo/access-log-0108.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/4/creating_data_processing_pipelines/www/foo/access-log-0108.gz -------------------------------------------------------------------------------- /src/4/creating_data_processing_pipelines/www/foo/access-log-0208.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/src/4/creating_data_processing_pipelines/www/foo/access-log-0208.gz -------------------------------------------------------------------------------- /src/4/creating_new_iteration_patterns_with_generators/example.py: -------------------------------------------------------------------------------- 1 | def frange(start, stop, increment): 2 | x = start 3 | while x < stop: 4 | yield x 5 | x += increment 6 | 7 | for n in frange(0, 4, 0.5): 8 | print(n) 9 | -------------------------------------------------------------------------------- /src/4/delegating-iteration/example.py: -------------------------------------------------------------------------------- 1 | # Example of delegating iteration to an internal container 2 | 3 | class Node: 4 | def __init__(self, value): 5 | self._value = value 6 | self._children = [] 7 | 8 | def __repr__(self): 9 | return 'Node({!r})'.format(self._value) 10 | 11 | def add_child(self, node): 12 | self._children.append(node) 13 | 14 | def __iter__(self): 15 | return iter(self._children) 16 | 17 | # Example 18 | if __name__ == '__main__': 19 | root = Node(0) 20 | child1 = Node(1) 21 | child2 = Node(2) 22 | root.add_child(child1) 23 | root.add_child(child2) 24 | for ch in root: 25 | print(ch) 26 | # Outputs: Node(1), Node(2) 27 | -------------------------------------------------------------------------------- /src/4/easy_implementation_of_the_iterator_protocol/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of depth-first search using a generator 4 | 5 | class Node: 6 | def __init__(self, value): 7 | self._value = value 8 | self._children = [] 9 | 10 | def __repr__(self): 11 | return 'Node({!r})'.format(self._value) 12 | 13 | def add_child(self, node): 14 | self._children.append(node) 15 | 16 | def __iter__(self): 17 | return iter(self._children) 18 | 19 | def depth_first(self): 20 | yield self 21 | for c in self: 22 | yield from c.depth_first() 23 | 24 | # Example 25 | if __name__ == '__main__': 26 | root = Node(0) 27 | child1 = Node(1) 28 | child2 = Node(2) 29 | root.add_child(child1) 30 | root.add_child(child2) 31 | child1.add_child(Node(3)) 32 | child1.add_child(Node(4)) 33 | child2.add_child(Node(5)) 34 | 35 | for ch in root.depth_first(): 36 | print(ch) 37 | # Outputs: Node(0), Node(1), Node(3), Node(4), Node(2), Node(5) 38 | -------------------------------------------------------------------------------- /src/4/generators_with_state/example.py: -------------------------------------------------------------------------------- 1 | # Example of a generator with extra state that can be 2 | # accessed. Simply define as a class! 3 | 4 | from collections import deque 5 | 6 | class linehistory: 7 | def __init__(self, lines, histlen=3): 8 | self.lines = lines 9 | self.history = deque(maxlen=histlen) 10 | 11 | def __iter__(self): 12 | for lineno, line in enumerate(self.lines,1): 13 | self.history.append((lineno, line)) 14 | yield line 15 | 16 | def clear(self): 17 | self.history.clear() 18 | 19 | with open('somefile.txt') as f: 20 | lines = linehistory(f) 21 | for line in lines: 22 | if 'python' in line: 23 | for lineno, hline in lines.history: 24 | print('{}:{}'.format(lineno, hline), end='') 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/4/generators_with_state/somefile.txt: -------------------------------------------------------------------------------- 1 | hello world 2 | this is a test 3 | of iterating over lines with a history 4 | python is fun 5 | -------------------------------------------------------------------------------- /src/4/how_to_flatten_a_nested_sequence/example.py: -------------------------------------------------------------------------------- 1 | # Example of flattening a nested sequence using subgenerators 2 | 3 | from collections import Iterable 4 | 5 | def flatten(items, ignore_types=(str, bytes)): 6 | for x in items: 7 | if isinstance(x, Iterable) and not isinstance(x, ignore_types): 8 | yield from flatten(x) 9 | else: 10 | yield x 11 | 12 | items = [1, 2, [3, 4, [5, 6], 7], 8] 13 | 14 | # Produces 1 2 3 4 5 6 7 8 15 | for x in flatten(items): 16 | print(x) 17 | 18 | items = ['Dave', 'Paula', ['Thomas', 'Lewis']] 19 | for x in flatten(items): 20 | print(x) 21 | -------------------------------------------------------------------------------- /src/4/iterate_over_the_index-value_pairs_of_a_list/example.py: -------------------------------------------------------------------------------- 1 | # Example of iterating over lines of a file with an extra lineno attribute 2 | def parse_data(filename): 3 | with open(filename, 'rt') as f: 4 | for lineno, line in enumerate(f, 1): 5 | fields = line.split() 6 | try: 7 | count = int(fields[1]) 8 | except ValueError as e: 9 | print('Line {}: Parse error: {}'.format(lineno, e)) 10 | 11 | parse_data('sample.dat') 12 | -------------------------------------------------------------------------------- /src/4/iterate_over_the_index-value_pairs_of_a_list/sample.dat: -------------------------------------------------------------------------------- 1 | 0 1 2 | 2 3 3 | 4 N/A 4 | 5 6 5 | 7 8 6 | 9 10 7 | -------------------------------------------------------------------------------- /src/4/iterating_in_reverse/example.py: -------------------------------------------------------------------------------- 1 | # Example of an object implementing both forward and reversed iterators 2 | 3 | class Countdown: 4 | def __init__(self, start): 5 | self.start = start 6 | 7 | # Forward iterator 8 | def __iter__(self): 9 | n = self.start 10 | while n > 0: 11 | yield n 12 | n -= 1 13 | 14 | # Reverse iterator 15 | def __reversed__(self): 16 | n = 1 17 | while n <= self.start: 18 | yield n 19 | n += 1 20 | 21 | c = Countdown(5) 22 | print("Forward:") 23 | for x in c: 24 | print(x) 25 | 26 | print("Reverse:") 27 | for x in reversed(c): 28 | print(x) 29 | -------------------------------------------------------------------------------- /src/4/iterating_in_sorted_order_over_merged_sorted_iterables/example.py: -------------------------------------------------------------------------------- 1 | # Iterating over merged sorted iterables 2 | 3 | import heapq 4 | a = [1, 4, 7, 10] 5 | b = [2, 5, 6, 11] 6 | for c in heapq.merge(a, b): 7 | print(c) 8 | 9 | -------------------------------------------------------------------------------- /src/4/iterating_on_items_in_separate_containers/example.py: -------------------------------------------------------------------------------- 1 | # Example of iterating over two sequences as one 2 | 3 | from itertools import chain 4 | a = [1, 2, 3, 4] 5 | b = ['x', 'y', 'z'] 6 | for x in chain(a, b): 7 | print(x) 8 | 9 | -------------------------------------------------------------------------------- /src/5/adding_or_changing_the_encoding_of_an_already_open_file/example.py: -------------------------------------------------------------------------------- 1 | # Example of adding a text encoding to existing file-like object 2 | 3 | import urllib.request 4 | import io 5 | 6 | u = urllib.request.urlopen('http://www.python.org') 7 | f = io.TextIOWrapper(u, encoding='utf-8') 8 | text = f.read() 9 | 10 | print(text) 11 | 12 | -------------------------------------------------------------------------------- /src/5/getting_a_directory_listing/example.py: -------------------------------------------------------------------------------- 1 | # Example of getting a directory listing 2 | 3 | import os 4 | import os.path 5 | import glob 6 | 7 | pyfiles = glob.glob('*.py') 8 | 9 | # Get file sizes and modification dates 10 | name_sz_date = [(name, os.path.getsize(name), os.path.getmtime(name)) 11 | for name in pyfiles] 12 | 13 | for r in name_sz_date: 14 | print(r) 15 | 16 | # Get file metadata 17 | file_metadata = [(name, os.stat(name)) for name in pyfiles] 18 | for name, meta in file_metadata: 19 | print(name, meta.st_size, meta.st_mtime) 20 | -------------------------------------------------------------------------------- /src/5/iterating_over_fixed-sized_records/data.bin: -------------------------------------------------------------------------------- 1 | 0 5412 N CLARK 3 5148 N CLARK 10 5800 E 58TH 4 2122 N CLARK 1 5645 N RAVENSWOOD 7 1060 W ADDISON 6 4801 N BROADWAY 1 1039 W GRANVILLE -------------------------------------------------------------------------------- /src/5/iterating_over_fixed-sized_records/example.py: -------------------------------------------------------------------------------- 1 | # Example of iterating of fixed-size records 2 | # 3 | # The file 'data.bin' contains 32-byte fixed size records 4 | # that consist of a 4-digit number followed by a 28-byte string. 5 | 6 | from functools import partial 7 | RECORD_SIZE = 32 8 | 9 | with open('data.bin', 'rb') as f: 10 | records = iter(partial(f.read, RECORD_SIZE), b'') 11 | for r in records: 12 | print(r) 13 | 14 | -------------------------------------------------------------------------------- /src/5/reading_and_writing_text_data/example.py: -------------------------------------------------------------------------------- 1 | # Some examples of reading text files with different options 2 | # 3 | # The file sample.txt is a UTF-8 encoded text file with Windows 4 | # line-endings (\r\n). 5 | 6 | # (a) Reading a basic text file (UTF-8 default encoding) 7 | 8 | print("Reading a simple text file (UTF-8)") 9 | with open('sample.txt', 'rt') as f: 10 | for line in f: 11 | print(repr(line)) 12 | 13 | # (b) Reading a text file with universal newlines turned off 14 | print("Reading text file with universal newlines off") 15 | with open('sample.txt', 'rt', newline='') as f: 16 | for line in f: 17 | print(repr(line)) 18 | 19 | # (c) Reading text file as ASCII with replacement error handling 20 | print("Reading text as ASCII with replacement error handling") 21 | with open('sample.txt', 'rt', encoding='ascii', errors='replace') as f: 22 | for line in f: 23 | print(repr(line)) 24 | 25 | # (d) Reading text file as ASCII with ignore error handling 26 | print("Reading text as ASCII with ignore error handling") 27 | with open('sample.txt', 'rt', encoding='ascii', errors='ignore') as f: 28 | for line in f: 29 | print(repr(line)) 30 | 31 | -------------------------------------------------------------------------------- /src/5/reading_and_writing_text_data/sample.txt: -------------------------------------------------------------------------------- 1 | Hello World 2 | Spicy Jalapeño 3 | -------------------------------------------------------------------------------- /src/5/wrapping_an_existing_file_descriptor_as_a_file_object/echo.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | 3 | def echo_client(client_sock, addr): 4 | print("Got connection from", addr) 5 | 6 | # Make text-mode file wrappers for socket reading/writing 7 | client_in = open(client_sock.fileno(), 'rt', encoding='latin-1', closefd=False) 8 | client_out = open(client_sock.fileno(), 'wt', encoding='latin-1', closefd=False) 9 | 10 | # Echo lines back to the client using file I/O 11 | for line in client_in: 12 | client_out.write(line) 13 | client_out.flush() 14 | client_sock.close() 15 | 16 | def echo_server(address): 17 | sock = socket(AF_INET, SOCK_STREAM) 18 | sock.bind(address) 19 | sock.listen(1) 20 | while True: 21 | client, addr = sock.accept() 22 | echo_client(client, addr) 23 | 24 | if __name__ == '__main__': 25 | print('Echo serving running on localhost:25000') 26 | echo_server(('', 25000)) 27 | -------------------------------------------------------------------------------- /src/5/writing_bytes_to_a_text_file/example.py: -------------------------------------------------------------------------------- 1 | # Example of writing raw bytes on a file opened in text mode 2 | 3 | import sys 4 | 5 | # A byte string 6 | data = b'Hello World\n' 7 | 8 | # Write onto the buffer attribute (bypassing text encoding) 9 | sys.stdout.buffer.write(data) 10 | -------------------------------------------------------------------------------- /src/6/incremental_parsing_of_huge_xml_files/example.py: -------------------------------------------------------------------------------- 1 | # Example of incremental XML parsing 2 | # 3 | # The file 'potholes.xml' is a greatly condensed version of a larger 4 | # file available for download at 5 | # 6 | # https://data.cityofchicago.org/api/views/7as2-ds3y/rows.xml?accessType=DOWNLOAD 7 | 8 | from xml.etree.ElementTree import iterparse 9 | 10 | def parse_and_remove(filename, path): 11 | path_parts = path.split('/') 12 | doc = iterparse(filename, ('start', 'end')) 13 | # Skip the root element 14 | next(doc) 15 | 16 | tag_stack = [] 17 | elem_stack = [] 18 | for event, elem in doc: 19 | if event == 'start': 20 | tag_stack.append(elem.tag) 21 | elem_stack.append(elem) 22 | elif event == 'end': 23 | if tag_stack == path_parts: 24 | yield elem 25 | elem_stack[-2].remove(elem) 26 | try: 27 | tag_stack.pop() 28 | elem_stack.pop() 29 | except IndexError: 30 | pass 31 | 32 | # Find zip code with most potholes 33 | 34 | from collections import Counter 35 | potholes_by_zip = Counter() 36 | 37 | data = parse_and_remove('potholes.xml', 'row/row') 38 | for pothole in data: 39 | potholes_by_zip[pothole.findtext('zip')] += 1 40 | 41 | for zipcode, num in potholes_by_zip.most_common(): 42 | print(zipcode, num) 43 | -------------------------------------------------------------------------------- /src/6/parsing_modifying_and_rewriting_xml/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of reading an XML document, making changes, and writing it back out 4 | 5 | from xml.etree.ElementTree import parse, Element 6 | doc = parse('pred.xml') 7 | root = doc.getroot() 8 | 9 | # Remove a few elements 10 | root.remove(root.find('sri')) 11 | root.remove(root.find('cr')) 12 | 13 | # Insert a new element after ... 14 | nm_index = root.getchildren().index(root.find('nm')) 15 | 16 | e = Element('spam') 17 | e.text = 'This is a test' 18 | root.insert(nm_index + 1, e) 19 | 20 | # Write back to a file 21 | doc.write('newpred.xml', xml_declaration=True) 22 | -------------------------------------------------------------------------------- /src/6/parsing_modifying_and_rewriting_xml/pred.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 14791 4 | Clark & Balmoral 5 | 6 | 22 7 | North Bound 8 |
North Bound
9 | 10 | 22 11 | 
12 |        5 MIN
13 |        Howard
14 |        1378
15 |        22
16 |
17 | 
18 |        15 MIN
19 |        Howard
20 |        1867
21 |        22
22 |
23 | 24 | -------------------------------------------------------------------------------- /src/6/parsing_simple_xml_data/example.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | from xml.etree.ElementTree import parse 3 | 4 | # Download the RSS feed and parse it 5 | u = urlopen('http://planet.python.org/rss20.xml') 6 | doc = parse(u) 7 | 8 | # Extract and output tags of interest 9 | for item in doc.iterfind('channel/item'): 10 | title = item.findtext('title') 11 | date = item.findtext('pubDate') 12 | link = item.findtext('link') 13 | 14 | print(title) 15 | print(date) 16 | print(link) 17 | print() 18 | -------------------------------------------------------------------------------- /src/6/parsing_xml_documents_with_namespaces/example.py: -------------------------------------------------------------------------------- 1 | # example.py 2 | # 3 | # Example of XML namespace handling 4 | 5 | from xml.etree.ElementTree import parse 6 | 7 | class XMLNamespaces: 8 | def __init__(self, **kwargs): 9 | self.namespaces = {} 10 | for name, uri in kwargs.items(): 11 | self.register(name, uri) 12 | def register(self, name, uri): 13 | self.namespaces[name] = '{'+uri+'}' 14 | def __call__(self, path): 15 | return path.format_map(self.namespaces) 16 | 17 | doc = parse('sample.xml') 18 | ns = XMLNamespaces(html='http://www.w3.org/1999/xhtml') 19 | 20 | e = doc.find(ns('content/{html}html')) 21 | print(e) 22 | 23 | text = doc.findtext(ns('content/{html}html/{html}head/{html}title')) 24 | print(text) 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/6/parsing_xml_documents_with_namespaces/sample.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | David Beazley 4 | 5 | 6 | 7 | Hello World 8 | 9 | 10 |

Hello World!

11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/6/reading_and_writing_binary_arrays_of_structures/readrecords.py: -------------------------------------------------------------------------------- 1 | from struct import Struct 2 | 3 | def read_records(format, f): 4 | record_struct = Struct(format) 5 | chunks = iter(lambda: f.read(record_struct.size), b'') 6 | return (record_struct.unpack(chunk) for chunk in chunks) 7 | 8 | # Example 9 | if __name__ == '__main__': 10 | with open('data.b','rb') as f: 11 | for rec in read_records('Albatross' 22 | print(make_element('item', 'Albatross', size='large', quantity=6)) 23 | print(make_element('p','')) 24 | -------------------------------------------------------------------------------- /src/7/functions_that_only_accept_keyword_arguments/example.py: -------------------------------------------------------------------------------- 1 | # examples of keyword-only argument functions 2 | 3 | # A simple keyword-only argument 4 | def recv(maxsize, *, block=True): 5 | print(maxsize, block) 6 | 7 | recv(8192, block=False) # Works 8 | try: 9 | recv(8192, False) # Fails 10 | except TypeError as e: 11 | print(e) 12 | 13 | # Adding keyword-only args to *args functions 14 | def minimum(*values, clip=None): 15 | m = min(values) 16 | if clip is not None: 17 | m = clip if clip > m else m 18 | return m 19 | 20 | print(minimum(1, 5, 2, -5, 10)) 21 | print(minimum(1, 5, 2, -5, 10, clip=0)) 22 | -------------------------------------------------------------------------------- /src/7/functions_with_default_arguments/example.py: -------------------------------------------------------------------------------- 1 | # Examples of a function with default arguments 2 | 3 | # (a) Dangers of using a mutable default argument 4 | 5 | def spam(b=[]): 6 | return b 7 | 8 | a = spam() 9 | print(a) 10 | a.append(1) 11 | a.append(2) 12 | b = spam() 13 | print(b) # Carefully observe result 14 | print('-'*10) 15 | 16 | # (b) Better alternative for mutable defaults 17 | def spam(b=None): 18 | if b is None: 19 | b = [] 20 | return b 21 | 22 | a = spam() 23 | print(a) 24 | a.append(1) 25 | a.append(2) 26 | b = spam() 27 | print(b) 28 | print('-'*10) 29 | 30 | # (c) Example of testing if an argument was supplied or not 31 | 32 | _no_value = object() 33 | def spam(b=_no_value): 34 | if b is _no_value: 35 | print("No b value supplied") 36 | else: 37 | print("b=", b) 38 | 39 | spam() 40 | spam(None) 41 | spam(0) 42 | spam([]) 43 | -------------------------------------------------------------------------------- /src/7/making_an_n-argument_callable_work_as_a_callable_with_fewer_arguments/example1.py: -------------------------------------------------------------------------------- 1 | # Example of using partial() with sorting a list of (x,y) coordinates 2 | 3 | points = [ (1, 2), (3, 4), (5, 6), (7, 7) ] 4 | 5 | import math 6 | def distance(p1, p2): 7 | x1, y1 = p1 8 | x2, y2 = p2 9 | return math.hypot(x2 - x1, y2 - y1) 10 | 11 | pt = (4,3) 12 | points.sort(key=partial(distance, pt)) 13 | print(points) 14 | -------------------------------------------------------------------------------- /src/7/making_an_n-argument_callable_work_as_a_callable_with_fewer_arguments/example2.py: -------------------------------------------------------------------------------- 1 | # Using partial to supply extra arguments to a callback function 2 | 3 | def output_result(result, log=None): 4 | if log is not None: 5 | log.debug('Got: %r', result) 6 | 7 | # A sample function 8 | def add(x, y): 9 | return x + y 10 | 11 | if __name__ == '__main__': 12 | import logging 13 | from multiprocessing import Pool 14 | from functools import partial 15 | 16 | logging.basicConfig(level=logging.DEBUG) 17 | log = logging.getLogger('test') 18 | 19 | p = Pool() 20 | p.apply_async(add, (3, 4), callback=partial(output_result, log=log)) 21 | p.close() 22 | p.join() 23 | -------------------------------------------------------------------------------- /src/7/making_an_n-argument_callable_work_as_a_callable_with_fewer_arguments/example3.py: -------------------------------------------------------------------------------- 1 | # Using partial to supply extra arguments to a class constructor 2 | from socketserver import StreamRequestHandler, TCPServer 3 | 4 | class EchoHandler(StreamRequestHandler): 5 | # ack is added keyword-only argument. *args, **kwargs are 6 | # any normal parameters supplied (which are passed on) 7 | def __init__(self, *args, ack, **kwargs): 8 | self.ack = ack 9 | super().__init__(*args, **kwargs) 10 | def handle(self): 11 | for line in self.rfile: 12 | self.wfile.write(self.ack + line) 13 | 14 | if __name__ == '__main__': 15 | from functools import partial 16 | serv = TCPServer(('', 15000), partial(EchoHandler, ack=b'RECEIVED:')) 17 | print('Echo server running on port 15000') 18 | serv.serve_forever() 19 | -------------------------------------------------------------------------------- /src/8/calling_a_method_on_a_parent_class/example1.py: -------------------------------------------------------------------------------- 1 | class A: 2 | def spam(self): 3 | print('A.spam') 4 | 5 | class B(A): 6 | def spam(self): 7 | print('B.spam') 8 | super().spam() # Call parent spam() 9 | 10 | if __name__ == '__main__': 11 | b = B() 12 | b.spam() 13 | -------------------------------------------------------------------------------- /src/8/calling_a_method_on_a_parent_class/example2.py: -------------------------------------------------------------------------------- 1 | class A: 2 | def __init__(self): 3 | self.x = 0 4 | 5 | class B(A): 6 | def __init__(self): 7 | super().__init__() 8 | self.y = 1 9 | 10 | if __name__ == '__main__': 11 | b = B() 12 | print(b.x, b.y) 13 | -------------------------------------------------------------------------------- /src/8/calling_a_method_on_a_parent_class/example3.py: -------------------------------------------------------------------------------- 1 | class Proxy: 2 | def __init__(self, obj): 3 | self._obj = obj 4 | 5 | # Delegate attribute lookup to internal obj 6 | def __getattr__(self, name): 7 | return getattr(self._obj, name) 8 | 9 | # Delegate attribute assignment 10 | def __setattr__(self, name, value): 11 | if name.startswith('_'): 12 | super().__setattr__(name, value) # Call original __setattr__ 13 | else: 14 | setattr(self._obj, name, value) 15 | 16 | if __name__ == '__main__': 17 | class A: 18 | def __init__(self, x): 19 | self.x = x 20 | def spam(self): 21 | print('A.spam') 22 | 23 | a = A(42) 24 | p = Proxy(a) 25 | print(p.x) 26 | print(p.spam()) 27 | p.x = 37 28 | print('Should be 37:', p.x) 29 | print('Should be 37:', a.x) 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/8/calling_a_method_on_a_parent_class/example4.py: -------------------------------------------------------------------------------- 1 | # Tricky initialization problem involving multiple inheritance. 2 | # Does NOT use super() 3 | 4 | class Base: 5 | def __init__(self): 6 | print('Base.__init__') 7 | 8 | class A(Base): 9 | def __init__(self): 10 | Base.__init__(self) 11 | print('A.__init__') 12 | 13 | class B(Base): 14 | def __init__(self): 15 | Base.__init__(self) 16 | print('B.__init__') 17 | 18 | class C(A,B): 19 | def __init__(self): 20 | A.__init__(self) 21 | B.__init__(self) 22 | print('C.__init__') 23 | 24 | if __name__ == '__main__': 25 | # Please observe double call of Base.__init__ 26 | c = C() 27 | -------------------------------------------------------------------------------- /src/8/calling_a_method_on_a_parent_class/example5.py: -------------------------------------------------------------------------------- 1 | # Tricky initialization problem involving multiple inheritance. 2 | # Uses super() 3 | 4 | class Base: 5 | def __init__(self): 6 | print('Base.__init__') 7 | 8 | class A(Base): 9 | def __init__(self): 10 | super().__init__() 11 | print('A.__init__') 12 | 13 | class B(Base): 14 | def __init__(self): 15 | super().__init__() 16 | print('B.__init__') 17 | 18 | class C(A,B): 19 | def __init__(self): 20 | super().__init__() # Only one call to super() here 21 | print('C.__init__') 22 | 23 | if __name__ == '__main__': 24 | # Observe that each class initialized only once 25 | c = C() 26 | -------------------------------------------------------------------------------- /src/8/calling_a_method_on_an_object_given_the_name_as_a_string/example.py: -------------------------------------------------------------------------------- 1 | # Example of calling methods by name 2 | 3 | import math 4 | class Point: 5 | def __init__(self, x, y): 6 | self.x = x 7 | self.y = y 8 | 9 | def __repr__(self): 10 | return 'Point({!r:},{!r:})'.format(self.x, self.y) 11 | 12 | def distance(self, x, y): 13 | return math.hypot(self.x - x, self.y - y) 14 | 15 | p = Point(2,3) 16 | 17 | # Method 1 : Use getattr 18 | d = getattr(p, 'distance')(0, 0) # Calls p.distance(0, 0) 19 | print(d) 20 | 21 | # Method 2: Use methodcaller 22 | import operator 23 | d = operator.methodcaller('distance', 0, 0)(p) 24 | print(d) 25 | 26 | # Application in sorting 27 | points = [ 28 | Point(1, 2), 29 | Point(3, 0), 30 | Point(10, -3), 31 | Point(-5, -7), 32 | Point(-1, 8), 33 | Point(3, 2) 34 | ] 35 | 36 | # Sort by distance from origin (0, 0) 37 | points.sort(key=operator.methodcaller('distance', 0, 0)) 38 | for p in points: 39 | print(p) 40 | 41 | -------------------------------------------------------------------------------- /src/8/changing_the_string_representation_of_instances/example.py: -------------------------------------------------------------------------------- 1 | class Pair: 2 | def __init__(self, x, y): 3 | self.x = x 4 | self.y = y 5 | def __repr__(self): 6 | return 'Pair({0.x!r}, {0.y!r})'.format(self) 7 | def __str__(self): 8 | return '({0.x}, {0.y})'.format(self) 9 | 10 | -------------------------------------------------------------------------------- /src/8/creating_a_new_kind_of_class_or_instance_attribute/example1.py: -------------------------------------------------------------------------------- 1 | # Descriptor attribute for an integer type-checked attribute 2 | class Integer: 3 | def __init__(self, name): 4 | self.name = name 5 | 6 | def __get__(self, instance, cls): 7 | if instance is None: 8 | return self 9 | else: 10 | return instance.__dict__[self.name] 11 | 12 | def __set__(self, instance, value): 13 | if not isinstance(value, int): 14 | raise TypeError('Expected an int') 15 | instance.__dict__[self.name] = value 16 | 17 | def __delete__(self, instance): 18 | del instance.__dict__[self.name] 19 | 20 | class Point: 21 | x = Integer('x') 22 | y = Integer('y') 23 | def __init__(self, x, y): 24 | self.x = x 25 | self.y = y 26 | 27 | if __name__ == '__main__': 28 | p = Point(2, 3) 29 | print(p.x) 30 | p.y = 5 31 | try: 32 | p.x = 2.3 33 | except TypeError as e: 34 | print(e) 35 | -------------------------------------------------------------------------------- /src/8/creating_an_instance_without_invoking_init/example.py: -------------------------------------------------------------------------------- 1 | from time import localtime 2 | 3 | class Date: 4 | def __init__(self, year, month, day): 5 | self.year = year 6 | self.month = month 7 | self.day = day 8 | 9 | # Class method that bypasses __init__ 10 | @classmethod 11 | def today(cls): 12 | d = cls.__new__(cls) 13 | t = localtime() 14 | d.year = t.tm_year 15 | d.month = t.tm_mon 16 | d.day = t.tm_mday 17 | return d 18 | 19 | d = Date.__new__(Date) 20 | print(d) 21 | print(hasattr(d,'year')) 22 | 23 | data = { 24 | 'year' : 2012, 25 | 'month' : 8, 26 | 'day' : 29 27 | } 28 | 29 | d.__dict__.update(data) 30 | print(d.year) 31 | print(d.month) 32 | 33 | d = Date.today() 34 | print(d.year, d.month, d.day) 35 | 36 | -------------------------------------------------------------------------------- /src/8/creating_cached_instances/example1.py: -------------------------------------------------------------------------------- 1 | # Simple example 2 | 3 | class Spam: 4 | def __init__(self, name): 5 | self.name = name 6 | 7 | # Caching support 8 | import weakref 9 | _spam_cache = weakref.WeakValueDictionary() 10 | 11 | def get_spam(name): 12 | if name not in _spam_cache: 13 | s = Spam(name) 14 | _spam_cache[name] = s 15 | else: 16 | s = _spam_cache[name] 17 | return s 18 | 19 | if __name__ == '__main__': 20 | a = get_spam('foo') 21 | b = get_spam('bar') 22 | print('a is b:', a is b) 23 | c = get_spam('foo') 24 | print('a is c:', a is c) 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/8/creating_cached_instances/example2.py: -------------------------------------------------------------------------------- 1 | import weakref 2 | 3 | class CachedSpamManager: 4 | def __init__(self): 5 | self._cache = weakref.WeakValueDictionary() 6 | def get_spam(self, name): 7 | if name not in self._cache: 8 | s = Spam(name) 9 | self._cache[name] = s 10 | else: 11 | s = self._cache[name] 12 | return s 13 | 14 | class Spam: 15 | def __init__(self, name): 16 | self.name = name 17 | 18 | Spam.manager = CachedSpamManager() 19 | 20 | def get_spam(name): 21 | return Spam.manager.get_spam(name) 22 | 23 | if __name__ == '__main__': 24 | a = get_spam('foo') 25 | b = get_spam('bar') 26 | print('a is b:', a is b) 27 | c = get_spam('foo') 28 | print('a is c:', a is c) 29 | -------------------------------------------------------------------------------- /src/8/creating_cached_instances/example3.py: -------------------------------------------------------------------------------- 1 | # Example involving new and some of its problems 2 | 3 | import weakref 4 | 5 | class Spam: 6 | _spam_cache = weakref.WeakValueDictionary() 7 | def __new__(cls, name): 8 | if name in cls._spam_cache: 9 | return cls._spam_cache[name] 10 | else: 11 | self = super().__new__(cls) 12 | cls._spam_cache[name] = self 13 | return self 14 | 15 | def __init__(self, name): 16 | print('Initializing Spam') 17 | self.name = name 18 | 19 | if __name__ == '__main__': 20 | print("This should print 'Initializing Spam' twice") 21 | s = Spam('Dave') 22 | t = Spam('Dave') 23 | print(s is t) 24 | 25 | -------------------------------------------------------------------------------- /src/8/creating_managed_attributes/example.py: -------------------------------------------------------------------------------- 1 | # Example of managed attributes via properties 2 | 3 | class Person: 4 | def __init__(self, first_name): 5 | self.first_name = first_name 6 | 7 | # Getter function 8 | @property 9 | def first_name(self): 10 | return self._first_name 11 | 12 | # Setter function 13 | @first_name.setter 14 | def first_name(self, value): 15 | if not isinstance(value, str): 16 | raise TypeError('Expected a string') 17 | self._first_name = value 18 | 19 | if __name__ == '__main__': 20 | a = Person('Guido') 21 | print(a.first_name) 22 | a.first_name = 'Dave' 23 | print(a.first_name) 24 | try: 25 | a.first_name = 42 26 | except TypeError as e: 27 | print(e) 28 | -------------------------------------------------------------------------------- /src/8/customized_formatting/example1.py: -------------------------------------------------------------------------------- 1 | _formats = { 2 | 'ymd' : '{d.year}-{d.month}-{d.day}', 3 | 'mdy' : '{d.month}/{d.day}/{d.year}', 4 | 'dmy' : '{d.day}/{d.month}/{d.year}' 5 | } 6 | 7 | class Date: 8 | def __init__(self, year, month, day): 9 | self.year = year 10 | self.month = month 11 | self.day = day 12 | 13 | def __format__(self, code): 14 | if code == '': 15 | code = 'ymd' 16 | fmt = _formats[code] 17 | return fmt.format(d=self) 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/8/delegation_and_proxies/example1.py: -------------------------------------------------------------------------------- 1 | class A: 2 | def spam(self, x): 3 | print('A.spam') 4 | 5 | def foo(self): 6 | print('A.foo') 7 | 8 | class B: 9 | def __init__(self): 10 | self._a = A() 11 | 12 | def bar(self): 13 | print('B.bar') 14 | 15 | # Expose all of the methods defined on class A 16 | def __getattr__(self, name): 17 | return getattr(self._a, name) 18 | 19 | if __name__ == '__main__': 20 | b = B() 21 | b.bar() 22 | b.spam(42) 23 | -------------------------------------------------------------------------------- /src/8/delegation_and_proxies/example2.py: -------------------------------------------------------------------------------- 1 | # A proxy class that wraps around another object, but 2 | # exposes its public attributes 3 | 4 | class Proxy: 5 | def __init__(self, obj): 6 | self._obj = obj 7 | 8 | # Delegate attribute lookup to internal obj 9 | def __getattr__(self, name): 10 | print('getattr:', name) 11 | return getattr(self._obj, name) 12 | 13 | # Delegate attribute assignment 14 | def __setattr__(self, name, value): 15 | if name.startswith('_'): 16 | super().__setattr__(name, value) 17 | else: 18 | print('setattr:', name, value) 19 | setattr(self._obj, name, value) 20 | 21 | # Delegate attribute deletion 22 | def __delattr__(self, name): 23 | if name.startswith('_'): 24 | super().__delattr__(name) 25 | else: 26 | print('delattr:', name) 27 | delattr(self._obj, name) 28 | 29 | if __name__ == '__main__': 30 | class Spam: 31 | def __init__(self, x): 32 | self.x = x 33 | def bar(self, y): 34 | print('Spam.bar:', self.x, y) 35 | 36 | # Create an instance 37 | s = Spam(2) 38 | 39 | # Create a proxy around it 40 | p = Proxy(s) 41 | 42 | # Access the proxy 43 | print(p.x) # Outputs 2 44 | p.bar(3) # Outputs "Spam.bar: 2 3" 45 | p.x = 37 # Changes s.x to 37 46 | -------------------------------------------------------------------------------- /src/8/delegation_and_proxies/example3.py: -------------------------------------------------------------------------------- 1 | class ListLike: 2 | def __init__(self): 3 | self._items = [] 4 | def __getattr__(self, name): 5 | return getattr(self._items, name) 6 | 7 | # Added special methods to support certain list operations 8 | def __len__(self): 9 | return len(self._items) 10 | def __getitem__(self, index): 11 | return self._items[index] 12 | def __setitem__(self, index, value): 13 | self._items[index] = value 14 | def __delitem__(self, index): 15 | del self._items[index] 16 | 17 | if __name__ == '__main__': 18 | a = ListLike() 19 | a.append(2) 20 | a.insert(0, 1) 21 | a.sort() 22 | print(len(a)) 23 | print(a[0]) 24 | 25 | -------------------------------------------------------------------------------- /src/8/delegation_and_proxies/example4.py: -------------------------------------------------------------------------------- 1 | class A: 2 | def spam(self): 3 | print('A.spam') 4 | 5 | def foo(self): 6 | print('A.foo') 7 | 8 | class B: 9 | def __init__(self): 10 | self._a = A() 11 | 12 | def spam(self): 13 | print('B.spam') 14 | self._a.spam() # Similar to super() 15 | 16 | def __getattr__(self, name): 17 | return getattr(self._a, name) 18 | 19 | if __name__ == '__main__': 20 | b = B() 21 | b.spam() 22 | b.foo() 23 | -------------------------------------------------------------------------------- /src/8/extending_a_property_in_a_subclass/example2.py: -------------------------------------------------------------------------------- 1 | # Example of managed attributes via properties 2 | 3 | class String: 4 | def __init__(self, name): 5 | self.name = name 6 | def __get__(self, instance, cls): 7 | if instance is None: 8 | return self 9 | return instance.__dict__[self.name] 10 | 11 | def __set__(self, instance, value): 12 | if not isinstance(value, str): 13 | raise TypeError('Expected a string') 14 | instance.__dict__[self.name] = value 15 | 16 | 17 | class Person: 18 | name = String('name') 19 | def __init__(self, name): 20 | self.name = name 21 | 22 | class SubPerson(Person): 23 | @property 24 | def name(self): 25 | print('Getting name') 26 | return super().name 27 | 28 | @name.setter 29 | def name(self, value): 30 | print('Setting name to', value) 31 | super(SubPerson, SubPerson).name.__set__(self, value) 32 | 33 | @name.deleter 34 | def name(self): 35 | print('Deleting name') 36 | super(SubPerson, SubPerson).name.__delete__(self) 37 | 38 | if __name__ == '__main__': 39 | a = Person('Guido') 40 | print(a.name) 41 | a.name = 'Dave' 42 | print(a.name) 43 | try: 44 | a.name = 42 45 | except TypeError as e: 46 | print(e) 47 | -------------------------------------------------------------------------------- /src/8/extending_classes_with_mixins/example2.py: -------------------------------------------------------------------------------- 1 | class RestrictKeysMixin: 2 | def __init__(self, *args, _restrict_key_type, **kwargs): 3 | self.__restrict_key_type = _restrict_key_type 4 | super().__init__(*args, **kwargs) 5 | 6 | def __setitem__(self, key, value): 7 | if not isinstance(key, self.__restrict_key_type): 8 | raise TypeError('Keys must be ' + str(self.__restrict_key_type)) 9 | super().__setitem__(key, value) 10 | 11 | # Example 12 | 13 | class RDict(RestrictKeysMixin, dict): 14 | pass 15 | 16 | d = RDict(_restrict_key_type=str) 17 | e = RDict([('name','Dave'), ('n',37)], _restrict_key_type=str) 18 | f = RDict(name='Dave', n=37, _restrict_key_type=str) 19 | print(f) 20 | try: 21 | f[42] = 10 22 | except TypeError as e: 23 | print(e) 24 | -------------------------------------------------------------------------------- /src/8/extending_classes_with_mixins/example3.py: -------------------------------------------------------------------------------- 1 | # Class decorator alternative to mixins 2 | 3 | def LoggedMapping(cls): 4 | cls_getitem = cls.__getitem__ 5 | cls_setitem = cls.__setitem__ 6 | cls_delitem = cls.__delitem__ 7 | 8 | def __getitem__(self, key): 9 | print('Getting %s' % key) 10 | return cls_getitem(self, key) 11 | 12 | def __setitem__(self, key, value): 13 | print('Setting %s = %r' % (key, value)) 14 | return cls_setitem(self, key, value) 15 | 16 | def __delitem__(self, key): 17 | print('Deleting %s' % key) 18 | return cls_delitem(self, key) 19 | 20 | cls.__getitem__ = __getitem__ 21 | cls.__setitem__ = __setitem__ 22 | cls.__delitem__ = __delitem__ 23 | return cls 24 | 25 | @LoggedMapping 26 | class LoggedDict(dict): 27 | pass 28 | 29 | d = LoggedDict() 30 | d['x'] = 23 31 | print(d['x']) 32 | del d['x'] 33 | -------------------------------------------------------------------------------- /src/8/how_to_define_an_interface_or_abstract_base_class/example2.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | 3 | class A(metaclass=ABCMeta): 4 | @property 5 | @abstractmethod 6 | def name(self): 7 | pass 8 | 9 | @name.setter 10 | @abstractmethod 11 | def name(self, value): 12 | pass 13 | 14 | @classmethod 15 | @abstractmethod 16 | def method1(cls): 17 | pass 18 | 19 | @staticmethod 20 | @abstractmethod 21 | def method2(): 22 | pass 23 | -------------------------------------------------------------------------------- /src/8/how_to_define_more_than_one_constructor_in_a_class/example.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | class Date: 4 | # Primary constructor 5 | def __init__(self, year, month, day): 6 | self.year = year 7 | self.month = month 8 | self.day = day 9 | 10 | # Alternate constructor 11 | @classmethod 12 | def today(cls): 13 | t = time.localtime() 14 | return cls(t.tm_year, t.tm_mon, t.tm_mday) 15 | 16 | if __name__ == '__main__': 17 | a = Date(2012, 12, 21) 18 | b = Date.today() 19 | print(a.year, a.month, a.day) 20 | print(b.year, b.month, b.day) 21 | 22 | class NewDate(Date): 23 | pass 24 | 25 | c = Date.today() 26 | d = NewDate.today() 27 | print('Should be Date instance:', Date) 28 | print('Should be NewDate instance:', NewDate) 29 | -------------------------------------------------------------------------------- /src/8/how_to_define_more_than_one_constructor_in_a_class/example2.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | class Date: 4 | # Primary constructor 5 | def __init__(self, year, month, day): 6 | self.year = year 7 | self.month = month 8 | self.day = day 9 | 10 | # Alternate constructor 11 | @classmethod 12 | def today(cls): 13 | t = time.localtime() 14 | d = cls.__new__(cls) 15 | d.year = t.tm_year 16 | d.month = t.tm_mon 17 | d.day = t.tm_mday 18 | return d 19 | 20 | if __name__ == '__main__': 21 | a = Date(2012, 12, 21) 22 | b = Date.today() 23 | print(a.year, a.month, a.day) 24 | print(b.year, b.month, b.day) 25 | 26 | class NewDate(Date): 27 | pass 28 | 29 | c = Date.today() 30 | d = NewDate.today() 31 | print('Should be Date instance:', Date) 32 | print('Should be NewDate instance:', NewDate) 33 | -------------------------------------------------------------------------------- /src/8/how_to_encapsulate_names_in_a_class/example.py: -------------------------------------------------------------------------------- 1 | # Example of using __ method name to implement a 2 | # non-overrideable method 3 | 4 | class B: 5 | def __init__(self): 6 | self.__private = 0 7 | def __private_method(self): 8 | print('B.__private_method', self.__private) 9 | 10 | def public_method(self): 11 | self.__private_method() 12 | 13 | class C(B): 14 | def __init__(self): 15 | super().__init__() 16 | self.__private = 1 # Does not override B.__private 17 | # Does not override B.__private_method() 18 | def __private_method(self): 19 | print('C.__private_method') 20 | 21 | c = C() 22 | c.public_method() 23 | 24 | -------------------------------------------------------------------------------- /src/8/implementing_custom_containers/example1.py: -------------------------------------------------------------------------------- 1 | # Example of a custom container 2 | 3 | import collections 4 | import bisect 5 | 6 | class SortedItems(collections.Sequence): 7 | def __init__(self, initial=None): 8 | self._items = sorted(initial) if initial is None else [] 9 | 10 | # Required sequence methods 11 | def __getitem__(self, index): 12 | return self._items[index] 13 | 14 | def __len__(self): 15 | return len(self._items) 16 | 17 | # Method for adding an item in the right location 18 | def add(self, item): 19 | bisect.insort(self._items, item) 20 | 21 | if __name__ == '__main__': 22 | items = SortedItems([5, 1, 3]) 23 | print(list(items)) 24 | print(items[0]) 25 | print(items[-1]) 26 | items.add(2) 27 | print(list(items)) 28 | items.add(-10) 29 | print(list(items)) 30 | print(items[1:4]) 31 | print(3 in items) 32 | print(len(items)) 33 | for n in items: 34 | print(n) 35 | -------------------------------------------------------------------------------- /src/8/implementing_custom_containers/example2.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | class Items(collections.MutableSequence): 4 | def __init__(self, initial=None): 5 | self._items = list(initial) if initial is None else [] 6 | 7 | # Required sequence methods 8 | def __getitem__(self, index): 9 | print('Getting:', index) 10 | return self._items[index] 11 | 12 | def __setitem__(self, index, value): 13 | print('Setting:', index, value) 14 | self._items[index] = value 15 | 16 | def __delitem__(self, index): 17 | print('Deleting:', index) 18 | del self._items[index] 19 | 20 | def insert(self, index, value): 21 | print('Inserting:', index, value) 22 | self._items.insert(index, value) 23 | 24 | def __len__(self): 25 | print('Len') 26 | return len(self._items) 27 | 28 | if __name__ == '__main__': 29 | a = Items([1, 2, 3]) 30 | print(len(a)) 31 | a.append(4) 32 | a.append(2) 33 | print(a.count(2)) 34 | a.remove(3) 35 | -------------------------------------------------------------------------------- /src/8/implementing_the_visitor_pattern_without_recursion/node.py: -------------------------------------------------------------------------------- 1 | # node.py 2 | # 3 | # Base class and non-recursive visitor implementation. 4 | # Used by various example files. 5 | 6 | import types 7 | 8 | class Node: 9 | pass 10 | 11 | import types 12 | class NodeVisitor: 13 | def visit(self, node): 14 | stack = [ node ] 15 | last_result = None 16 | while stack: 17 | try: 18 | last = stack[-1] 19 | if isinstance(last, types.GeneratorType): 20 | stack.append(last.send(last_result)) 21 | last_result = None 22 | elif isinstance(last, Node): 23 | stack.append(self._visit(stack.pop())) 24 | else: 25 | last_result = stack.pop() 26 | except StopIteration: 27 | stack.pop() 28 | return last_result 29 | 30 | def _visit(self, node): 31 | methname = 'visit_' + type(node).__name__ 32 | meth = getattr(self, methname, None) 33 | if meth is None: 34 | meth = self.generic_visit 35 | return meth(node) 36 | 37 | def generic_visit(self, node): 38 | raise RuntimeError('No {} method'.format('visit_' + type(node).__name__)) 39 | -------------------------------------------------------------------------------- /src/8/lazily_computed_attributes/example1.py: -------------------------------------------------------------------------------- 1 | class lazyproperty: 2 | def __init__(self, func): 3 | self.func = func 4 | def __get__(self, instance, cls): 5 | if instance is None: 6 | return self 7 | else: 8 | value = self.func(instance) 9 | setattr(instance, self.func.__name__, value) 10 | return value 11 | 12 | if __name__ == '__main__': 13 | import math 14 | class Circle: 15 | def __init__(self, radius): 16 | self.radius = radius 17 | 18 | @lazyproperty 19 | def area(self): 20 | print('Computing area') 21 | return math.pi * self.radius ** 2 22 | 23 | @lazyproperty 24 | def perimeter(self): 25 | print('Computing perimeter') 26 | return 2 * math.pi * self.radius 27 | 28 | -------------------------------------------------------------------------------- /src/8/lazily_computed_attributes/example2.py: -------------------------------------------------------------------------------- 1 | def lazyproperty(func): 2 | name = '_lazy_' + func.__name__ 3 | @property 4 | def lazy(self): 5 | if hasattr(self, name): 6 | return getattr(self, name) 7 | else: 8 | value = func(self) 9 | setattr(self, name, value) 10 | return value 11 | return lazy 12 | 13 | if __name__ == '__main__': 14 | import math 15 | class Circle: 16 | def __init__(self, radius): 17 | self.radius = radius 18 | 19 | @lazyproperty 20 | def area(self): 21 | print('Computing area') 22 | return math.pi * self.radius ** 2 23 | 24 | @lazyproperty 25 | def perimeter(self): 26 | print('Computing perimeter') 27 | return 2 * math.pi * self.radius 28 | 29 | -------------------------------------------------------------------------------- /src/8/making_objects_support_the_context_manager_protocol/example1.py: -------------------------------------------------------------------------------- 1 | from socket import socket, AF_INET, SOCK_STREAM 2 | 3 | class LazyConnection: 4 | def __init__(self, address, family=AF_INET, type=SOCK_STREAM): 5 | self.address = address 6 | self.family = AF_INET 7 | self.type = SOCK_STREAM 8 | self.sock = None 9 | 10 | def __enter__(self): 11 | if self.sock is not None: 12 | raise RuntimeError('Already connected') 13 | self.sock = socket(self.family, self.type) 14 | self.sock.connect(self.address) 15 | return self.sock 16 | 17 | def __exit__(self, exc_ty, exc_val, tb): 18 | self.sock.close() 19 | self.sock = None 20 | 21 | if __name__ == '__main__': 22 | from functools import partial 23 | 24 | c = LazyConnection(('www.python.org', 80)) 25 | # Connection closed 26 | with c as s: 27 | # c.__enter__() executes: connection open 28 | s.send(b'GET /index.html HTTP/1.0\r\n') 29 | s.send(b'Host: www.python.org\r\n') 30 | s.send(b'\r\n') 31 | resp = b''.join(iter(partial(s.recv, 8192), b'')) 32 | # c.__exit__() executes: connection closed 33 | 34 | print('Got %d bytes' % len(resp)) 35 | -------------------------------------------------------------------------------- /src/8/managing_memory_in_cyclic_data_structures/example.py: -------------------------------------------------------------------------------- 1 | import weakref 2 | 3 | class Node: 4 | def __init__(self, value): 5 | self.value = value 6 | self._parent = None 7 | self.children = [] 8 | 9 | def __repr__(self): 10 | return 'Node({!r:})'.format(self.value) 11 | 12 | # property that manages the parent as a weak-reference 13 | @property 14 | def parent(self): 15 | return self._parent if self._parent is None else self._parent() 16 | 17 | @parent.setter 18 | def parent(self, node): 19 | self._parent = weakref.ref(node) 20 | 21 | def add_child(self, child): 22 | self.children.append(child) 23 | child.parent = self 24 | 25 | if __name__ == '__main__': 26 | root = Node('parent') 27 | c1 = Node('c1') 28 | c2 = Node('c2') 29 | root.add_child(c1) 30 | root.add_child(c2) 31 | 32 | print(c1.parent) 33 | del root 34 | print(c1.parent) 35 | 36 | -------------------------------------------------------------------------------- /src/8/simplified_initialization_of_data_structures/example1.py: -------------------------------------------------------------------------------- 1 | class Structure: 2 | # Class variable that specifies expected fields 3 | _fields= [] 4 | def __init__(self, *args): 5 | if len(args) != len(self._fields): 6 | raise TypeError('Expected {} arguments'.format(len(self._fields))) 7 | 8 | # Set the arguments 9 | for name, value in zip(self._fields, args): 10 | setattr(self, name, value) 11 | 12 | # Example class definitions 13 | if __name__ == '__main__': 14 | class Stock(Structure): 15 | _fields = ['name', 'shares', 'price'] 16 | 17 | class Point(Structure): 18 | _fields = ['x','y'] 19 | 20 | class Circle(Structure): 21 | _fields = ['radius'] 22 | def area(self): 23 | return math.pi * self.radius ** 2 24 | 25 | if __name__ == '__main__': 26 | s = Stock('ACME', 50, 91.1) 27 | print(s.name, s.shares, s.price) 28 | 29 | p = Point(2,3) 30 | print(p.x, p.y) 31 | 32 | c = Circle(4.5) 33 | print(c.radius) 34 | 35 | try: 36 | s2 = Stock('ACME', 50) 37 | except TypeError as e: 38 | print(e) 39 | -------------------------------------------------------------------------------- /src/8/simplified_initialization_of_data_structures/example2.py: -------------------------------------------------------------------------------- 1 | class Structure: 2 | _fields= [] 3 | def __init__(self, *args, **kwargs): 4 | if len(args) > len(self._fields): 5 | raise TypeError('Expected {} arguments'.format(len(self._fields))) 6 | 7 | # Set all of the positional arguments 8 | for name, value in zip(self._fields, args): 9 | setattr(self, name, value) 10 | 11 | # Set the remaining keyword arguments 12 | for name in self._fields[len(args):]: 13 | setattr(self, name, kwargs.pop(name)) 14 | 15 | # Check for any remaining unknown arguments 16 | if kwargs: 17 | raise TypeError('Invalid argument(s): {}'.format(','.join(kwargs))) 18 | 19 | # Example use 20 | if __name__ == '__main__': 21 | class Stock(Structure): 22 | _fields = ['name', 'shares', 'price'] 23 | 24 | s1 = Stock('ACME', 50, 91.1) 25 | s2 = Stock('ACME', 50, price=91.1) 26 | s3 = Stock('ACME', shares=50, price=91.1) 27 | -------------------------------------------------------------------------------- /src/8/simplified_initialization_of_data_structures/example3.py: -------------------------------------------------------------------------------- 1 | class Structure: 2 | # Class variable that specifies expected fields 3 | _fields= [] 4 | def __init__(self, *args, **kwargs): 5 | if len(args) != len(self._fields): 6 | raise TypeError('Expected {} arguments'.format(len(self._fields))) 7 | 8 | # Set the arguments 9 | for name, value in zip(self._fields, args): 10 | setattr(self, name, value) 11 | 12 | # Set the additional arguments (if any) 13 | extra_args = kwargs.keys() - self._fields 14 | for name in extra_args: 15 | setattr(self, name, kwargs.pop(name)) 16 | if kwargs: 17 | raise TypeError('Duplicate values for {}'.format(','.join(kwargs))) 18 | 19 | # Example use 20 | if __name__ == '__main__': 21 | class Stock(Structure): 22 | _fields = ['name', 'shares', 'price'] 23 | 24 | s1 = Stock('ACME', 50, 91.1) 25 | s2 = Stock('ACME', 50, 91.1, date='8/2/2012') 26 | -------------------------------------------------------------------------------- /src/9/applying_decorators_to_class_and_static_methods/example.py: -------------------------------------------------------------------------------- 1 | import time 2 | from functools import wraps 3 | 4 | # A simple decorator 5 | def timethis(func): 6 | @wraps(func) 7 | def wrapper(*args, **kwargs): 8 | start = time.time() 9 | r = func(*args, **kwargs) 10 | end = time.time() 11 | print(end-start) 12 | return r 13 | return wrapper 14 | 15 | # Class illustrating application of the decorator to different kinds of methods 16 | class Spam: 17 | @timethis 18 | def instance_method(self, n): 19 | print(self, n) 20 | while n > 0: 21 | n -= 1 22 | 23 | @classmethod 24 | @timethis 25 | def class_method(cls, n): 26 | print(cls, n) 27 | while n > 0: 28 | n -= 1 29 | 30 | @staticmethod 31 | @timethis 32 | def static_method(n): 33 | print(n) 34 | while n > 0: 35 | n -= 1 36 | 37 | if __name__ == '__main__': 38 | s = Spam() 39 | s.instance_method(10000000) 40 | Spam.class_method(10000000) 41 | Spam.static_method(10000000) 42 | -------------------------------------------------------------------------------- /src/9/avoiding_repetitive_property_methods/example1.py: -------------------------------------------------------------------------------- 1 | def typed_property(name, expected_type): 2 | storage_name = '_' + name 3 | 4 | @property 5 | def prop(self): 6 | return getattr(self, storage_name) 7 | 8 | @prop.setter 9 | def prop(self, value): 10 | if not isinstance(value, expected_type): 11 | raise TypeError('{} must be a {}'.format(name, expected_type)) 12 | setattr(self, storage_name, value) 13 | return prop 14 | 15 | # Example use 16 | class Person: 17 | name = typed_property('name', str) 18 | age = typed_property('age', int) 19 | def __init__(self, name, age): 20 | self.name = name 21 | self.age = age 22 | 23 | if __name__ == '__main__': 24 | p = Person('Dave', 39) 25 | p.name = 'Guido' 26 | try: 27 | p.age = 'Old' 28 | except TypeError as e: 29 | print(e) 30 | 31 | -------------------------------------------------------------------------------- /src/9/capturing_class_attribute_definition_order/example2.py: -------------------------------------------------------------------------------- 1 | # Example of a metaclass that rejects duplicate definitions 2 | 3 | from collections import OrderedDict 4 | 5 | class NoDupOrderedDict(OrderedDict): 6 | def __init__(self, clsname): 7 | self.clsname = clsname 8 | super().__init__() 9 | def __setitem__(self, name, value): 10 | if name in self: 11 | raise TypeError('{} already defined in {}'.format(name, self.clsname)) 12 | super().__setitem__(name, value) 13 | 14 | class OrderedMeta(type): 15 | def __new__(cls, clsname, bases, clsdict): 16 | d = dict(clsdict) 17 | d['_order'] = [name for name in clsdict if name[0] != '_'] 18 | return type.__new__(cls, clsname, bases, d) 19 | 20 | @classmethod 21 | def __prepare__(cls, clsname, bases): 22 | return NoDupOrderedDict(clsname) 23 | 24 | # Example 25 | class A(metaclass=OrderedMeta): 26 | def spam(self): 27 | pass 28 | 29 | print('**** A type error is expected now:') 30 | def spam(self): 31 | pass 32 | 33 | -------------------------------------------------------------------------------- /src/9/defining_a_decorator_that_takes_an_optional_argument/example.py: -------------------------------------------------------------------------------- 1 | from functools import wraps, partial 2 | import logging 3 | 4 | def logged(func=None, *, level=logging.DEBUG, name=None, message=None): 5 | if func is None: 6 | return partial(logged, level=level, name=name, message=message) 7 | 8 | logname = name if name else func.__module__ 9 | log = logging.getLogger(logname) 10 | logmsg = message if message else func.__name__ 11 | @wraps(func) 12 | def wrapper(*args, **kwargs): 13 | log.log(level, logmsg) 14 | return func(*args, **kwargs) 15 | return wrapper 16 | 17 | # Example use 18 | @logged 19 | def add(x, y): 20 | return x + y 21 | 22 | @logged() 23 | def sub(x, y): 24 | return x - y 25 | 26 | @logged(level=logging.CRITICAL, name='example') 27 | def spam(): 28 | print('Spam!') 29 | 30 | if __name__ == '__main__': 31 | import logging 32 | logging.basicConfig(level=logging.DEBUG) 33 | add(2,3) 34 | sub(2,3) 35 | spam() 36 | 37 | -------------------------------------------------------------------------------- /src/9/defining_a_decorator_that_takes_arguments/example.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | import logging 3 | 4 | def logged(level, name=None, message=None): 5 | ''' 6 | Add logging to a function. level is the logging 7 | level, name is the logger name, and message is the 8 | log message. If name and message aren't specified, 9 | they default to the function's module and name. 10 | ''' 11 | def decorate(func): 12 | logname = name if name else func.__module__ 13 | log = logging.getLogger(logname) 14 | logmsg = message if message else func.__name__ 15 | 16 | @wraps(func) 17 | def wrapper(*args, **kwargs): 18 | log.log(level, logmsg) 19 | return func(*args, **kwargs) 20 | return wrapper 21 | return decorate 22 | 23 | # Example use 24 | @logged(logging.DEBUG) 25 | def add(x, y): 26 | return x + y 27 | 28 | @logged(logging.CRITICAL, 'example') 29 | def spam(): 30 | print('Spam!') 31 | 32 | if __name__ == '__main__': 33 | import logging 34 | logging.basicConfig(level=logging.DEBUG) 35 | print(add(2,3)) 36 | spam() 37 | -------------------------------------------------------------------------------- /src/9/defining_a_metaclass_that_takes_optional_arguments/example.py: -------------------------------------------------------------------------------- 1 | # Example of a metaclass that takes optional arguments 2 | 3 | class MyMeta(type): 4 | # Optional 5 | @classmethod 6 | def __prepare__(cls, name, bases, *, debug=False, synchronize=False): 7 | # Custom processing 8 | return super().__prepare__(name, bases) 9 | 10 | # Required 11 | def __new__(cls, name, bases, ns, *, debug=False, synchronize=False): 12 | # Custom processing 13 | return super().__new__(cls, name, bases, ns) 14 | 15 | def __init__(self, name, bases, ns, *, debug=False, synchronize=False): 16 | # Custom processing 17 | super().__init__(name, bases, ns) 18 | 19 | # Examples 20 | class A(metaclass=MyMeta, debug=True, synchronize=True): 21 | pass 22 | 23 | class B(metaclass=MyMeta): 24 | pass 25 | 26 | class C(metaclass=MyMeta, synchronize=True): 27 | pass 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/9/defining_classes_programmatically/example1.py: -------------------------------------------------------------------------------- 1 | # Example of making a class manually from parts 2 | 3 | # Methods 4 | def __init__(self, name, shares, price): 5 | self.name = name 6 | self.shares = shares 7 | self.price = price 8 | 9 | def cost(self): 10 | return self.shares * self.price 11 | 12 | cls_dict = { 13 | '__init__' : __init__, 14 | 'cost' : cost, 15 | } 16 | 17 | # Make a class 18 | import types 19 | 20 | Stock = types.new_class('Stock', (), {}, lambda ns: ns.update(cls_dict)) 21 | 22 | if __name__ == '__main__': 23 | s = Stock('ACME', 50, 91.1) 24 | print(s) 25 | print(s.cost()) 26 | -------------------------------------------------------------------------------- /src/9/defining_classes_programmatically/example2.py: -------------------------------------------------------------------------------- 1 | # An alternative formulation of namedtuples 2 | 3 | import operator 4 | import types 5 | import sys 6 | 7 | def named_tuple(classname, fieldnames): 8 | # Populate a dictionary of field property accessors 9 | cls_dict = { name: property(operator.itemgetter(n)) 10 | for n, name in enumerate(fieldnames) } 11 | 12 | # Make a __new__ function and add to the class dict 13 | def __new__(cls, *args): 14 | if len(args) != len(fieldnames): 15 | raise TypeError('Expected {} arguments'.format(len(fieldnames))) 16 | return tuple.__new__(cls, (args)) 17 | 18 | cls_dict['__new__'] = __new__ 19 | 20 | # Make the class 21 | cls = types.new_class(classname, (tuple,), {}, 22 | lambda ns: ns.update(cls_dict)) 23 | cls.__module__ = sys._getframe(1).f_globals['__name__'] 24 | return cls 25 | 26 | if __name__ == '__main__': 27 | Point = named_tuple('Point', ['x', 'y']) 28 | print(Point) 29 | p = Point(4, 5) 30 | print(len(p)) 31 | print(p.x, p[0]) 32 | print(p.y, p[1]) 33 | try: 34 | p.x = 2 35 | except AttributeError as e: 36 | print(e) 37 | print('%s %s' % p) 38 | -------------------------------------------------------------------------------- /src/9/defining_context_managers_the_easy_way/example1.py: -------------------------------------------------------------------------------- 1 | import time 2 | from contextlib import contextmanager 3 | 4 | @contextmanager 5 | def timethis(label): 6 | start = time.time() 7 | try: 8 | yield 9 | finally: 10 | end = time.time() 11 | print('{}: {}'.format(label, end - start)) 12 | 13 | # Example use 14 | with timethis('counting'): 15 | n = 10000000 16 | while n > 0: 17 | n -= 1 18 | -------------------------------------------------------------------------------- /src/9/defining_context_managers_the_easy_way/example2.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | 3 | @contextmanager 4 | def list_transaction(orig_list): 5 | working = list(orig_list) 6 | yield working 7 | orig_list[:] = working 8 | 9 | # Example 10 | if __name__ == '__main__': 11 | items = [1, 2, 3] 12 | with list_transaction(items) as working: 13 | working.append(4) 14 | working.append(5) 15 | print(items) 16 | try: 17 | with list_transaction(items) as working: 18 | working.append(6) 19 | working.append(7) 20 | raise RuntimeError('oops') 21 | except RuntimeError as e: 22 | print(e) 23 | 24 | print(items) 25 | 26 | -------------------------------------------------------------------------------- /src/9/defining_decorators_as_classes/example1.py: -------------------------------------------------------------------------------- 1 | # Example of defining a decorator as a class 2 | import types 3 | from functools import wraps 4 | 5 | class Profiled: 6 | def __init__(self, func): 7 | wraps(func)(self) 8 | self.ncalls = 0 9 | 10 | def __call__(self, *args, **kwargs): 11 | self.ncalls += 1 12 | return self.__wrapped__(*args, **kwargs) 13 | 14 | def __get__(self, instance, cls): 15 | if instance is None: 16 | return self 17 | else: 18 | return types.MethodType(self, instance) 19 | 20 | # Example 21 | 22 | @Profiled 23 | def add(x, y): 24 | return x + y 25 | 26 | class Spam: 27 | @Profiled 28 | def bar(self, x): 29 | print(self, x) 30 | 31 | if __name__ == '__main__': 32 | print(add(2,3)) 33 | print(add(4,5)) 34 | print('ncalls:', add.ncalls) 35 | 36 | s = Spam() 37 | s.bar(1) 38 | s.bar(2) 39 | s.bar(3) 40 | print('ncalls:', Spam.bar.ncalls) 41 | -------------------------------------------------------------------------------- /src/9/defining_decorators_as_classes/example2.py: -------------------------------------------------------------------------------- 1 | # Reformulation using closures and function attributes 2 | from functools import wraps 3 | 4 | def profiled(func): 5 | ncalls = 0 6 | @wraps(func) 7 | def wrapper(*args, **kwargs): 8 | nonlocal ncalls 9 | ncalls += 1 10 | return func(*args, **kwargs) 11 | wrapper.ncalls = lambda: ncalls 12 | return wrapper 13 | 14 | # Example 15 | 16 | @profiled 17 | def add(x, y): 18 | return x + y 19 | 20 | class Spam: 21 | @profiled 22 | def bar(self, x): 23 | print(self, x) 24 | 25 | if __name__ == '__main__': 26 | print(add(2,3)) 27 | print(add(4,5)) 28 | print('ncalls:', add.ncalls()) 29 | 30 | s = Spam() 31 | s.bar(1) 32 | s.bar(2) 33 | s.bar(3) 34 | print('ncalls:', Spam.bar.ncalls()) 35 | -------------------------------------------------------------------------------- /src/9/defining_decorators_as_classes/example3.py: -------------------------------------------------------------------------------- 1 | # Reformulation using closures and function attributes 2 | # This example tests the composability of decorators 3 | 4 | import time 5 | from functools import wraps 6 | 7 | def timethis(func): 8 | @wraps(func) 9 | def wrapper(*args, **kwargs): 10 | start = time.time() 11 | r = func(*args, **kwargs) 12 | end = time.time() 13 | print(func.__name__, end - start) 14 | return r 15 | return wrapper 16 | 17 | def profiled(func): 18 | ncalls = 0 19 | @wraps(func) 20 | def wrapper(*args, **kwargs): 21 | nonlocal ncalls 22 | ncalls += 1 23 | return func(*args, **kwargs) 24 | wrapper.ncalls = lambda: ncalls 25 | return wrapper 26 | 27 | # Example 28 | 29 | @profiled 30 | def add(x, y): 31 | return x + y 32 | 33 | class Spam: 34 | @profiled 35 | def bar(self, x): 36 | print(self, x) 37 | 38 | @timethis 39 | @profiled 40 | def countdown(n): 41 | while n > 0: 42 | n -= 1 43 | 44 | if __name__ == '__main__': 45 | print(add(2,3)) 46 | print(add(4,5)) 47 | print('ncalls:', add.ncalls()) 48 | 49 | s = Spam() 50 | s.bar(1) 51 | s.bar(2) 52 | s.bar(3) 53 | print('ncalls:', Spam.bar.ncalls()) 54 | 55 | countdown(100000) 56 | countdown(10000000) 57 | print(countdown.ncalls()) 58 | -------------------------------------------------------------------------------- /src/9/defining_decorators_as_part_of_a_class/example1.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | class A: 4 | # Decorator as an instance method 5 | def decorator1(self, func): 6 | @wraps(func) 7 | def wrapper(*args, **kwargs): 8 | print('Decorator 1') 9 | return func(*args, **kwargs) 10 | return wrapper 11 | 12 | # Decorator as a class method 13 | @classmethod 14 | def decorator2(cls, func): 15 | @wraps(func) 16 | def wrapper(*args, **kwargs): 17 | print('Decorator 2') 18 | return func(*args, **kwargs) 19 | return wrapper 20 | 21 | # Example 22 | # As an instance method 23 | a = A() 24 | 25 | @a.decorator1 26 | def spam(): 27 | pass 28 | 29 | # As a class method 30 | @A.decorator2 31 | def grok(): 32 | pass 33 | 34 | spam() 35 | grok() 36 | -------------------------------------------------------------------------------- /src/9/defining_decorators_as_part_of_a_class/example2.py: -------------------------------------------------------------------------------- 1 | # Property example 2 | 3 | class Person: 4 | first_name = property() 5 | @first_name.getter 6 | def first_name(self): 7 | return self._first_name 8 | 9 | @first_name.setter 10 | def first_name(self, value): 11 | if not isinstance(value, str): 12 | raise TypeError('Expected a string') 13 | self._first_name = value 14 | 15 | p = Person() 16 | p.first_name = 'Dave' 17 | print(p.first_name) 18 | -------------------------------------------------------------------------------- /src/9/disassembling_python_byte_code/example.py: -------------------------------------------------------------------------------- 1 | # Example of manual disassembly of bytecode 2 | 3 | import opcode 4 | 5 | def generate_opcodes(codebytes): 6 | extended_arg = 0 7 | i = 0 8 | n = len(codebytes) 9 | while i < n: 10 | op = codebytes[i] 11 | i += 1 12 | if op >= opcode.HAVE_ARGUMENT: 13 | oparg = codebytes[i] + codebytes[i+1]*256 + extended_arg 14 | extended_arg = 0 15 | i += 2 16 | if op == opcode.EXTENDED_ARG: 17 | extended_arg = oparg * 65536 18 | continue 19 | else: 20 | oparg = None 21 | yield (op, oparg) 22 | 23 | # Example 24 | def countdown(n): 25 | while n > 0: 26 | print('T-minus', n) 27 | n -= 1 28 | print('Blastoff!') 29 | 30 | for op, oparg in generate_opcodes(countdown.__code__.co_code): 31 | print(op, opcode.opname[op], oparg) 32 | -------------------------------------------------------------------------------- /src/9/enforcing_coding_conventions_in_classes/example1.py: -------------------------------------------------------------------------------- 1 | # A metaclass that disallows mixed case identifier names 2 | 3 | class NoMixedCaseMeta(type): 4 | def __new__(cls, clsname, bases, clsdict): 5 | for name in clsdict: 6 | if name.lower() != name: 7 | raise TypeError('Bad attribute name: ' + name) 8 | return super().__new__(cls, clsname, bases, clsdict) 9 | 10 | class Root(metaclass=NoMixedCaseMeta): 11 | pass 12 | 13 | class A(Root): 14 | def foo_bar(self): # Ok 15 | pass 16 | 17 | print('**** About to generate a TypeError') 18 | class B(Root): 19 | def fooBar(self): # TypeError 20 | pass 21 | -------------------------------------------------------------------------------- /src/9/enforcing_coding_conventions_in_classes/example2.py: -------------------------------------------------------------------------------- 1 | # Using a metaclass to issue warnings about signature mismatches 2 | 3 | from inspect import signature 4 | import logging 5 | 6 | class MatchSignaturesMeta(type): 7 | def __init__(self, clsname, bases, clsdict): 8 | super().__init__(clsname, bases, clsdict) 9 | sup = super(self, self) 10 | for name, value in clsdict.items(): 11 | if name.startswith('_') or not callable(value): 12 | continue 13 | # Get the previous definition (if any) and compare the signatures 14 | prev_dfn = getattr(sup,name,None) 15 | if prev_dfn: 16 | prev_sig = signature(prev_dfn) 17 | val_sig = signature(value) 18 | if prev_sig != val_sig: 19 | logging.warning('Signature mismatch in %s. %s != %s', 20 | value.__qualname__, str(prev_sig), str(val_sig)) 21 | 22 | # Example 23 | class Root(metaclass=MatchSignaturesMeta): 24 | pass 25 | 26 | class A(Root): 27 | def foo(self, x, y): 28 | pass 29 | 30 | def spam(self, x, *, z): 31 | pass 32 | 33 | # Class with redefined methods, but slightly different signatures 34 | class B(A): 35 | def foo(self, a, b): 36 | pass 37 | 38 | def spam(self,x,z): 39 | pass 40 | -------------------------------------------------------------------------------- /src/9/executing_code_with_local_side_effects/example.py: -------------------------------------------------------------------------------- 1 | def test(): 2 | a = 13 3 | loc = locals() 4 | exec('b = a + 1') 5 | b = loc['b'] 6 | print(b) # --> 14 7 | 8 | def test1(): 9 | x = 0 10 | exec('x += 1') 11 | print(x) # --> 0 12 | 13 | def test2(): 14 | x = 0 15 | loc = locals() 16 | print('before:', loc) 17 | exec('x += 1') 18 | print('after:', loc) 19 | print('x =', x) 20 | 21 | def test3(): 22 | x = 0 23 | loc = locals() 24 | print(loc) 25 | exec('x += 1') 26 | print(loc) 27 | locals() 28 | print(loc) 29 | 30 | def test4(): 31 | a = 13 32 | loc = { 'a' : a } 33 | glb = { } 34 | exec('b = a + 1', glb, loc) 35 | b = loc['b'] 36 | print(b) 37 | 38 | if __name__ == '__main__': 39 | print(':::: Running test()') 40 | test() 41 | 42 | print(':::: Running test1()') 43 | test1() 44 | 45 | print(':::: Running test2()') 46 | test2() 47 | 48 | print(':::: Running test3()') 49 | test3() 50 | 51 | print(':::: Running test4()') 52 | test4() 53 | 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /src/9/initializing_class_members_at_definition_time/example.py: -------------------------------------------------------------------------------- 1 | import operator 2 | 3 | class StructTupleMeta(type): 4 | def __init__(cls, *args, **kwargs): 5 | super().__init__(*args, **kwargs) 6 | for n, name in enumerate(cls._fields_): 7 | setattr(cls, name, property(operator.itemgetter(n))) 8 | 9 | class StructTuple(tuple, metaclass=StructTupleMeta): 10 | _fields_ = [] 11 | def __new__(cls, *args): 12 | if len(args) != len(cls._fields_): 13 | raise ValueError('{} arguments required'.format(len(cls._fields_))) 14 | return super().__new__(cls,args) 15 | 16 | # Examples 17 | class Stock(StructTuple): 18 | _fields_ = ['name', 'shares', 'price'] 19 | 20 | class Point(StructTuple): 21 | _fields_ = ['x', 'y'] 22 | 23 | if __name__ == '__main__': 24 | s = Stock('ACME', 50, 91.1) 25 | print(s) 26 | print(s[0]) 27 | print(s.name) 28 | print(s.shares * s.price) 29 | try: 30 | s.shares = 23 31 | except AttributeError as e: 32 | print(e) 33 | -------------------------------------------------------------------------------- /src/9/monkeypatching_class_definitions/example.py: -------------------------------------------------------------------------------- 1 | def log_getattribute(cls): 2 | # Get the original implementation 3 | orig_getattribute = cls.__getattribute__ 4 | 5 | # Make a new definition 6 | def new_getattribute(self, name): 7 | print('getting:', name) 8 | return orig_getattribute(self, name) 9 | 10 | # Attach to the class and return 11 | cls.__getattribute__ = new_getattribute 12 | return cls 13 | 14 | # Example use 15 | @log_getattribute 16 | class A: 17 | def __init__(self,x): 18 | self.x = x 19 | def spam(self): 20 | pass 21 | 22 | if __name__ == '__main__': 23 | a = A(42) 24 | print(a.x) 25 | a.spam() 26 | -------------------------------------------------------------------------------- /src/9/parsing_and_analyzing_python_source/example1.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | class CodeAnalyzer(ast.NodeVisitor): 4 | def __init__(self): 5 | self.loaded = set() 6 | self.stored = set() 7 | self.deleted = set() 8 | def visit_Name(self, node): 9 | if isinstance(node.ctx, ast.Load): 10 | self.loaded.add(node.id) 11 | elif isinstance(node.ctx, ast.Store): 12 | self.stored.add(node.id) 13 | elif isinstance(node.ctx, ast.Del): 14 | self.deleted.add(node.id) 15 | 16 | # Sample usage 17 | if __name__ == '__main__': 18 | # Some python code 19 | code = ''' 20 | for i in range(10): 21 | print(i) 22 | del i 23 | ''' 24 | # Parse into an AST 25 | top = ast.parse(code, mode='exec') 26 | 27 | # Feed the AST to analyze name usage 28 | c = CodeAnalyzer() 29 | c.visit(top) 30 | print('Loaded:', c.loaded) 31 | print('Stored:', c.stored) 32 | print('Deleted:', c.deleted) 33 | -------------------------------------------------------------------------------- /src/9/preserving_function_metadata_when_writing_decorators/example.py: -------------------------------------------------------------------------------- 1 | import time 2 | from functools import wraps 3 | 4 | def timethis(func): 5 | ''' 6 | Decorator that reports the execution time. 7 | ''' 8 | @wraps(func) 9 | def wrapper(*args, **kwargs): 10 | start = time.time() 11 | result = func(*args, **kwargs) 12 | end = time.time() 13 | print(func.__name__, end-start) 14 | return result 15 | return wrapper 16 | 17 | if __name__ == '__main__': 18 | @timethis 19 | def countdown(n:int): 20 | ''' 21 | Counts down 22 | ''' 23 | while n > 0: 24 | n -= 1 25 | 26 | countdown(100000) 27 | print('Name:', countdown.__name__) 28 | print('Docstring:', repr(countdown.__doc__)) 29 | print('Annotations:', countdown.__annotations__) 30 | -------------------------------------------------------------------------------- /src/9/unwrapping_a_decorator/example.py: -------------------------------------------------------------------------------- 1 | # Example of unwrapping a decorator 2 | 3 | from functools import wraps 4 | 5 | def decorator1(func): 6 | @wraps(func) 7 | def wrapper(*args, **kwargs): 8 | print('Decorator 1') 9 | return func(*args, **kwargs) 10 | return wrapper 11 | 12 | def decorator2(func): 13 | @wraps(func) 14 | def wrapper(*args, **kwargs): 15 | print('Decorator 2') 16 | return func(*args, **kwargs) 17 | return wrapper 18 | 19 | @decorator1 20 | @decorator2 21 | def add(x, y): 22 | return x + y 23 | 24 | # Calling wrapped function 25 | print(add(2,3)) 26 | 27 | # Calling original function 28 | print(add.__wrapped__(2,3)) 29 | -------------------------------------------------------------------------------- /src/9/using_metaclasses_to_control_instance_creation/example1.py: -------------------------------------------------------------------------------- 1 | # example1.py 2 | # 3 | # Not allowing direct instantiation 4 | 5 | class NoInstances(type): 6 | def __call__(self, *args, **kwargs): 7 | raise TypeError("Can't instantiate directly") 8 | 9 | class Spam(metaclass=NoInstances): 10 | @staticmethod 11 | def grok(x): 12 | print('Spam.grok') 13 | 14 | if __name__ == '__main__': 15 | try: 16 | s = Spam() 17 | except TypeError as e: 18 | print(e) 19 | 20 | Spam.grok(42) 21 | -------------------------------------------------------------------------------- /src/9/using_metaclasses_to_control_instance_creation/example2.py: -------------------------------------------------------------------------------- 1 | # example2.py 2 | # 3 | # Singleton 4 | 5 | class Singleton(type): 6 | def __init__(self, *args, **kwargs): 7 | self.__instance = None 8 | super().__init__(*args, **kwargs) 9 | 10 | def __call__(self, *args, **kwargs): 11 | if self.__instance is None: 12 | self.__instance = super().__call__(*args, **kwargs) 13 | return self.__instance 14 | else: 15 | return self.__instance 16 | 17 | class Spam(metaclass=Singleton): 18 | def __init__(self): 19 | print('Creating Spam') 20 | 21 | if __name__ == '__main__': 22 | a = Spam() 23 | b = Spam() 24 | print(a is b) 25 | 26 | -------------------------------------------------------------------------------- /src/9/using_metaclasses_to_control_instance_creation/example3.py: -------------------------------------------------------------------------------- 1 | # example3.py 2 | # 3 | # Cached instances 4 | 5 | import weakref 6 | 7 | class Cached(type): 8 | def __init__(self, *args, **kwargs): 9 | super().__init__(*args, **kwargs) 10 | self.__cache = weakref.WeakValueDictionary() 11 | 12 | def __call__(self, *args): 13 | if args in self.__cache: 14 | return self.__cache[args] 15 | else: 16 | obj = super().__call__(*args) 17 | self.__cache[args] = obj 18 | return obj 19 | 20 | class Spam(metaclass=Cached): 21 | def __init__(self, name): 22 | print('Creating Spam({!r})'.format(name)) 23 | self.name = name 24 | 25 | if __name__ == '__main__': 26 | a = Spam('foo') 27 | b = Spam('bar') 28 | print('a is b:', a is b) 29 | c = Spam('foo') 30 | print('a is c:', a is c) 31 | 32 | 33 | -------------------------------------------------------------------------------- /《Python Cookbook》第三版中文v2.0.0.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xxg1413/python-cookbook/1a4e432261ac6f08d10bb970ab5d22ebacdd210e/《Python Cookbook》第三版中文v2.0.0.pdf --------------------------------------------------------------------------------