├── .gitignore ├── 1_process ├── code_demo │ ├── 1.4 │ │ ├── euid_backup.py │ │ ├── euid_cp │ │ ├── euid_cp.c │ │ ├── euid_cp_snapshot.jpg │ │ ├── euid_non_root_snapshot.jpg │ │ └── euid_perm.sh │ └── 1.5 │ │ └── get_fd_file.py ├── 其一杀不死的shell子进程.md ├── 其三pgid是个什么鬼.md ├── 其二裸用类os.system函数的原罪.md ├── 其五为啥打不开新文件.md └── 其四一次性踩透uid_euid_gid_egid的坑坑洼洼.md ├── 2_filesystem └── 其一目录也是个文件.md ├── LICENSE ├── README.md └── weixin_nosign.png /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | -------------------------------------------------------------------------------- /1_process/code_demo/1.4/euid_backup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -* 3 | # Copyright: See LICENSE for details. 4 | # Authors: Guannan Ma (@mythmgn), 5 | """ 6 | backup files 7 | """ 8 | 9 | from __future__ import print_function 10 | import os 11 | import time 12 | 13 | print('euid is {0}'.format(os.geteuid())) 14 | 15 | if os.geteuid() == 0: 16 | print('start to copy under root') 17 | time.sleep(2) 18 | print('end copying things') 19 | print('drop privileges from root') 20 | else: 21 | print('non-root, euid {0} will exit'.format(os.geteuid())) 22 | 23 | # vi:set tw=0 ts=4 sw=4 nowrap fdm=indent 24 | -------------------------------------------------------------------------------- /1_process/code_demo/1.4/euid_cp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mythmgn/awesome_py_traps/e927746f4d982b4165467df46f07be4f291d5c31/1_process/code_demo/1.4/euid_cp -------------------------------------------------------------------------------- /1_process/code_demo/1.4/euid_cp.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Author: Guannan Ma 3 | * @mythmgn 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | int isRunUnderRoot(){ 13 | if(geteuid() != 0){ 14 | return 0; 15 | }else{ 16 | return 1; 17 | } 18 | } 19 | 20 | int runNewProcess(char *pFolder, char* pName){ 21 | pid_t child; 22 | if( (child=fork()) < 0 ){ 23 | exit(EXIT_FAILURE); 24 | } 25 | else if (child ==0){ 26 | /*child process*/ 27 | FILE *in; 28 | char pExe[2056]; 29 | char buff[1024]; 30 | int status; 31 | sprintf(pExe, "cd %s && %s", pFolder, pName); 32 | setuid(geteuid()); 33 | if(!(in = popen(pExe, "r"))){ 34 | exit(EXIT_FAILURE); 35 | } 36 | while(fgets(buff, sizeof(buff), in)!=NULL){ 37 | printf("%s", buff); 38 | } 39 | status = pclose(in); 40 | if (0!=status) 41 | exit(WEXITSTATUS(status)); 42 | else 43 | exit(EXIT_SUCCESS); 44 | }else{ 45 | /*parent process*/ 46 | int status; 47 | if( child!=waitpid(child, &status, 0) || WEXITSTATUS(status)!=EXIT_SUCCESS){ 48 | fprintf(stderr, "Error when check waitpid and WEXITSTATUS\n"); 49 | return WEXITSTATUS(status); 50 | } 51 | return EXIT_SUCCESS; 52 | } 53 | } 54 | 55 | int main(int argc, char **argv){ 56 | if(0==isRunUnderRoot()){ 57 | fprintf(stderr,"Elavator does not run under +S attribute. Exiting....\n"); 58 | return EXIT_FAILURE; 59 | } 60 | exit(runNewProcess("./", "env python ./euid_backup.py")); 61 | } 62 | -------------------------------------------------------------------------------- /1_process/code_demo/1.4/euid_cp_snapshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mythmgn/awesome_py_traps/e927746f4d982b4165467df46f07be4f291d5c31/1_process/code_demo/1.4/euid_cp_snapshot.jpg -------------------------------------------------------------------------------- /1_process/code_demo/1.4/euid_non_root_snapshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mythmgn/awesome_py_traps/e927746f4d982b4165467df46f07be4f291d5c31/1_process/code_demo/1.4/euid_non_root_snapshot.jpg -------------------------------------------------------------------------------- /1_process/code_demo/1.4/euid_perm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # ########################################################################## 3 | # Author: Guannan Ma 4 | 5 | # Arguments: 6 | # None 7 | # 8 | # Returns: 9 | # succ: 0 10 | # fail: not 0 11 | # ########################################################################## 12 | sudo rm -f ./euid_cp 13 | sudo gcc euid_cp.c -o euid_cp 14 | # 设置文件owner为root 15 | sudo chown root euid_cp 16 | # 设置a. 非root只读 b. 增加执行权限 17 | sudo chmod 755 euid_cp 18 | # 设置stick bit, 执行euid_cp即可短暂获取root 权限, 执行任务 19 | sudo chmod +s euid_cp 20 | -------------------------------------------------------------------------------- /1_process/code_demo/1.5/get_fd_file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -* 3 | # Copyright - See LICENSE for details 4 | # Authors: Guannan Ma @mythmgn 5 | """ 6 | :description: 7 | say something here 8 | """ 9 | 10 | from __future__ import print_function 11 | import os 12 | import sys 13 | 14 | _TOP = os.path.dirname(os.path.abspath(__file__)) + '/../' 15 | sys.path.insert(0, _TOP) 16 | 17 | from cup.shell import oper 18 | from cup.res import linux 19 | 20 | 21 | def test_get_fd(): 22 | """test get fd""" 23 | # get_pid 第一个参数是启动程序的路径, 第二个是匹配参数, 比如mysqld 24 | arrow_pid = oper.get_pid('/home/maguannan/.jumbo/bin/', 'arrow_agent') 25 | print('pid {0}'.format(arrow_pid)) 26 | process = linux.Process(arrow_pid) 27 | # 通过 get_open_files 函数获得打开文件数目 28 | print('opened fd {0}'.format(process.get_open_files())) 29 | # linux.Process(pid) 对象可以获取所有该进程的有效信息, 比如open fd, cpu时间 30 | 31 | # vi:set tw=0 ts=4 sw=4 nowrap fdm=indent -------------------------------------------------------------------------------- /1_process/其一杀不死的shell子进程.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: "[知识积累]Python踩坑之旅其一杀不死的Shell子进程" 4 | categories: 知识积累 5 | tags: Python踩坑之旅,知识积累 6 | toc: true 7 | comments: true 8 | --- 9 | [TOC] 10 | 11 | # Python踩坑之旅其一杀不死的Shell子进程 12 | 13 | ## 1.1 踩坑案例 14 | 15 | 踩坑的程序是个常驻的Agent类管理进程, 包括但不限于如下类型的任务在执行: 16 | 17 | - a. 多线程的网络通信包处理 18 | - 和控制Master节点交互 19 | - 有固定Listen端口 20 | - b. 定期作业任务, 通过subprocess.Pipe执行shell命令 21 | - c. etc 22 | 23 | 发现坑的过程很有意思: 24 | 25 | - **a.重启Agent发现Port被占用了** 26 | - => 立刻**想到可能进程没被杀死, 是不是停止脚本出问题** 27 | - => 排除发现不是, Agent进程确实死亡了 28 | - => 通过 `netstat -tanop|grep port_number` 发现端口确实有人占用 29 | - => 调试环境, **直接杀掉占用进程了之, 错失首次发现问题的机会** 30 | - **b.问题**在一段时间后**重现**， 重启后Port还是被占用 31 | - 定位问题出现在一个叫做xxxxxx.sh的脚本, 该脚本占用了Agent使用的端口 32 | - => 奇了怪了, 一个xxx.sh脚本使用这个奇葩Port干啥(大于60000的Port, 有兴趣的砖友可以想下为什么Agent默认使用6W+的端口) 33 | - => review**该脚本并没有进行端口监听的代码** 34 | - 一拍脑袋, **c.进程共享了父进程资源**了 35 | - => 溯源该脚本,发现确实是Agent启动的任务中的脚本之一 36 | - => 问题基本定位, 该脚本属于Agent调用的脚本 37 | - => 该Agent继承了Agent原来的资源FD, 也就是这个port 38 | - => 虽然该脚本由于超时被动触发了terminate机制, 但terminate并没有干掉这个子进程 39 | - => 该脚本进程的父进程(ppid) 被重置为了1 40 | - **d.问题****出在脚本进程超时kill逻辑** 41 | 42 | ## 1.2 填坑解法 43 | 44 | 通过代码review, 找到shell具体执行的库代码如下: 45 | 46 | ```python 47 | self._subpro = subprocess.Popen( 48 | cmd, shell=True, stdout=subprocess.PIPE, 49 | stderr=subprocess.PIPE, 50 | preexec_fn=_signal_handle 51 | ) 52 | # 重点是shell=True ! 53 | ``` 54 | 55 | 把上述代码改为: 56 | 57 | ```python 58 | self._subpro = subprocess.Popen( 59 | cmd.split(), stdout=subprocess.PIPE, 60 | stderr=subprocess.PIPE, preexec_fn=_signal_handle 61 | ) 62 | # 重点是去掉了shell=True 63 | ``` 64 | 65 | ## 1.3 坑位分析 66 | 67 | Agent会在一个新创建的threading线程中执行这段代码, 如果线程执行时间超时(xx seconds), 会调用 ```self._subpro.terminate()```终止该脚本. 68 | 69 | 表面正常: 70 | 71 | - 启用新线程执行该脚本 72 | - 如果出现问题,执行超时防止hang住其他任务执行调用terminate杀死进程 73 | 74 | 深层问题: 75 | 76 | - Python 2.7.x中subprocess.Pipe 如果shell=True, 会默认把相关的pid设置为shell(sh/bash/etc)本身(执行命令的shell父进程), 并非执行cmd任务的那个进程 77 | - 子进程由于会复制父进程的opened FD表, 导致即使被杀死, 依然保留了拥有这个Listened Port FD 78 | 79 | 这样虽然杀死了shell进程(未必死亡, 可能进入defunct状态), 但实际的执行进程确活着. 于是`1.1`中的坑就被结实的踩上了. 80 | 81 | ## 1.4 坑后扩展 82 | 83 | ### 1.4.1 扩展知识 84 | 85 | 本节扩展知识包括二个部分: 86 | 87 | - Linux系统中, 子进程一般会继承父进程的哪些信息 88 | - Agent这种常驻进程选择>60000端口的意义 89 | 90 | 扩展知识留到下篇末尾讲述, 感兴趣的可以自行搜索 91 | 92 | ### 1.4.1 技术关键字 93 | 94 | - Linux系统进程 95 | - Linux随机端口选择 96 | - 程序多线程执行 97 | - Shell执行 98 | 99 | ## 1.5 填坑总结 100 | 101 | 1. 子进程会继承父进程的资源信息 102 | 2. 如果只kill某进程的父进程, 集成了父进程资源的子进程会继续占用父进程的资源不释放, 包括但不限于 103 | 104 | - listened port 105 | - opened fd 106 | - etc 107 | 108 | 3. Python Popen使用上, shell的bool状态决定了进程kill的逻辑, 需要根据场景选择使用方式 -------------------------------------------------------------------------------- /1_process/其三pgid是个什么鬼.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: "Python 踩坑之旅进程篇其三pgid是个什么鬼" 4 | categories: 知识积累 5 | tags: Python踩坑之旅,知识积累 6 | toc: true 7 | comments: true 8 | --- 9 | [TOC] 10 | 11 | # Python 踩坑之旅进程篇其三pgid是个什么鬼 12 | 13 | | 代码示例支持| 14 | |-| 15 | |平台: Centos 6.3| 16 | |Python: 2.7.14 | 17 | |Github上搜索 **CUP**
![cup](./cup_github.png)| 18 | 19 | ## 1.1 踩坑案例 20 | 21 | pid, ppid是大家比较常见的术语, 代表进程号,父进程号. 但pgid是个什么鬼? 22 | 23 | 了解pgid之前, 我们先复习下: 24 | 25 | - [进程篇其一](2019-05-05-Python踩坑之旅进程篇其一杀不死的shell子进程.md) 26 | - 里面场景是: 一个进程通过`os.system`或者`Popen`家族启动子进程 27 | - 后通过杀死父进程的方式无法杀死它的连带子进程 28 | - 我们通过其他方式进行了解决 29 | 30 | 这个场景还有个后续就是: 31 | 32 | - 如果这个子进程还有孙子怎么办? 33 | - 它还有孙子的孙子怎么办? 34 | 35 | 这个就是今天我们遇到的坑, 怎么处理孙子进程. 大家注意, 不仅是Python会遇到这个问题, 其他语言包括 Shell 都一样会遇到这种"孙子"进程怎么进程异常处理的问题. 36 | 37 | ## 1.2 填坑解法 38 | 39 | 本期的坑位解法其实有两种, 第一种比较暴力, 简称穷尽搜索孙子法. 40 | 41 | a. 穷尽搜索孙子法, 代码示例 42 | 43 | 关键点: 44 | 45 | - 使用cup.res.linux中的Process类, 获得该进程所有的子孙进程 46 | - 使用kill方法全部杀死 47 | 48 | ```python 49 | from cup.res import linux 50 | pstatus = linux.Process(pid) 51 | for child in pstatus.children(recursive=True): 52 | os.kill(child, signal.SIGKILL) 53 | ``` 54 | 55 | b. 获得该进程的 PGID, 进行 kill 操作 56 | 57 | b1. 先讲个 shell 操作的做法, 使用ps 获取进程的**pgid**, 注意**不是**pid 58 | 59 | ```bash 60 | # 以mysqld为例, 注意 pgid 项 61 | ps -e -o uid,pid,gid,pgid,cmd|grep mysql 62 | ``` 63 | 64 | 结果: 65 | 66 | - 注意其中第三列, 该进程和子进程都使用了同样的pgid **9779** 67 | 68 | 9790 0 9779 /bin/sh /usr/bin/mysqld_safe --datadir=/home/maguannan/mysql/mysql/.... 69 | 70 | 10171 501 9779 /home/maguannan/bin/mysqld --basedir=/home/maguannan/mysql/.... 71 | 72 | - 通过`kill -9 -9779`的方式可以杀死该pgid底下的所有**子孙**进程 73 | 74 | b2. 在讲 Python 里的处理方式 75 | 76 | ```python 77 | import os 78 | import signal 79 | from cup.res import linux 80 | pstatus = linux.Process(pid) 81 | os.killpg(pstatus.getpgid(), signal.SIGKILL) 82 | ``` 83 | 84 | ## 1.3 坑位分析 85 | 86 | **进程组特性** 87 | 88 | a. 在*unix 编程中, 进程组(`man getpgid`)概念是个很重要但容易被忽略的内容 89 | 90 | - 进程组ID (pgid) 标记了一系列相关的进程 91 | - 进程组有一个组长进程, 一般组长进程 ID 等于进程组 ID 92 | - 进程组只要任一进程存在, 进程组就存在. 进程组存在与否与组长死活无关 93 | - 可以通过setpgid方式设置一个进程 pgid 94 | - 一个进程只能为自己或者子进程设置进程组 id 95 | - 子进程一旦执行了exec函数, 它就不能改变子进程的进程组 id 96 | 97 | b. 进程组内的所有成员会收到来自相同的信号 98 | 99 | 引用 wikipedia 原文: 100 | ``` 101 | a process group is used to control the distribution of a signal; when a signal is directed to a process group, the signal is delivered to each process that is a member of the group. 102 | ``` 103 | 104 | **坑位解决** 105 | 106 | - 由于进程组拥有以上的特性, 进程组内的进程可以被当做相同的处理单元 107 | - 默认子进程与父进程拥有同样的进程组 108 | - 组内每个进程收到相同的信号) 109 | - 使用kill发送信号 SIGKILL 即可满足杀死所有子**孙**进程的目的 110 | 111 | ### 1.4.1 技术关键字 112 | 113 | - pgid 进程组 114 | - pid, ppid 进程ID, 父进程ID 115 | 116 | ## 下期坑位预告 117 | 118 | - 踩坑之旅进程篇其四: 一次性踩透uid, euid, gid, egid的坑坑洼洼 119 | 120 | 121 | --- 122 | Life is short. We use Python. -------------------------------------------------------------------------------- /1_process/其二裸用类os.system函数的原罪.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: "[知识积累]Python踩坑之旅其二裸用os.sytem的原罪" 4 | categories: 知识积累 5 | tags: Python踩坑之旅,知识积累 6 | toc: true 7 | comments: true 8 | --- 9 | [TOC] 10 | 11 | # [知识积累]Python踩坑之旅其二裸用os.sytem的原罪 12 | 13 | ## 1.1 踩坑案例 14 | 15 | 今天的坑不仅包括裸用os.system还包括裸用相关的家族: 16 | 17 | - os.popen 18 | - subprocess家族 19 | - subprocess.call 20 | - subprocess.Popen 21 | - subprocess.run 22 | - commands家族 (py2.6后已不推荐使用， depreciated. Py3删除) 23 | - commands.getstatusoutput 24 | 25 | 26 | 这些坑是新同学非常容易踩，而且 code review 过程中容易漏掉: 27 | 28 | **[1] 长期运行 Service 中裸用以函数家族** 29 | 30 | - 裸用以上 shell 执行家族可能出现script 执行 hang 住进而 hang 住逻辑执行线程，长时间积累进而占满所有执行线程而服务宕机情况 31 | - 大内存消耗 service fork 子进程直接执行 script 32 | - 如果该 script hang 住 33 | - 并且原进程内存进行频繁修改（或者堆积修改， 虽然有 Copy-On-Write技术），但由于内存巨大，依然有内存风险 34 | 35 | **[2] 自动化测试中裸用以上函数家族而不加以保护** 36 | 37 | - 单个 case 如果执行 script 脚本 hang 住会导致 hang 掉整个case 集 38 | - 不设计 case 超时机制导致case 集合运行时间不可控 39 | 40 | ## 1.2 填坑解法 41 | 42 | 1. 支持超时 kill 策略，禁止任何情况下的 shell 执行裸用家族函数 43 | 44 | 提供一个作者的代码参考: https://github.com/baidu/CUP/blob/master/cup/shell/oper.py 45 | 46 | ```python 47 | from cup import shell 48 | shellexec = shell.ShellExec() 49 | # timeout=None will block the execution until it finishes 50 | shellexec.run('/bin/ls', timeout=None) 51 | # timeout>=0 will open non-blocking mode 52 | # The process will be killed if the cmd timeouts 53 | shellexec.run(cmd='/bin/ls', timeout=100) 54 | ``` 55 | 56 | 见ShellExec类的run函数 57 | 58 | 2. 内存消耗型服务/进程， 长期运行服务进程避免fork 进程执行 shell 命令 59 | 60 | ## 1.3 坑位分析 61 | 62 | 建议看下第二章节关于进程和子进程继承类信息，script使用上述家族进行执行时，采用了启动一个子进程的方式 63 | 64 | ### 1.4.1 技术关键字 65 | 66 | - os.system家族 67 | - subprocess家族 68 | 69 | ## 1.5 填坑总结 70 | 71 | Shell执行是个非常常见的操作，所以很多同学特别是新同学，在使用过程中经常不注意而随意使用。 裸用一时爽，进程死亡火葬场 72 | 73 | # 2. 前坑回顾 74 | 75 | ## 2.1 Linux中, 子进程拷贝父进程哪些信息 76 | 77 | - 先说与父进程不同的 78 | - pid, ppid 79 | - memory locks 80 | - tms_utime、tms_stime、tms_cutime、tms_ustime 81 | - pending signals 82 | - [semaphore adjustments](https://stackoverflow.com/questions/34452654/what-child-does-not-inherit-semaphore-adjustments-from-its-parent-semop2-mea) 83 | - file lock 84 | - pending alarms 85 | 86 | 参考资料来源: 87 | 88 | - Linux Programmer's Manual ( `man fork` ) 89 | - CentOS release 6.3 (Final) 90 | - Linux Kernel 2.6.32 91 | 92 | ```bash 93 | 94 | fork() creates a new process by duplicating the calling process. The new process, referred to as the child, is an exact duplicate of the calling process, referred to as the parent, except for the follow- 95 | ing points: 96 | 97 | * The child has its own unique process ID, and this PID does not match the ID of any existing process group (setpgid(2)). 98 | 99 | * The child's parent process ID is the same as the parent's process ID. 100 | 101 | * The child does not inherit its parent's memory locks (mlock(2), mlockall(2)). 102 | 103 | * Process resource utilizations (getrusage(2)) and CPU time counters (times(2)) are reset to zero in the child. 104 | 105 | * The child's set of pending signals is initially empty (sigpending(2)). 106 | 107 | * The child does not inherit semaphore adjustments from its parent (semop(2)). 108 | 109 | * The child does not inherit record locks from its parent (fcntl(2)). 110 | 111 | * The child does not inherit timers from its parent (setitimer(2), alarm(2), timer_create(2)). 112 | 113 | * The child does not inherit outstanding asynchronous I/O operations from its parent (aio_read(3), aio_write(3)), nor does it inherit any asynchronous I/O contexts from its parent (seeio_setup(2)). 114 | 115 | The process attributes in the preceding list are all specified in POSIX.1-2001. The parent and child also differ with respect to the following Linux-specific process attributes: 116 | 117 | * The child does not inherit directory change notifications (dnotify) from its parent (see the description of F_NOTIFY in fcntl(2)). 118 | 119 | * The prctl(2) PR_SET_PDEATHSIG setting is reset so that the child does not receive a signal when its parent terminates. 120 | 121 | * Memory mappings that have been marked with the madvise(2) MADV_DONTFORK flag are not inherited across a fork(). 122 | 123 | * The termination signal of the child is always SIGCHLD (see clone(2)). 124 | 125 | ``` 126 | 127 | 在说继承、拷贝父进程的 128 | 129 | - 包括 130 | - 内部数据空间 131 | - 堆栈 132 | - 用户 ID、组 ID、eid 有效用户 id、有效组 id、用户 id 标志和设置组 id 标志 133 | - 进程组 id 134 | - 会话 id 135 | - 终端 136 | - 当前目录、根目录 137 | - 文件模式屏蔽字 138 | - 信号屏蔽设置 139 | - 打开文件描述符 140 | - 环境 141 | - 共享存储段 142 | - 存储映射 143 | - 资源限制 144 | 145 | 此外 146 | 147 | - 在父进程创建 (fork) 出一个子进程过程中, 为了加速, 会使用叫做 copy-on-write 的技术. 148 | - 这项技术在存储软件领域也经常使用 149 | - [附上一个关于它的讨论，点击查看](https://unix.stackexchange.com/questions/58145/how-does-copy-on-write-in-fork-handle-multiple-fork) 150 | 151 | ## 2.2 Agent常驻进程选择>60000端口的意义 152 | 153 | 在 Linux 系统中， 一般系统会自动替程序选择端口连接到用户指定的目的端口， 而这个端口范围是提前设定好的， 比如作者的 centos： 154 | 155 | ```bash 156 | $ cat /proc/sys/net/ipv4/ip_local_port_range 157 | 10000 60000 158 | ``` 159 | 160 | - 选择 60000 以上的端口可以避免冲突 161 | - [附上一篇讨论该ip_local_port_range的文章，欢迎查看](https://www.cnblogs.com/solohac/p/4154180.html) -------------------------------------------------------------------------------- /1_process/其五为啥打不开新文件.md: -------------------------------------------------------------------------------- 1 | [TOC] 2 | 3 | 4 | | 代码示例支持| 5 | |-| 6 | |平台: Centos 6.3| 7 | |Python: 2.7.14 | 8 | |代码示例: 菜单 - Python踩坑指南代码示例| 9 | 10 | ## 1.1 踩坑案例 11 | 12 | 长期运行的daemon进程或者socket测试类进程, 经常遇到的坑是: 13 | 14 | `IOError: [Errno 24] Too many open files` 15 | 16 | 即进程遇到 IO 错误, 无法打开更多的文件. 17 | 18 | ## 1.2 填坑和分析 19 | 20 | 一般从两个方面入手: 21 | 22 | ### 1.2.1 从程序优化入手 23 | 24 | - 检查文件打开是否遵循了"谁打开谁关闭"原则 25 | - 文件是否存在关闭泄露 26 | 27 | a. 谁打开谁关闭是个普适的原则: 28 | 29 | - 只有逻辑设计者自己最熟悉 30 | - 哪些文件 FD 需要一直维持打开状态 31 | - 哪些文件直到某个事件发生后关闭 32 | 33 | - 短暂的文件读写打开推荐使用 pythonic 的 with statement 34 | 35 | ```python 36 | # with 语法会在生命周期后自动关闭打开的文件 FD 37 | with open('xxxx_path.file', 'w') as fhandle: 38 | fhandle.dosth() 39 | ``` 40 | 41 | b. 检查文件 FD 是否存在泄漏 42 | 43 | 系统设计阶段一般会预估系统总体可打开的 FD 情况. 当出现如下情况时可能出现了泄漏 BUG 44 | 45 | - 外围监控系统发现该进程 FD 大量突破了设计预估 46 | - 打开 FD 增长趋势异常 47 | - 一般随着业务增加, FD 会线性增长, 但有限度和规律 48 | - 如果增长曲线不停的出现陡峭增长且在业务低峰期也如此可能出现了泄露 49 | 50 | 51 | Python 基础库 CUP 提供对进程打开 FD 的支持, 详见示例代码. 52 | 53 | 54 | ### 1.2.2 从资源软硬限入手 55 | 56 | - 了解系统的资源软硬限制 57 | - 检查进程可打开的FD是否突破了系统限制 58 | - 长期运行的 daemon 进程尤其注意 59 | 60 | 以 `Centos 6.3 Linux系统`为例, 查看 /etc/security/limits.conf 获得系统软硬限资源 61 | 62 | ``` bash 63 | * soft nofile 10240 64 | * hard nofile 10240 65 | ``` 66 | 67 | 其中, 用户不能突破系统的硬线 `hard nofile limit`. 68 | 69 | 用户也可以通过 shell 命令 ulimit -n 来限定该 shell 启动的所有进程的 nofile 70 | 71 | - 当然非 root 用户是不能突破系统硬线的 72 | - 用户为了进程控制, 可以设定nofile更小 73 | 74 | `ulimit -a` 可以查看当前用户被设定的限制, 示例: 75 | 76 | ```bash 77 | [test@agent1 ~]$ ulimit -a 78 | core file size (blocks, -c) 0 79 | ....... 80 | open files (-n) 10240 81 | ..... 82 | virtual memory (kbytes, -v) unlimited 83 | file locks (-x) unlimited 84 | ``` 85 | 86 | 87 | ### 1.4.1 技术关键字 88 | 89 | - Open FD 90 | - 资源 Soft limit / Hard limit 91 | 92 | 93 | ## 下期坑位预告 94 | 95 | - PyDaemon 进程长什么样 96 | 97 | --- 98 | Life is short. We use Python. -------------------------------------------------------------------------------- /1_process/其四一次性踩透uid_euid_gid_egid的坑坑洼洼.md: -------------------------------------------------------------------------------- 1 | [TOC] 2 | 3 | | 代码示例支持| 4 | |-| 5 | |平台: Centos 6.3| 6 | |Python: 2.7.14 | 7 | |代码示例: code_demo目录| 8 | 9 | ## 1.1 踩坑案例 10 | 11 | `请注意该案例为虚拟案例, 为了展示uid euid等功能性, 不能直接应用于生产环境` 12 | 13 | 小明是个服务器管理员, 他从老管理员手里接手了一个非常繁琐的运维工作: 短暂授权root 账号给不同的 team 接口人运行备份任务 14 | 15 | 该运维任务有几个特点: 16 | 17 | 1. 任务需且仅需运行在 root 下 18 | - root 账号只能短暂授权给各个小组 19 | - 通过账号管理平台, 提前申请一段时间的临时密码 20 | - 将临时密码提供给小组接口人 21 | - 时间超时后密码自动变更 22 | 2. 不同 team 分时使用, 无法并发使用 23 | 24 | 小明非常烦躁, 为了填上这个坑, 他调研了填坑解法. 25 | 26 | ## 1.2 填坑解法 27 | 28 | 填坑解法满足: 29 | 30 | - 短时出借权限 31 | - (在权限范围内)执行该任务时才能使用 root 权限 32 | - 做完任务立即失去 root 权限 33 | - 权限范围必须清晰 34 | - 能做什么 35 | - 能做数据备份 36 | - 不能做什么 37 | - 除了数据备份其他什么都不在 root 权限下做 38 | 39 | **具体做法:** 40 | 41 | 1. 利用c/c++程序出借部分 root 权限 42 | 43 | - 该c程序限定执行的备份操作为 python 代码 euid_backup.py 44 | ```c 45 | int main(int argc, char **argv){ 46 | if(0==isRunUnderRoot()){ 47 | fprintf(stderr,"does not run under +S attribute. Exiting....\n"); 48 | return EXIT_FAILURE; 49 | } 50 | exit(runNewProcess("./", "/usr/bin/backup/secured/python /usr/bin/backup/secured/euid_backup.py")); 51 | } 52 | ``` 53 | - euid_backup.py owner 为 root, 非 root 用户不准更改备份操作内容 54 | 55 | 2. 为生成的执行文件euid_cp及euid_backup.py 设置root权限借用 56 | 57 | ```bash 58 | sudo rm -f /usr/bin/backup/secured/euid_cp 59 | sudo gcc euid_cp.c -o /usr/bin/backup/secured/euid_cp 60 | # 设置文件owner为root, 非root用户无法更改执行内容 61 | sudo chown root /usr/bin/backup/secured/* 62 | # 设置a. 非root只读 b. 增加执行权限 63 | sudo chmod 755 /usr/bin/backup/secured/euid_cp 64 | # 设置stick bit, 执行euid_cp即可短暂获取root 权限, 执行任务 65 | sudo chmod +s /usr/bin/backup/secured/euid_cp 66 | ``` 67 | 68 | - /usr/bin/backup/secured/euid_backup.py Python 代码执行具体的备份任务 69 | 70 | ```python 71 | from __future__ import print_function 72 | import os 73 | import time 74 | 75 | print('euid is {0}'.format(os.geteuid())) 76 | 77 | if os.geteuid() == 0: 78 | print('start to copy under root') 79 | print('do some operations here') 80 | time.sleep(2) 81 | print('end copying things') 82 | print('drop privileges from root') 83 | else: 84 | print('non-root, euid {0} will exit'.format(os.geteuid())) 85 | ``` 86 | 87 | 运行试验: 88 | 1. 通过 /usr/bin/backup/secured/euid_cp 执行, 可以在非 root 下执行 root 权限才能执行的备份任务(euid_backup.py) 89 | 90 | 2. 直接执行备份任务(/usr/bin/backup/secured/euid_backup.py) 会失败, 没有权限 91 | 92 | ## 1.3 坑位分析 93 | 94 | 1. uid / euid / suid 是什么 95 | 96 | - uid 是用户的 userid 97 | - 登陆后, 在不切换用户情况下 uid 一般不变 98 | - euid 是用户的有效 id 99 | - euid 在正常执行情况下一般等于uid 100 | - euid 一般决定了用户对系统中存储介质的access (访问) 权限 101 | - suid (saved uid) 是文件在被访问过程中的短暂切换用户euid的属性设置 102 | - 简单来说, suid让本没有权限的用户可以短暂访问这些资源 103 | - suid 在执行过程中进行了权限切换 104 | - 执行之初, 切换到这个saved uid(文中为 root) 105 | - fork执行过程中, euid == suid 106 | - 执行完成后, euid 在切换回 uid 107 | 108 | 2. gid, egid 等同理, [*]uid的判断优先 109 | 110 | ## 1.4 技术关键字 111 | 112 | - uid euid suid 113 | - gid egid sgid 114 | 115 | ## 1.5 坑后思考 116 | 117 | 1. 为什么本文没有直接对euid_backup.py文件进行设置+s操作, 而是用可执行的c/c++程序做执行器 118 | 119 | 2. Linux 系统里的passwd 程序是否也是这个原理? 它跟哪些文件/命令相关 120 | 121 | 122 | P.S. 示例代码经 py-cn 圈朋友提醒作下进一步说明: 123 | 124 | 1. 该示例代码和场景更多的是为了展示 suid 的特性 125 | 2. 如果真实环境要进行如上的备份任务有多种实现选择, 该项并非最优 126 | - 比如任务执行器实现 (普通用户下, 提交任务到 root 启动的执行器), 该执行器鉴权后在 root 下执行具体备份任务 127 | - 分配 sudo 权限到不同的用户组进行权限控制 128 | 129 | 欢迎大家对内容\示例代码等提更多的建议. 130 | 131 | ## 下期坑位预告 132 | 133 | 进程篇其五之眼花缭乱的进程间通信 134 | 135 | --- 136 | Life is short. We use Python. 137 | -------------------------------------------------------------------------------- /2_filesystem/其一目录也是个文件.md: -------------------------------------------------------------------------------- 1 | [TOC] 2 | 3 | | 代码示例支持| 4 | |-| 5 | |平台: Mac OS| 6 | |Python: 2.7.10 | 7 | |**代码示例:**
- wx: 菜单 - Python踩坑指南代码示例
- github 见code_demo| 8 | 9 | ## 1.1 案例 10 | 11 | 这期案例讲的是Unix-Like系统中, 常听到的一句话: 目录也是个文件 或者 everything is a file. 12 | 刚接触 Linux 文件系统的同学有时候听到这个很懵, 目录怎么是个文件呢?目录不应该是内部包含文件的载体么? 13 | 14 | ## 1.2 分析 15 | 16 | 分析主要从2个方面展开: 17 | 18 | - `ls` 实际是使用大量文件系统标准接口实现的结果, 是处理过后的用户**程序** 19 | - 从文件系统的组织结构来看穿数据存储和读写方式 20 | 21 | --- 22 | 如果大家已习惯了 Linux 系统中 `ls` 命令 (有时候由于`alias` 存在, 实际是`ls --color`), 容易产生一种错觉:`文件夹和文件这不是天然的被区分为不同的类别了吗?` 23 | 24 | - 比如蓝色的文件夹? 25 | - 黑色的文件?` 26 | 27 | 实际不是这样子的, `ls` `mkdir` `touch` 一类的文件系统操作命令其实是通过调用文件系统接口实现的用户态程序, 你自己利用python也可以实现一个一摸一样的. 28 | 29 | 我们来看一些使用 python 访问文件系统的简单例子: 30 | 31 | ```python 32 | from __future__ import print_function 33 | import os 34 | 35 | # 简单文件写 36 | with open('./test', 'w+') as fhandle: 37 | fhandle.write('test\n') 38 | 39 | # 创建文件夹 40 | 41 | dirname = os.path.abspath('./test_dir') 42 | if not os.path.exists(dirname): 43 | os.makedirs(dirname) 44 | 45 | for ind in range(0, 10): 46 | with open('{0}/test_file_{1}'.format(dirname, ind), 'w+') as fhandle: 47 | fhandle.write('1') 48 | dname = '{0}/test_dir_{1}'.format(dirname, ind) 49 | if not os.path.exists(dname): 50 | os.mkdir(dname) 51 | 52 | # 读文件夹 53 | for obj in os.listdir(dirname): 54 | objpath = os.path.join(dirname, obj) 55 | if os.path.isfile(objpath): 56 | print('{0} is a file'.format(objpath)) 57 | elif os.path.isdir(objpath): 58 | print('{0} is a dir'.format(objpath)) 59 | ``` 60 | 61 | 因此, 大家理解 `ls` 类耳熟能详的 Linux 命令是经过代码实现的用户程序, 如果你想且有时间完成可以实现一个 python 版 的`ls` 62 | 63 | --- 64 | 65 | 更进一步的说, 对文件或者文件夹的操作本质上是用户层的代码实现调用了系统相关的接口. 这代表着文件夹和文件对系统来讲, 就是数据组织上的不同 （数据结构的不同）. 那数据或者文件数目是怎么进行组织的? 66 | 67 | 想了解这个问题就要先了解 Linux 系统上的文件存储层次, 以在 Linux 上挂载的文件系统进程读写为例: 68 | 69 | - 最上层, 用户的程序进程 Process, 通过调用类似open write close 等通用系统函数读写所在挂载目录的文件 70 | 71 | - 中间 Kernel VFS (Virtual Filesystem, 虚拟文件系统) 72 | - 市面上主流的文件系统并不少, 为了让上层应用不关心如何读写这些内部实现各异的文件系统, Kernel 实现了虚拟文件系统 73 | - 虚拟文件系统包含一系列的标准; 74 | - 为了方便理解, 可以简化理解为提供了一系列读写接口标准 75 | - 上层用户应用使用下层的文件系统不需要关心你是哪个文件系统, 我只要挂载好你到我的系统就能使用标准接口读写 76 | - 底层, Kernel 内核, 各个设备厂家不同的VFS实现嵌入 kernel 中以支持具体的读写等操作 77 | - 物理介质层 (块设备等), 真正的硬件设备层 78 | 79 | 而我们要聚焦到 VFS 这层来看, 因为它: 80 | 81 | 1. 屏蔽下层不同设备厂商数据存储实现 82 | 2. 抽象并统一了数据存储接口 83 | 84 | 只要明白了它如何组织文件/文件夹, 基本上就明白了人们常说 everything is a file 的意思. 85 | 86 | 具体到数据结构上, 要看虚拟文件系统上规定了针对文件系统的4类数据结构: 87 | 88 | - superblock 89 | - 用来存储挂载的文件系统的元信息 (比如inode 数目等) 90 | - 简化理解起来就像是文件系统的索引系统, superblock 决定了如下几个数据结构的分布 91 | - inode, 用来存储具体数据的单元 (包括人们通常理解的 file 实体和文件夹) 92 | - dentry, directory entry, 用来描述文件夹信息 93 | - file obj, 进程打开文件描述 94 | 95 | 对VFS来讲, 无论是存储了具体字节数据的文件, 还是文件夹, 本质都是个 inode 作元信息描述的逻辑结构. 96 | 97 | 无非文件夹不包括具体数据信息描述, 但包含一些指针 (指向该文件夹包含的一系列数据文件或者子文件夹). 相反, 一个指向数据的 inode 不包含子目录或文件们. 98 | 99 | ## 1.3 扩展 100 | 101 | 基本了解了文件系统的组织方式后, 留几个问题大家给大家做扩展思考? 102 | 103 | 1. 通常我们说一个文件系统满盘了, 可能扩展哪几种满盘? 104 | 2. 我只创建文件夹但不创建文件, 文件系统会满盘么? 105 | 3. 文件系统有时候出现错乱, 需要进行 fs check, 这个时候可能是什么坏掉了? 106 | 107 | ## 1.4 技术关键字 108 | 109 | **关键字** 110 | 111 | 1. Linux Kernel 112 | 2. VFS 113 | 3. Inode / Dentry / Superblock 114 | 115 | **一些可以参考的资料** 116 | 117 | - 什么是 superblock, inode, dentry的网友讨论: [https://unix.stackexchange.com/questions/4402/what-is-a-superblock-inode-dentry-and-a-file](https://unix.stackexchange.com/questions/4402/what-is-a-superblock-inode-dentry-and-a-file) 118 | - Inode structure and Inode table [http://140.120.7.21/LinuxKernel/LinuxKernel/node17.html](http://140.120.7.21/LinuxKernel/LinuxKernel/node17.html) 119 | - Dentry 数据结构介绍和描述 [http://books.gigatux.nl/mirror/kerneldevelopment/0672327201/ch12lev1sec7.html](http://books.gigatux.nl/mirror/kerneldevelopment/0672327201/) 120 | 121 | ## 下期预告 122 | 123 | `文件系统篇 Umask 到底影响了谁` 124 | 125 | --- 126 | 127 | - 水平有限, 有问题欢迎指正. 128 | - Life is short. We use Python -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Awesome Py Traps 的初衷** 2 | 1. 梳理自己的 python 知识树, 总结在使用 python 编程过程中的经验教训 3 | 2. 让 Unix-Like 系统知识 和 python代码实现/试验 相结合, 互相印证 4 | 5 | 6 | - 文档更新: 7 | - 欢迎感兴趣的朋友一起写稿填坑 8 | - 发起人预计每周更新一篇, 持续至 100 篇 9 | - 已有章目但未有链接的代表还在撰写 pending list 中 10 | 11 | - 发起人简介 12 | - 某厂专有云测试开发负责人, 十年分布式存储\计算项目经验 13 | - Python 开源基础库负责人和主要作者: https://github.com/baidu/CUP 14 | - 微信公众号: 程序员的梦呓指南 ![](weixin_nosign.png) 15 | 16 | 17 | # 1. Python 踩坑之旅 18 | 19 | ## 进程篇 20 | 21 | - [1. 杀不死的 Shell 子进程.md](./1_process/其一杀不死的shell子进程.md) 22 | - [2. 裸用 os.system 类函数的原罪.md](./1_process/其二裸用类os.system函数的原罪.md) 23 | - [3. 进程 pgid 是个什么鬼.md](./1_process/其三pgid是个什么鬼.md) 24 | - [4. 一次性踩透 uid_euid_gid_egid 的坑坑洼洼.md](./1_process/其四一次性踩透uid_euid_gid_egid的坑坑洼洼.md) 25 | - [5. 为啥打不开新文件](./1_process/其五为啥打不开新文件.md) 26 | - [6. PyDaemon 进程长什么样]() 27 | 28 | ## 线程篇 29 | 30 | - [Python 线程是摆设吗 (GIL 问题)] 31 | - [线程是越多越好还是越少越好] 32 | 33 | 34 | ## 日志篇 35 | 36 | ## Pythonic 特色篇 37 | 38 | 39 | # 2. 系统知识 坑洼之旅 40 | 41 | ## 文件系统篇 42 | 43 | - **[文件夹也是 File Object](./2_filesystem/其一目录也是个文件.md) <-------- (新增2019.9.20)** 44 | - [Umask 到底影响了谁] 45 | - [一次性搞清楚文件目录权限判定] 46 | - [慎用软硬链接] 47 | - [Readonly 数据恢复小窍门] 48 | 49 | 50 | ## 输入输出IO篇 51 | 52 | - [你不知道的File Open] 53 | - [读写的几种口味] 54 | - [论游标的重要性] 55 | - [谁打开谁关闭] 56 | - [谁动了我的数据缓存] 57 | - [为什么你要关心编码] 58 | - [临时文件也疯狂] 59 | 60 | ## 网络篇 61 | 62 | - [服务器端为什么非得绑定端口] 63 | - [tcp 为什么不一定比 udp 好] 64 | - [链接池越大越好么] 65 | - [非阻塞一定比阻塞好吗] 66 | - [epoll的几种坏味道] 67 | - [上下文管理在管理什么] 68 | - [心跳机制解决了什么问题] 69 | - [网络传输数据为什么要序列化和反序列化] 70 | - [该死的wait_time_out] 71 | 72 | 73 | -------------------------------------------------------------------------------- /weixin_nosign.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mythmgn/awesome_py_traps/e927746f4d982b4165467df46f07be4f291d5c31/weixin_nosign.png --------------------------------------------------------------------------------