├── .gitignore ├── README.md ├── cmd_sh ├── check_code.sh ├── create_env.sh ├── dlbase │ ├── dlbase.sh │ ├── dlboot.sh │ ├── dlfile.sh │ ├── dlgit.sh │ ├── dllog.sh │ ├── dlroot.sh │ └── dluser.sh ├── dlbase_test │ ├── dlbase_test.sh │ ├── dlgit_test.sh │ └── dllog_test.sh ├── etc.sh ├── install_pybase.sh ├── pylint.conf └── remove_env.sh ├── gpu_memory_log.py ├── gpu_run.sh ├── gpu_test.ipynb ├── gpu_test.py └── run.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pytorch_gpu_memory 2 | pytorch gpu memory check 3 | 4 | ## how to use gpu_memory_log 5 | 6 | ```python 7 | import torch 8 | from gpu_memory_log import gpu_memory_log 9 | 10 | dtype = torch.float 11 | N, D_in, H, D_out = 64, 1000, 100, 10 12 | 13 | device = torch.device("cuda") 14 | x = torch.randn(N, D_in, device=device, dtype=dtype) 15 | y = torch.randn(N, D_out, device=device, dtype=dtype) 16 | 17 | w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True) 18 | w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True) 19 | 20 | learning_rate = 1e-6 21 | gpu_memory_log() 22 | for t in range(500): 23 | y_pred = x.mm(w1).clamp(min=0).mm(w2) 24 | loss = (y_pred - y).pow(2).sum() 25 | #print(t, loss.item()) 26 | loss.backward() 27 | with torch.no_grad(): 28 | w1 -= learning_rate * w1.grad 29 | w2 -= learning_rate * w2.grad 30 | 31 | w1.grad.zero_() 32 | w2.grad.zero_() 33 | 34 | print(loss.item()) 35 | gpu_memory_log() 36 | ``` 37 | yes, just put the code gpu_memory_log() , where you want to see the gpu memory status. 38 | so easy. 39 | 40 | ## how to run 41 | 42 | you can choose how to run. 43 | this will run gpu_test.py 44 | ```shell 45 | bash gpu_run.sh 46 | ``` 47 | 48 | output 49 | ```shell 50 | LINE:27, FUNC:, FILE:gpu_test.py, TIME:2019-06-02 17:16:23.288399, CONTENT: 51 | [tensor: 1 * Size:(100, 10) | Memory: 0.0038 M | ] 52 | [tensor: 1 * Size:(64, 1000) | Memory: 0.2441 M | ] 53 | [tensor: 1 * Size:(64, 10) | Memory: 0.0024 M | ] 54 | [tensor: 1 * Size:(1000, 100) | Memory: 0.3814 M | ] 55 | memory_allocated:1.457520 Mb 56 | max_memory_allocated:1.457520 Mb 57 | memory_cached:2.000000 Mb 58 | max_memory_cached:2.000000 Mb 59 | Used Memory:9983.625000 Mb 60 | Free Memory:1185.625000 Mb 61 | Total Memory:11169.250000 Mb 62 | 0.0032593868672847748 63 | LINE:41, FUNC:, FILE:gpu_test.py, TIME:2019-06-02 17:16:23.791826, CONTENT: 64 | [tensor: 1 * Size:(100, 10) | Memory: 0.0038 M | ] 65 | [tensor: 1 * Size:() | Memory: 3.8146 M | ] 66 | [tensor: 1 * Size:(64, 1000) | Memory: 0.2441 M | ] 67 | [tensor: 2 * Size:(64, 10) | Memory: 0.0048 M | ] 68 | [tensor: 1 * Size:(1000, 100) | Memory: 0.3814 M | ] 69 | memory_allocated:1.846191 Mb 70 | max_memory_allocated:2.252930 Mb 71 | memory_cached:4.000000 Mb 72 | max_memory_cached:4.000000 Mb 73 | Used Memory:9989.625000 Mb 74 | Free Memory:1179.625000 Mb 75 | Total Memory:11169.250000 Mb 76 | ``` 77 | 78 | note: the size is different from nvidia-smi. 79 | 80 | this will run gpu_test.ipynb, just open the jupyter 81 | ```shell 82 | bash run.sh 83 | ``` 84 | 85 | -------------------------------------------------------------------------------- /cmd_sh/check_code.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : check_code.sh 4 | ## Create date : 2018-11-25 15:57 5 | ## Modified date : 2019-03-20 13:23 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | export basedir=$(dirname "$realpath") 13 | export filename=$(basename "$realpath") 14 | export PATH=$PATH:$basedir/dlbase 15 | export PATH=$PATH:$basedir/dlproc 16 | #base sh file 17 | . dlbase.sh 18 | #function sh file 19 | . etc.sh 20 | 21 | # source $env_path/py2env/bin/activate 22 | # pip freeze > python2_requiements.txt 23 | # deactivate 24 | 25 | source $env_path/py3env/bin/activate 26 | pylint --rcfile=pylint.conf main.py 27 | pylint --rcfile=pylint.conf etc.py 28 | pylint --rcfile=pylint.conf func.py 29 | pylint --rcfile=pylint.conf show.py 30 | pylint --rcfile=pylint.conf train_graph.py 31 | pylint --rcfile=pylint.conf eval_graph.py 32 | pylint --rcfile=pylint.conf rocstories_dataset.py 33 | 34 | pylint --rcfile=pylint.conf ./openai_gpt/gpt_double_heads_model.py 35 | pylint --rcfile=pylint.conf ./openai_gpt/gpt_lm_head_model.py 36 | 37 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/gpt_cache.py 38 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/gpt_config.py 39 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/gpt_opt.py 40 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/gpt_pretrained_model.py 41 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/gpt_token.py 42 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/gpt_transformer.py 43 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/load_model.py 44 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/model_base.py 45 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/model_block.py 46 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/model_lm_head.py 47 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/model_multiple_choice_head.py 48 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/token_basic.py 49 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/token_bpe.py 50 | pylint --rcfile=pylint.conf ./openai_gpt/gpt/base_gpt/token_tf_torch.py 51 | 52 | pip freeze > python3_requiements.txt 53 | deactivate 54 | -------------------------------------------------------------------------------- /cmd_sh/create_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : create_env.sh 4 | ## Create date : 2018-11-25 15:54 5 | ## Modified date : 2019-06-02 17:08 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | export basedir=$(dirname "$realpath") 13 | export filename=$(basename "$realpath") 14 | export PATH=$PATH:$basedir/dlbase 15 | export PATH=$PATH:$basedir/dlproc 16 | #base sh file 17 | . dlbase.sh 18 | #function sh file 19 | . etc.sh 20 | #asumming installed virtualenv　 21 | 22 | rm -rf $env_path 23 | mkdir $env_path 24 | cd $env_path 25 | 26 | # virtualenv -p /usr/bin/python2 py2env 27 | # source $env_path/py2env/bin/activate 28 | # pip install Pillow 29 | # #pip install tornado 30 | # #pip install mysqlclient 31 | 32 | # deactivate 33 | 34 | ##virtualenv -p /usr/bin/python3 py3env 35 | virtualenv --no-site-packages -p /usr/bin/python3 py3env 36 | #virtualenv --no-site-packages -p /usr/local/bin/python3 py3env 37 | source $env_path/py3env/bin/activate 38 | pip install Pillow 39 | pip install tornado 40 | #pip install mysqlclient 41 | #3.5 现在还不支持MySQLdb 42 | pip install PyMySQL 43 | pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade spicy 44 | pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade scikit-learn 45 | pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade torch 46 | #pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade tensorflow 47 | pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade torchvision 48 | pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade pandas 49 | 50 | pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade jupyter 51 | pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade nltk 52 | pip install nvidia-ml-py3 53 | 54 | deactivate 55 | -------------------------------------------------------------------------------- /cmd_sh/dlbase/dlbase.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$dlbase" ]; then 3 | return 4 | fi 5 | 6 | export dlbase="dlbase.sh" 7 | 8 | . dllog.sh 9 | . dlfile.sh 10 | . dlgit.sh 11 | . dlroot.sh 12 | . dlboot.sh 13 | -------------------------------------------------------------------------------- /cmd_sh/dlbase/dlboot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$dlboot" ]; then 3 | return 4 | fi 5 | 6 | export dlboot="dlboot.sh" 7 | 8 | . dllog.sh 9 | . dlfile.sh 10 | 11 | function dlboot_create_auto_start_file(){ 12 | local user_name="$1" 13 | local file_name="$2" 14 | local file_path="/home/$user_name" 15 | local file_full_name="$file_path/$file_name" 16 | $DLLOG_INFO $file_full_name 17 | dlfile_check_is_have_file $file_full_name 18 | 19 | if [[ $? -eq 0 ]]; then 20 | echo \#!/bin/sh > $file_full_name 21 | chmod +x $file_full_name 22 | dlboot_set_shell_auto_run "$user_name" "$file_full_name" 23 | else 24 | echo \#!/bin/sh > $file_full_name 25 | chmod +x $file_full_name 26 | $DLLOG_INFO "$fill_full_name exist auto run file" 27 | fi 28 | } 29 | 30 | function dlboot_set_shell_auto_run(){ 31 | local user_name="$1" 32 | local file_full_name="$2" 33 | echo su $user_name -c \"$file_full_name\" >> /etc/rc.d/rc.local 34 | chmod +x /etc/rc.d/rc.local 35 | } 36 | 37 | function dlboot_set_progress_auto_start(){ 38 | local user_name="$1" 39 | local file_name="$2" 40 | local content="$3" 41 | local file_path="/home/$user_name" 42 | local file_full_name="$file_path/$file_name" 43 | dlfile_check_is_have_file $file_full_name 44 | if [[ $? -eq 0 ]]; then 45 | dlboot_create_auto_start_file "$user_name" "$file_name" 46 | echo -e $content >> $file_full_name 47 | else 48 | echo -e $content >> $file_full_name 49 | fi 50 | } 51 | 52 | if [ -n "$BASH_SOURCE" -a "$BASH_SOURCE" != "$0" ] 53 | then 54 | echo 55 | else 56 | echo self run 57 | fi 58 | -------------------------------------------------------------------------------- /cmd_sh/dlbase/dlfile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$dlfile" ]; then 3 | return 4 | fi 5 | 6 | export dlfile="dlfile" 7 | 8 | . dllog.sh 9 | 10 | function dlfile_create_dir(){ 11 | mkdir $1 12 | if [[ $? -eq 0 ]]; then 13 | echo create $1 folder sucess; 14 | return 0 15 | else 16 | echo create $1 folder fail; 17 | return 1 18 | fi 19 | } 20 | 21 | function dlfile_touch_file(){ 22 | if [ ! -f "$1" ]; then 23 | touch "$1" 24 | fi 25 | } 26 | 27 | function dlfile_check_is_have_dir(){ 28 | if [ ! -d "$1" ]; then 29 | # echo $1 folder do not exist 30 | return 0 31 | else 32 | # echo $1 folder exist 33 | return 1 34 | fi 35 | } 36 | 37 | function dlfile_check_is_have_file(){ 38 | if [ ! -f "$1" ]; then 39 | $DLLOG_DEBUG "$1 file do not exit" 40 | return 0 41 | else 42 | $DLLOG_DEBUG "$1 file exit" 43 | return 1 44 | fi 45 | } 46 | 47 | function dlfile_try_create_dir(){ 48 | dlfile_check_is_have_dir $1 49 | if [[ $? -eq 0 ]]; then 50 | dlfile_create_dir $1 51 | fi 52 | } 53 | 54 | function dlfile_scp_file(){ 55 | local copy_comand=$1 56 | local file_host_password=$2 57 | local file_copy_full_name=$3 58 | 59 | dlfile_check_is_have_file "$file_copy_full_name" 60 | 61 | if [[ $? -eq 0 ]]; then 62 | dlfile_scp_file_without_check "$copy_comand" "$file_host_password" 63 | else 64 | $DLLOG_INFO "$3 have the file" 65 | fi 66 | } 67 | 68 | function dlfile_scp_file_without_check(){ 69 | local copy_comand=$1 70 | local file_host_password=$2 71 | # should install expect 72 | # yum -y install expect 73 | 74 | expect -c " 75 | spawn scp -r $copy_comand 76 | expect { 77 | \"*assword\" {set timeout 300; send \"$file_host_password\r\";} 78 | \"yes/no\" {send \"yes\r\"; exp_continue;} 79 | } 80 | expect eof" 81 | } 82 | 83 | -------------------------------------------------------------------------------- /cmd_sh/dlbase/dlgit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$dlgit" ]; then 3 | return 4 | fi 5 | 6 | export dlgit="dlgit.sh" 7 | 8 | . dllog.sh 9 | 10 | function dlgit_down(){ 11 | local down_url="$1" 12 | local folder="$2" 13 | 14 | dlfile_check_is_have_dir $folder 15 | 16 | if [[ $? -eq 0 ]]; then 17 | dlfile_try_create_dir "$folder" 18 | git clone $down_url "$folder" 19 | else 20 | $DLLOG_INFO "$1 git had been clone" 21 | fi 22 | } 23 | 24 | function dlgit_clone_git(){ 25 | local down_url="$1" 26 | local folder="$2" 27 | dlgit_down $down_url $folder 28 | } 29 | 30 | if [ -n "$BASH_SOURCE" -a "$BASH_SOURCE" != "$0" ] 31 | then 32 | echo 33 | else 34 | echo fun self 35 | fi 36 | -------------------------------------------------------------------------------- /cmd_sh/dlbase/dllog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$dllog" ]; then 3 | return 4 | fi 5 | 6 | export dllog="dllog.sh" 7 | 8 | . dlfile.sh 9 | 10 | log_path="./shlog" 11 | 12 | DLLOG_DEBUG="dllog_log debug $LINENO $0" 13 | DLLOG_INFO="dllog_log info $LINENO $0" 14 | DLLOG_WARNING="dllog_log warning $LINENO $0" 15 | DLLOG_ERROR="dllog_log error $LINENO $0" 16 | 17 | function dllog_log(){ 18 | local now_date=`date "+%Y-%m-%d %H:%M:%S"` 19 | local user_name=`whoami` 20 | local log_level=$1 21 | local line_number=$2 22 | local file_name=$3 23 | local content=$4 24 | local now_day=`date "+%Y-%m-%d"` 25 | 26 | local log_content="LEVEL:$log_level\tUSER:$user_name\tFILE:$file_name\tLINE:$line_number\tFUNC:${FUNCNAME[1]}\tDATE:$now_date CONTENT:$content" 27 | local log_debug_content="LINE:$line_number\tFUNC:${FUNCNAME[1]}\tFILE:$file_name\tCONTENT:$content" 28 | local log_write_content="LEVEL:$log_level USER:$user_name FILE:$file_name LINE:$line_number FUNC:${FUNCNAME[1]} DATE:$now_date CONTENT:$content" 29 | 30 | if [ "$log_level" == "error" ] ; then 31 | echo -e "\033[31m$log_content\033[0m" 32 | elif [ "$log_level" == "info" ] ; then 33 | echo -e "\033[32m$log_debug_content\033[0m" 34 | elif [ "$log_level" == "warning" ] ; then 35 | echo -e "\033[33m$log_content\033[0m" 36 | else 37 | echo -e "\033[32m$log_debug_content\033[0m" 38 | fi 39 | dlfile_try_create_dir "$log_path" 40 | echo $log_write_content >> "$log_path/$now_day.log" 41 | } 42 | 43 | function log_test(){ 44 | $DLLOG_DEBUG "debug test" 45 | $DLLOG_INFO "test" 46 | $DLLOG_WARNING "test" 47 | $DLLOG_ERROR "error test" 48 | } 49 | 50 | if [ -n "$BASH_SOURCE" -a "$BASH_SOURCE" != "$0" ] 51 | then 52 | echo 53 | else 54 | log_test 55 | fi 56 | -------------------------------------------------------------------------------- /cmd_sh/dlbase/dlroot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$dlroot" ]; then 3 | return 4 | fi 5 | 6 | export dlroot="dlroot.sh" 7 | 8 | . dllog.sh 9 | 10 | function dlroot_check_root_user(){ 11 | local user_name=`whoami` 12 | if [ "$user_name" == "root" ] ; then 13 | return 0 14 | else 15 | $DLLOG_ERROR "need run with root" 16 | return 1 17 | fi 18 | } 19 | 20 | function dlroot_change_file_owner(){ 21 | local user_name=$1 22 | local file_path=$2 23 | dlroot_check_root_user 24 | 25 | if [[ $? -eq 0 ]]; then 26 | chown -R $user_name:$user_name "$file_path" 27 | else 28 | return 29 | fi 30 | } 31 | 32 | function dlroot_create_and_write_file(){ 33 | local content=$1 34 | local file_path=$2 35 | dlroot_check_root_user 36 | 37 | if [[ $? -eq 0 ]]; then 38 | echo -e $content > $file_path 39 | else 40 | return 41 | fi 42 | } 43 | 44 | function dlroot_write_to_file_back(){ 45 | local content=$1 46 | local file_path=$2 47 | echo -e $content >> $file_path 48 | } 49 | 50 | if [ -n "$BASH_SOURCE" -a "$BASH_SOURCE" != "$0" ] 51 | then 52 | echo 53 | else 54 | dlroot_check_root_user 55 | echo run self 56 | fi 57 | -------------------------------------------------------------------------------- /cmd_sh/dlbase/dluser.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$dluser" ]; then 3 | echo do not need import dluser.sh 4 | return 5 | fi 6 | 7 | export dluser="dluser.sh" 8 | 9 | . dlroot.sh 10 | 11 | function dluser_find_user(){ 12 | user_name=$1 13 | [ -z $1 ] && return 0 14 | if id $1 &> /dev/null ;then 15 | return 1 16 | else 17 | return 0 18 | fi 19 | } 20 | 21 | function dluser_del_user(){ 22 | user_name=$1 23 | dluser_find_user $user_name 24 | 25 | Res=$? 26 | if [ $Res -eq 1 ] 27 | then 28 | dlroot_check_root_user 29 | if [[ $? -eq 0 ]]; then 30 | $DLLOG_INFO "delete the user:$user_name" 31 | userdel -r $user_name 32 | else 33 | return 34 | fi 35 | else 36 | $DLLOG_INFO "do not have the user" 37 | fi 38 | } 39 | 40 | function dluser_create_user(){ 41 | user_name=$1 42 | dluser_find_user $user_name 43 | 44 | Res=$? 45 | if [ $Res -eq 1 ] 46 | then 47 | $DLLOG_INFO "have the user" 48 | else 49 | dlroot_check_root_user 50 | if [[ $? -eq 0 ]]; then 51 | $DLLOG_INFO "do not have the user:$user_name create the user" 52 | adduser $1 53 | else 54 | return 55 | fi 56 | fi 57 | } 58 | 59 | function dluser_set_user_password(){ 60 | user_name=$1 61 | user_password=$2 62 | 63 | dluser_find_user $user_name 64 | 65 | Res=$? 66 | if [ $Res -eq 1 ] 67 | then 68 | #判断密码是否被锁定 69 | is_user_locked=`passwd -S $user_name|awk '{print $2}'` 70 | if [ "$is_user_locked" == "LK" ] 71 | then 72 | $DLLOG_INFO "You need input user password" 73 | passwd $user_name; 74 | elif [ "$is_user_locked" == "PS" ] 75 | then 76 | $DLLOG_INFO "user password has been set before,you can check it" 77 | else 78 | $DLLOG_WARNING "unknow lock status: $is_user_locked" 79 | fi 80 | else 81 | $DLLOG_WARNING "do not have the user:$user_name" 82 | fi 83 | } 84 | 85 | function dluser_create_user_and_set_password(){ 86 | user_name=$1 87 | dluser_create_user $user_name 88 | dluser_set_user_password $user_name 89 | } 90 | 91 | if [ -n "$BASH_SOURCE" -a "$BASH_SOURCE" != "$0" ] 92 | then 93 | echo 94 | else 95 | echo "run self" 96 | fi 97 | -------------------------------------------------------------------------------- /cmd_sh/dlbase_test/dlbase_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : dlbase_test.sh 4 | ## Create date : 2018-11-25 17:37 5 | ## Modified date : 2018-11-25 17:38 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | basedir=$(dirname "$realpath") 13 | export basedir=$basedir/../ 14 | export filename=$(basename "$realpath") 15 | export PATH=$PATH:$basedir/dlbase 16 | export PATH=$PATH:$basedir/dlproc 17 | #base sh file 18 | . dlbase.sh 19 | #function sh file 20 | . dlgit_test.sh 21 | . dllog_test.sh 22 | 23 | 24 | function dlbase_test(){ 25 | dlgit_test 26 | dllog_test 27 | } 28 | 29 | -------------------------------------------------------------------------------- /cmd_sh/dlbase_test/dlgit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : dlgit_test.sh 4 | ## Create date : 2018-11-25 16:56 5 | ## Modified date : 2019-01-27 22:21 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | basedir=$(dirname "$realpath") 13 | export basedir=$basedir/../ 14 | export filename=$(basename "$realpath") 15 | export PATH=$PATH:$basedir/dlbase 16 | export PATH=$PATH:$basedir/dlproc 17 | #base sh file 18 | . dlbase.sh 19 | #function sh file 20 | 21 | function dlgit_down_test(){ 22 | down_url="https://github.com/matplotlib/matplotlib.git" 23 | folder="/tmp/matplotlib" 24 | dlgit_clone_git $down_url $folder 25 | rm -rf /home/liuzy/matplotlib 26 | } 27 | 28 | function dlgit_down_test2(){ 29 | down_url="https://github.com/darr/pybase.git" 30 | folder="./pybase" 31 | dlgit_clone_git $down_url $folder 32 | rm -rf ./pybase 33 | } 34 | 35 | function dlgit_test(){ 36 | dlgit_down_test2 37 | } 38 | 39 | dlgit_test 40 | -------------------------------------------------------------------------------- /cmd_sh/dlbase_test/dllog_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : dllog_test.sh 4 | ## Create date : 2018-11-25 17:32 5 | ## Modified date : 2018-11-25 17:37 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | basedir=$(dirname "$realpath") 13 | export basedir=$basedir/../ 14 | export filename=$(basename "$realpath") 15 | export PATH=$PATH:$basedir/dlbase 16 | export PATH=$PATH:$basedir/dlproc 17 | #base sh file 18 | . dlbase.sh 19 | #function sh file 20 | 21 | function dllog_log_test(){ 22 | $DLLOG_DEBUG "调试 debug" 23 | $DLLOG_INFO "信息 info" 24 | $DLLOG_WARNING "警告 warning" 25 | $DLLOG_ERROR "错误 error" 26 | } 27 | 28 | function dllog_test(){ 29 | dllog_log_test 30 | } 31 | 32 | dllog_test 33 | -------------------------------------------------------------------------------- /cmd_sh/etc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : etc.sh 4 | ## Create date : 2018-11-25 15:53 5 | ## Modified date : 2019-06-02 13:30 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | export basedir=$(dirname "$realpath") 13 | export filename=$(basename "$realpath") 14 | export PATH=$PATH:$basedir/dlbase 15 | export PATH=$PATH:$basedir/dlproc 16 | #base sh file 17 | . dlbase.sh 18 | #function sh file 19 | 20 | env_path=~/.pytorch_gpu_memory_env 21 | -------------------------------------------------------------------------------- /cmd_sh/install_pybase.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : install_pybase.sh 4 | ## Create date : 2018-11-25 16:03 5 | ## Modified date : 2019-03-18 15:58 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | export basedir=$(dirname "$realpath") 13 | export filename=$(basename "$realpath") 14 | export PATH=$PATH:$basedir/dlbase 15 | export PATH=$PATH:$basedir/dlproc 16 | #base sh file 17 | . dlbase.sh 18 | #function sh file 19 | . etc.sh 20 | 21 | pybase_path="../pybase" 22 | 23 | function dlgit_down_pybase(){ 24 | down_url="https://github.com/darr/pybase.git" 25 | folder=$pybase_path 26 | dlgit_clone_git $down_url $folder 27 | } 28 | 29 | function dlgit_rm_pybase(){ 30 | rm -rf $pybase_path 31 | } 32 | 33 | function create_pybase(){ 34 | dlfile_check_is_have_dir $pybase_path 35 | 36 | if [[ $? -eq 0 ]]; then 37 | dlgit_down_pybase 38 | else 39 | $DLLOG_INFO "$1 pybase have been created" 40 | # dlgit_rm_pybase 41 | fi 42 | cd $pybase_path 43 | pwd 44 | bash ./set_up.sh 45 | cd - 46 | } 47 | 48 | source $env_path/py3env/bin/activate 49 | create_pybase 50 | deactivate 51 | -------------------------------------------------------------------------------- /cmd_sh/pylint.conf: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. 21 | jobs=1 22 | 23 | # List of plugins (as comma separated values of python modules names) to load, 24 | # usually to register additional checkers. 25 | load-plugins= 26 | 27 | # Pickle collected data for later comparisons. 28 | persistent=no 29 | 30 | # Specify a configuration file. 31 | #rcfile= 32 | 33 | # When enabled, pylint would attempt to guess common misconfiguration and emit 34 | # user-friendly hints instead of false-positive error messages 35 | suggestion-mode=yes 36 | 37 | # Allow loading of arbitrary C extensions. Extensions are imported into the 38 | # active Python interpreter and may run arbitrary code. 39 | unsafe-load-any-extension=no 40 | 41 | 42 | [MESSAGES CONTROL] 43 | 44 | # Only show warnings with the listed confidence levels. Leave empty to show 45 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 46 | confidence= 47 | 48 | # Disable the message, report, category or checker with the given id(s). You 49 | # can either give multiple identifiers separated by comma (,) or put this 50 | # option multiple times (only on the command line, not in the configuration 51 | # file where it should appear only once).You can also use "--disable=all" to 52 | # disable everything first and then reenable specific checks. For example, if 53 | # you want to run only the similarities checker, you can use "--disable=all 54 | # --enable=similarities". If you want to run only the classes checker, but have 55 | # no Warning level messages displayed, use"--disable=all --enable=classes 56 | # --disable=W" 57 | disable=print-statement, 58 | parameter-unpacking, 59 | unpacking-in-except, 60 | old-raise-syntax, 61 | backtick, 62 | long-suffix, 63 | old-ne-operator, 64 | old-octal-literal, 65 | import-star-module-level, 66 | non-ascii-bytes-literal, 67 | invalid-unicode-literal, 68 | raw-checker-failed, 69 | bad-inline-option, 70 | locally-disabled, 71 | locally-enabled, 72 | file-ignored, 73 | suppressed-message, 74 | useless-suppression, 75 | deprecated-pragma, 76 | apply-builtin, 77 | basestring-builtin, 78 | buffer-builtin, 79 | cmp-builtin, 80 | coerce-builtin, 81 | execfile-builtin, 82 | file-builtin, 83 | long-builtin, 84 | raw_input-builtin, 85 | reduce-builtin, 86 | standarderror-builtin, 87 | unicode-builtin, 88 | xrange-builtin, 89 | coerce-method, 90 | delslice-method, 91 | getslice-method, 92 | setslice-method, 93 | no-absolute-import, 94 | old-division, 95 | dict-iter-method, 96 | dict-view-method, 97 | next-method-called, 98 | metaclass-assignment, 99 | indexing-exception, 100 | raising-string, 101 | reload-builtin, 102 | oct-method, 103 | hex-method, 104 | nonzero-method, 105 | cmp-method, 106 | input-builtin, 107 | round-builtin, 108 | intern-builtin, 109 | unichr-builtin, 110 | map-builtin-not-iterating, 111 | zip-builtin-not-iterating, 112 | range-builtin-not-iterating, 113 | filter-builtin-not-iterating, 114 | using-cmp-argument, 115 | eq-without-hash, 116 | div-method, 117 | idiv-method, 118 | rdiv-method, 119 | exception-message-attribute, 120 | invalid-str-codec, 121 | sys-max-int, 122 | bad-python3-import, 123 | deprecated-string-function, 124 | deprecated-str-translate-call, 125 | deprecated-itertools-function, 126 | deprecated-types-field, 127 | next-method-defined, 128 | dict-items-not-iterating, 129 | dict-keys-not-iterating, 130 | dict-values-not-iterating, 131 | deprecated-operator-function, 132 | deprecated-urllib-function, 133 | xreadlines-attribute, 134 | deprecated-sys-function, 135 | C0111, #self add disable docstring check 136 | line-too-long, #self add 137 | relative-import, #self add 138 | global-statement, 139 | import-error, 140 | exception-escape, 141 | comprehension-escape 142 | 143 | # Enable the message, report, category or checker with the given id(s). You can 144 | # either give multiple identifier separated by comma (,) or put this option 145 | # multiple time (only on the command line, not in the configuration file where 146 | # it should appear only once). See also the "--disable" option for examples. 147 | enable=c-extension-no-member 148 | 149 | 150 | [REPORTS] 151 | 152 | # Python expression which should return a note less than 10 (10 is the highest 153 | # note). You have access to the variables errors warning, statement which 154 | # respectively contain the number of errors / warnings messages and the total 155 | # number of statements analyzed. This is used by the global evaluation report 156 | # (RP0004). 157 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 158 | 159 | # Template used to display messages. This is a python new-style format string 160 | # used to format the message information. See doc for all details 161 | #msg-template= 162 | 163 | # Set the output format. Available formats are text, parseable, colorized, json 164 | # and msvs (visual studio).You can also give a reporter class, eg 165 | # mypackage.mymodule.MyReporterClass. 166 | output-format=text 167 | 168 | # Tells whether to display a full report or only the messages 169 | reports=no 170 | 171 | # Activate the evaluation score. 172 | score=yes 173 | 174 | 175 | [REFACTORING] 176 | 177 | # Maximum number of nested blocks for function / method body 178 | max-nested-blocks=5 179 | 180 | # Complete name of functions that never returns. When checking for 181 | # inconsistent-return-statements if a never returning function is called then 182 | # it will be considered as an explicit return statement and no message will be 183 | # printed. 184 | never-returning-functions=optparse.Values,sys.exit 185 | 186 | 187 | [BASIC] 188 | 189 | # Naming style matching correct argument names 190 | argument-naming-style=snake_case 191 | 192 | # Regular expression matching correct argument names. Overrides argument- 193 | # naming-style 194 | #argument-rgx= 195 | 196 | # Naming style matching correct attribute names 197 | attr-naming-style=snake_case 198 | 199 | # Regular expression matching correct attribute names. Overrides attr-naming- 200 | # style 201 | #attr-rgx= 202 | 203 | # Bad variable names which should always be refused, separated by a comma 204 | bad-names=foo, 205 | bar, 206 | baz, 207 | toto, 208 | tutu, 209 | tata 210 | 211 | # Naming style matching correct class attribute names 212 | class-attribute-naming-style=any 213 | 214 | # Regular expression matching correct class attribute names. Overrides class- 215 | # attribute-naming-style 216 | #class-attribute-rgx= 217 | 218 | # Naming style matching correct class names 219 | class-naming-style=PascalCase 220 | 221 | # Regular expression matching correct class names. Overrides class-naming-style 222 | #class-rgx= 223 | 224 | # Naming style matching correct constant names 225 | const-naming-style=UPPER_CASE 226 | 227 | # Regular expression matching correct constant names. Overrides const-naming- 228 | # style 229 | #const-rgx= 230 | 231 | # Minimum line length for functions/classes that require docstrings, shorter 232 | # ones are exempt. 233 | docstring-min-length=-1 234 | 235 | # Naming style matching correct function names 236 | function-naming-style=snake_case 237 | 238 | # Regular expression matching correct function names. Overrides function- 239 | # naming-style 240 | #function-rgx= 241 | 242 | # Good variable names which should always be accepted, separated by a comma 243 | good-names=i, 244 | ip, 245 | j, 246 | s, 247 | a, 248 | b, 249 | c, 250 | p, 251 | X, 252 | Y, 253 | f, 254 | l, 255 | k, 256 | ex, 257 | Run, 258 | _ 259 | 260 | # Include a hint for the correct naming format with invalid-name 261 | include-naming-hint=no 262 | 263 | # Naming style matching correct inline iteration names 264 | inlinevar-naming-style=any 265 | 266 | # Regular expression matching correct inline iteration names. Overrides 267 | # inlinevar-naming-style 268 | #inlinevar-rgx= 269 | 270 | # Naming style matching correct method names 271 | method-naming-style=snake_case 272 | 273 | # Regular expression matching correct method names. Overrides method-naming- 274 | # style 275 | #method-rgx= 276 | 277 | # Naming style matching correct module names 278 | module-naming-style=snake_case 279 | 280 | # Regular expression matching correct module names. Overrides module-naming- 281 | # style 282 | #module-rgx= 283 | 284 | # Colon-delimited sets of names that determine each other's naming style when 285 | # the name regexes allow several styles. 286 | name-group= 287 | 288 | # Regular expression which should only match function or class names that do 289 | # not require a docstring. 290 | no-docstring-rgx=^_ 291 | 292 | # List of decorators that produce properties, such as abc.abstractproperty. Add 293 | # to this list to register other decorators that produce valid properties. 294 | property-classes=abc.abstractproperty 295 | 296 | # Naming style matching correct variable names 297 | variable-naming-style=snake_case 298 | 299 | # Regular expression matching correct variable names. Overrides variable- 300 | # naming-style 301 | #variable-rgx= 302 | 303 | 304 | [SPELLING] 305 | 306 | # Limits count of emitted suggestions for spelling mistakes 307 | max-spelling-suggestions=4 308 | 309 | # Spelling dictionary name. Available dictionaries: none. To make it working 310 | # install python-enchant package. 311 | spelling-dict= 312 | 313 | # List of comma separated words that should not be checked. 314 | spelling-ignore-words= 315 | 316 | # A path to a file that contains private dictionary; one word per line. 317 | spelling-private-dict-file= 318 | 319 | # Tells whether to store unknown words to indicated private dictionary in 320 | # --spelling-private-dict-file option instead of raising a message. 321 | spelling-store-unknown-words=no 322 | 323 | 324 | [FORMAT] 325 | 326 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 327 | expected-line-ending-format= 328 | 329 | # Regexp for a line that is allowed to be longer than the limit. 330 | ignore-long-lines=^\s*(# )??$ 331 | 332 | # Number of spaces of indent required inside a hanging or continued line. 333 | indent-after-paren=4 334 | 335 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 336 | # tab). 337 | indent-string=' ' 338 | 339 | # Maximum number of characters on a single line. 340 | max-line-length=100 341 | 342 | # Maximum number of lines in a module 343 | max-module-lines=1000 344 | 345 | # List of optional constructs for which whitespace checking is disabled. `dict- 346 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 347 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 348 | # `empty-line` allows space-only lines. 349 | no-space-check=trailing-comma, 350 | dict-separator 351 | 352 | # Allow the body of a class to be on the same line as the declaration if body 353 | # contains single statement. 354 | single-line-class-stmt=no 355 | 356 | # Allow the body of an if to be on the same line as the test if there is no 357 | # else. 358 | single-line-if-stmt=no 359 | 360 | 361 | [LOGGING] 362 | 363 | # Logging modules to check that the string format arguments are in logging 364 | # function parameter format 365 | logging-modules=logging 366 | 367 | 368 | [VARIABLES] 369 | 370 | # List of additional names supposed to be defined in builtins. Remember that 371 | # you should avoid to define new builtins when possible. 372 | additional-builtins= 373 | 374 | # Tells whether unused global variables should be treated as a violation. 375 | allow-global-unused-variables=yes 376 | 377 | # List of strings which can identify a callback function by name. A callback 378 | # name must start or end with one of those strings. 379 | callbacks=cb_, 380 | _cb 381 | 382 | # A regular expression matching the name of dummy variables (i.e. expectedly 383 | # not used). 384 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 385 | 386 | # Argument names that match this expression will be ignored. Default to name 387 | # with leading underscore 388 | ignored-argument-names=_.*|^ignored_|^unused_ 389 | 390 | # Tells whether we should check for unused import in __init__ files. 391 | init-import=no 392 | 393 | # List of qualified module names which can have objects that can redefine 394 | # builtins. 395 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins 396 | 397 | 398 | [MISCELLANEOUS] 399 | 400 | # List of note tags to take in consideration, separated by a comma. 401 | notes=FIXME, 402 | XXX, 403 | TODO 404 | 405 | 406 | [SIMILARITIES] 407 | 408 | # Ignore comments when computing similarities. 409 | ignore-comments=yes 410 | 411 | # Ignore docstrings when computing similarities. 412 | ignore-docstrings=yes 413 | 414 | # Ignore imports when computing similarities. 415 | ignore-imports=no 416 | 417 | # Minimum lines number of a similarity. 418 | min-similarity-lines=4 419 | 420 | 421 | [TYPECHECK] 422 | 423 | # List of decorators that produce context managers, such as 424 | # contextlib.contextmanager. Add to this list to register other decorators that 425 | # produce valid context managers. 426 | contextmanager-decorators=contextlib.contextmanager 427 | 428 | # List of members which are set dynamically and missed by pylint inference 429 | # system, and so shouldn't trigger E1101 when accessed. Python regular 430 | # expressions are accepted. 431 | generated-members= 432 | 433 | # Tells whether missing members accessed in mixin class should be ignored. A 434 | # mixin class is detected if its name ends with "mixin" (case insensitive). 435 | ignore-mixin-members=yes 436 | 437 | # This flag controls whether pylint should warn about no-member and similar 438 | # checks whenever an opaque object is returned when inferring. The inference 439 | # can return multiple potential results while evaluating a Python object, but 440 | # some branches might not be evaluated, which results in partial inference. In 441 | # that case, it might be useful to still emit no-member and other checks for 442 | # the rest of the inferred objects. 443 | ignore-on-opaque-inference=yes 444 | 445 | # List of class names for which member attributes should not be checked (useful 446 | # for classes with dynamically set attributes). This supports the use of 447 | # qualified names. 448 | ignored-classes=optparse.Values,thread._local,_thread._local 449 | 450 | # List of module names for which member attributes should not be checked 451 | # (useful for modules/projects where namespaces are manipulated during runtime 452 | # and thus existing member attributes cannot be deduced by static analysis. It 453 | # supports qualified module names, as well as Unix pattern matching. 454 | ignored-modules= 455 | 456 | # Show a hint with possible names when a member name was not found. The aspect 457 | # of finding the hint is based on edit distance. 458 | missing-member-hint=yes 459 | 460 | # The minimum edit distance a name should have in order to be considered a 461 | # similar match for a missing member name. 462 | missing-member-hint-distance=1 463 | 464 | # The total number of similar names that should be taken in consideration when 465 | # showing a hint for a missing member. 466 | missing-member-max-choices=1 467 | 468 | 469 | [DESIGN] 470 | 471 | # Maximum number of arguments for function / method 472 | max-args=5 473 | 474 | # Maximum number of attributes for a class (see R0902). 475 | max-attributes=7 476 | 477 | # Maximum number of boolean expressions in a if statement 478 | max-bool-expr=5 479 | 480 | # Maximum number of branch for function / method body 481 | max-branches=12 482 | 483 | # Maximum number of locals for function / method body 484 | max-locals=15 485 | 486 | # Maximum number of parents for a class (see R0901). 487 | max-parents=7 488 | 489 | # Maximum number of public methods for a class (see R0904). 490 | max-public-methods=20 491 | 492 | # Maximum number of return / yield for function / method body 493 | max-returns=6 494 | 495 | # Maximum number of statements in function / method body 496 | max-statements=50 497 | 498 | # Minimum number of public methods for a class (see R0903). 499 | min-public-methods=2 500 | 501 | 502 | [CLASSES] 503 | 504 | # List of method names used to declare (i.e. assign) instance attributes. 505 | defining-attr-methods=__init__, 506 | __new__, 507 | setUp 508 | 509 | # List of member names, which should be excluded from the protected access 510 | # warning. 511 | exclude-protected=_asdict, 512 | _fields, 513 | _replace, 514 | _source, 515 | _make 516 | 517 | # List of valid names for the first argument in a class method. 518 | valid-classmethod-first-arg=cls 519 | 520 | # List of valid names for the first argument in a metaclass class method. 521 | valid-metaclass-classmethod-first-arg=mcs 522 | 523 | 524 | [IMPORTS] 525 | 526 | # Allow wildcard imports from modules that define __all__. 527 | allow-wildcard-with-all=no 528 | 529 | # Analyse import fallback blocks. This can be used to support both Python 2 and 530 | # 3 compatible code, which means that the block might have code that exists 531 | # only in one or another interpreter, leading to false positives when analysed. 532 | analyse-fallback-blocks=no 533 | 534 | # Deprecated modules which should not be used, separated by a comma 535 | deprecated-modules=regsub, 536 | TERMIOS, 537 | Bastion, 538 | rexec 539 | 540 | # Create a graph of external dependencies in the given file (report RP0402 must 541 | # not be disabled) 542 | ext-import-graph= 543 | 544 | # Create a graph of every (i.e. internal and external) dependencies in the 545 | # given file (report RP0402 must not be disabled) 546 | import-graph= 547 | 548 | # Create a graph of internal dependencies in the given file (report RP0402 must 549 | # not be disabled) 550 | int-import-graph= 551 | 552 | # Force import order to recognize a module as part of the standard 553 | # compatibility libraries. 554 | known-standard-library= 555 | 556 | # Force import order to recognize a module as part of a third party library. 557 | known-third-party=enchant 558 | 559 | 560 | [EXCEPTIONS] 561 | 562 | # Exceptions that will emit a warning when being caught. Defaults to 563 | # "Exception" 564 | overgeneral-exceptions=Exception 565 | -------------------------------------------------------------------------------- /cmd_sh/remove_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : remove_env.sh 4 | ## Create date : 2018-11-25 16:01 5 | ## Modified date : 2019-01-27 16:25 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | export basedir=$(dirname "$realpath") 13 | export filename=$(basename "$realpath") 14 | export PATH=$PATH:$basedir/dlbase 15 | export PATH=$PATH:$basedir/dlproc 16 | #base sh file 17 | . dlbase.sh 18 | #function sh file 19 | . etc.sh 20 | 21 | rm -rf $env_path 22 | -------------------------------------------------------------------------------- /gpu_memory_log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################### 4 | # File name : gpu_mem.py 5 | # Create date : 2019-06-02 16:56 6 | # Modified date : 2019-06-02 16:59 7 | # Author : DARREN 8 | # Describe : not set 9 | # Email : lzygzh@126.com 10 | ##################################### 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import gc 15 | import datetime 16 | import pynvml 17 | 18 | import torch 19 | import numpy as np 20 | import sys 21 | 22 | def _get_tensors(): 23 | for obj in gc.get_objects(): 24 | if torch.is_tensor(obj): 25 | tensor = obj 26 | else: 27 | continue 28 | if tensor.is_cuda: 29 | yield tensor 30 | 31 | def _write_log(f, write_str): 32 | print(write_str) 33 | f.write("%s\n" % write_str) 34 | 35 | def gpu_memory_log(gpu_log_file="gpu_mem.log", device=0): 36 | stack_layer = 1 37 | func_name = sys._getframe(stack_layer).f_code.co_name 38 | file_name = sys._getframe(stack_layer).f_code.co_filename 39 | line = sys._getframe(stack_layer).f_lineno 40 | now_time = datetime.datetime.now() 41 | log_format = 'LINE:%s, FUNC:%s, FILE:%s, TIME:%s, CONTENT:%s' 42 | 43 | pynvml.nvmlInit() 44 | handle = pynvml.nvmlDeviceGetHandleByIndex(device) 45 | meminfo = pynvml.nvmlDeviceGetMemoryInfo(handle) 46 | 47 | with open(gpu_log_file, 'a+') as f: 48 | write_str = log_format % (line, func_name, file_name, now_time, "") 49 | _write_log(f, write_str) 50 | 51 | ts_list = [tensor.size() for tensor in _get_tensors()] 52 | new_tensor_sizes = {(type(x), 53 | tuple(x.size()), 54 | ts_list.count(x.size()), 55 | np.prod(np.array(x.size()))*4/1024**2) 56 | for x in _get_tensors()} 57 | for t, s, n, m in new_tensor_sizes: 58 | write_str = '[tensor: %s * Size:%s | Memory: %s M | %s]' %(str(n), str(s), str(m*n)[:6], str(t)) 59 | _write_log(f, write_str) 60 | 61 | write_str = "memory_allocated:%f Mb" % float(torch.cuda.memory_allocated()/1024**2) 62 | _write_log(f, write_str) 63 | write_str = "max_memory_allocated:%f Mb" % float(torch.cuda.max_memory_allocated()/1024**2) 64 | _write_log(f, write_str) 65 | write_str = "memory_cached:%f Mb" % float(torch.cuda.memory_cached()/1024**2) 66 | _write_log(f, write_str) 67 | write_str = "max_memory_cached:%f Mb" % float(torch.cuda.max_memory_cached()/1024**2) 68 | _write_log(f, write_str) 69 | write_str = "Used Memory:%f Mb" % float(meminfo.used/1024**2) 70 | _write_log(f, write_str) 71 | write_str = "Free Memory:%f Mb" % float(meminfo.free/1024**2) 72 | _write_log(f, write_str) 73 | write_str = "Total Memory:%f Mb" % float(meminfo.total/1024**2) 74 | _write_log(f, write_str) 75 | 76 | pynvml.nvmlShutdown() 77 | -------------------------------------------------------------------------------- /gpu_run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : gpu_run.sh 4 | ## Create date : 2018-11-26 11:19 5 | ## Modified date : 2019-06-02 15:18 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | basedir=$(dirname "$realpath") 13 | export basedir=$basedir/cmd_sh/ 14 | export filename=$(basename "$realpath") 15 | export PATH=$PATH:$basedir 16 | export PATH=$PATH:$basedir/dlbase 17 | export PATH=$PATH:$basedir/dlproc 18 | #base sh file 19 | . dlbase.sh 20 | #function sh file 21 | . etc.sh 22 | #. install_pybase.sh 23 | 24 | kill -9 `ps -ef|grep main.py|grep -v grep|awk '{print $2}'` 25 | kill -9 `ps -ef|grep defunct|grep -v grep|awk '{print$3}'` 26 | 27 | function create_vir_env(){ 28 | dlfile_check_is_have_dir $env_path 29 | 30 | if [[ $? -eq 0 ]]; then 31 | bash ./cmd_sh/create_env.sh 32 | else 33 | $DLLOG_INFO "$1 virtual env had been created" 34 | fi 35 | } 36 | 37 | create_vir_env 38 | 39 | #bash ./cmd_sh/check_code.sh 40 | 41 | # source $env_path/py2env/bin/activate 42 | # deactivate 43 | echo $env_path 44 | 45 | source $env_path/py3env/bin/activate 46 | #create_pybase 47 | python gpu_test.py 48 | 49 | deactivate 50 | -------------------------------------------------------------------------------- /gpu_test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "scrolled": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stdout", 12 | "output_type": "stream", 13 | "text": [ 14 | "LINE:15, FUNC:, FILE:, TIME:2019-06-02 17:10:55.856389, CONTENT:\n", 15 | "[tensor: 1 * Size:(100, 10) | Memory: 0.0038 M | ]\n", 16 | "[tensor: 1 * Size:(64, 1000) | Memory: 0.2441 M | ]\n", 17 | "[tensor: 1 * Size:(64, 10) | Memory: 0.0024 M | ]\n", 18 | "[tensor: 1 * Size:(1000, 100) | Memory: 0.3814 M | ]\n", 19 | "memory_allocated:1.457520 Mb\n", 20 | "max_memory_allocated:1.457520 Mb\n", 21 | "memory_cached:2.000000 Mb\n", 22 | "max_memory_cached:2.000000 Mb\n", 23 | "Used Memory:9386.625000 Mb\n", 24 | "Free Memory:1782.625000 Mb\n", 25 | "Total Memory:11169.250000 Mb\n" 26 | ] 27 | }, 28 | { 29 | "name": "stderr", 30 | "output_type": "stream", 31 | "text": [ 32 | "/home/liuzy/.pytorch_gpu_memory_env/py3env/lib/python3.5/site-packages/torch/distributed/distributed_c10d.py:100: UserWarning: torch.distributed.reduce_op is deprecated, please use torch.distributed.ReduceOp instead\n", 33 | " warnings.warn(\"torch.distributed.reduce_op is deprecated, please use \"\n" 34 | ] 35 | }, 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "3.0322433303808793e-05\n", 41 | "LINE:29, FUNC:, FILE:, TIME:2019-06-02 17:10:56.354555, CONTENT:\n", 42 | "[tensor: 1 * Size:() | Memory: 3.8146 M | ]\n", 43 | "[tensor: 1 * Size:(100, 10) | Memory: 0.0038 M | ]\n", 44 | "[tensor: 2 * Size:(64, 10) | Memory: 0.0048 M | ]\n", 45 | "[tensor: 1 * Size:(64, 1000) | Memory: 0.2441 M | ]\n", 46 | "[tensor: 1 * Size:(1000, 100) | Memory: 0.3814 M | ]\n", 47 | "memory_allocated:1.846191 Mb\n", 48 | "max_memory_allocated:2.252930 Mb\n", 49 | "memory_cached:4.000000 Mb\n", 50 | "max_memory_cached:4.000000 Mb\n", 51 | "Used Memory:9392.625000 Mb\n", 52 | "Free Memory:1776.625000 Mb\n", 53 | "Total Memory:11169.250000 Mb\n" 54 | ] 55 | } 56 | ], 57 | "source": [ 58 | "import torch\n", 59 | "from gpu_memory_log import gpu_memory_log\n", 60 | "\n", 61 | "dtype = torch.float\n", 62 | "N, D_in, H, D_out = 64, 1000, 100, 10\n", 63 | "\n", 64 | "device = torch.device(\"cuda\")\n", 65 | "x = torch.randn(N, D_in, device=device, dtype=dtype)\n", 66 | "y = torch.randn(N, D_out, device=device, dtype=dtype)\n", 67 | "\n", 68 | "w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)\n", 69 | "w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)\n", 70 | "\n", 71 | "learning_rate = 1e-6\n", 72 | "gpu_memory_log()\n", 73 | "for t in range(500):\n", 74 | " y_pred = x.mm(w1).clamp(min=0).mm(w2)\n", 75 | " loss = (y_pred - y).pow(2).sum()\n", 76 | " #print(t, loss.item())\n", 77 | " loss.backward()\n", 78 | " with torch.no_grad():\n", 79 | " w1 -= learning_rate * w1.grad\n", 80 | " w2 -= learning_rate * w2.grad\n", 81 | "\n", 82 | " w1.grad.zero_()\n", 83 | " w2.grad.zero_()\n", 84 | "\n", 85 | "print(loss.item())\n", 86 | "gpu_memory_log()\n" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [] 123 | } 124 | ], 125 | "metadata": { 126 | "kernelspec": { 127 | "display_name": "Python 3", 128 | "language": "python", 129 | "name": "python3" 130 | }, 131 | "language_info": { 132 | "codemirror_mode": { 133 | "name": "ipython", 134 | "version": 3 135 | }, 136 | "file_extension": ".py", 137 | "mimetype": "text/x-python", 138 | "name": "python", 139 | "nbconvert_exporter": "python", 140 | "pygments_lexer": "ipython3", 141 | "version": "3.5.2" 142 | } 143 | }, 144 | "nbformat": 4, 145 | "nbformat_minor": 1 146 | } 147 | -------------------------------------------------------------------------------- /gpu_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | ##################################### 4 | # File name : gpu_test.py 5 | # Create date : 2019-06-02 15:17 6 | # Modified date : 2019-06-02 17:16 7 | # Author : DARREN 8 | # Describe : not set 9 | # Email : lzygzh@126.com 10 | ##################################### 11 | from __future__ import division 12 | from __future__ import print_function 13 | import torch 14 | from gpu_memory_log import gpu_memory_log 15 | 16 | dtype = torch.float 17 | N, D_in, H, D_out = 64, 1000, 100, 10 18 | 19 | device = torch.device("cuda") 20 | x = torch.randn(N, D_in, device=device, dtype=dtype) 21 | y = torch.randn(N, D_out, device=device, dtype=dtype) 22 | 23 | w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True) 24 | w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True) 25 | 26 | learning_rate = 1e-6 27 | gpu_memory_log() 28 | for t in range(500): 29 | y_pred = x.mm(w1).clamp(min=0).mm(w2) 30 | loss = (y_pred - y).pow(2).sum() 31 | #print(t, loss.item()) 32 | loss.backward() 33 | with torch.no_grad(): 34 | w1 -= learning_rate * w1.grad 35 | w2 -= learning_rate * w2.grad 36 | 37 | w1.grad.zero_() 38 | w2.grad.zero_() 39 | 40 | print(loss.item()) 41 | gpu_memory_log() 42 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##################################### 3 | ## File name : run.sh 4 | ## Create date : 2018-11-26 11:19 5 | ## Modified date : 2019-05-20 23:43 6 | ## Author : DARREN 7 | ## Describe : not set 8 | ## Email : lzygzh@126.com 9 | #################################### 10 | 11 | realpath=$(readlink -f "$0") 12 | basedir=$(dirname "$realpath") 13 | export basedir=$basedir/cmd_sh/ 14 | export filename=$(basename "$realpath") 15 | export PATH=$PATH:$basedir 16 | export PATH=$PATH:$basedir/dlbase 17 | export PATH=$PATH:$basedir/dlproc 18 | #base sh file 19 | . dlbase.sh 20 | #function sh file 21 | . etc.sh 22 | #. install_pybase.sh 23 | 24 | kill -9 `ps -ef|grep main.py|grep -v grep|awk '{print $2}'` 25 | kill -9 `ps -ef|grep defunct|grep -v grep|awk '{print$3}'` 26 | 27 | function create_vir_env(){ 28 | dlfile_check_is_have_dir $env_path 29 | 30 | if [[ $? -eq 0 ]]; then 31 | bash ./cmd_sh/create_env.sh 32 | else 33 | $DLLOG_INFO "$1 virtual env had been created" 34 | fi 35 | } 36 | 37 | create_vir_env 38 | 39 | #bash ./cmd_sh/check_code.sh 40 | 41 | # source $env_path/py2env/bin/activate 42 | # deactivate 43 | echo $env_path 44 | 45 | source $env_path/py3env/bin/activate 46 | #create_pybase 47 | jupyter notebook 48 | 49 | deactivate 50 | --------------------------------------------------------------------------------