├── .github └── workflows │ ├── ci-test.yml │ └── version-check.yml ├── .gitignore ├── .scrutinizer.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── codecov.yml ├── examples ├── .qiniu_pythonsdk_hostscache.json ├── batch.py ├── batch_copy.py ├── batch_delete.py ├── batch_move.py ├── batch_rename.py ├── batch_restoreAr.py ├── batch_stat.py ├── bucket_domain.py ├── bucket_info.py ├── cdn_bandwidth.py ├── cdn_flux.py ├── cdn_log.py ├── change_bucket_permission.py ├── change_mime.py ├── change_status.py ├── change_type.py ├── copy_to.py ├── create_bucket.py ├── delete.py ├── delete_afte_days.py ├── domain_relevant.py ├── download.py ├── fetch.py ├── fops.py ├── get_domaininfo.py ├── kirk │ ├── README.md │ ├── list_apps.py │ ├── list_services.py │ └── list_stacks.py ├── list.py ├── list_buckets.py ├── list_domains.py ├── mk_bucket.py ├── move_to.py ├── pfop_vframe.py ├── pfop_watermark.py ├── prefetch_to_bucket.py ├── prefetch_to_cdn.py ├── refresh_dirs.py ├── refresh_urls.py ├── restorear.py ├── rtc_server.py ├── set_object_lifecycle.py ├── sms_test.py ├── stat.py ├── timestamp_url.py ├── update_cdn_sslcert.py ├── upload.py ├── upload_callback.py ├── upload_pfops.py ├── upload_token.py ├── upload_with_qvmzone.py └── upload_with_zone.py ├── manual_test_kirk.py ├── qiniu ├── __init__.py ├── auth.py ├── compat.py ├── config.py ├── http │ ├── __init__.py │ ├── client.py │ ├── default_client.py │ ├── endpoint.py │ ├── endpoints_provider.py │ ├── endpoints_retry_policy.py │ ├── middleware │ │ ├── __init__.py │ │ ├── base.py │ │ ├── retry_domains.py │ │ └── ua.py │ ├── region.py │ ├── regions_provider.py │ ├── regions_retry_policy.py │ ├── response.py │ └── single_flight.py ├── main.py ├── region.py ├── retry │ ├── __init__.py │ ├── abc │ │ ├── __init__.py │ │ └── policy.py │ ├── attempt.py │ └── retrier.py ├── services │ ├── __init__.py │ ├── cdn │ │ ├── __init__.py │ │ └── manager.py │ ├── compute │ │ ├── __init__.py │ │ ├── app.py │ │ ├── config.py │ │ └── qcos_api.py │ ├── pili │ │ ├── __init__.py │ │ └── rtc_server_manager.py │ ├── processing │ │ ├── __init__.py │ │ ├── cmd.py │ │ └── pfop.py │ ├── sms │ │ ├── __init__.py │ │ └── sms.py │ └── storage │ │ ├── __init__.py │ │ ├── _bucket_default_retrier.py │ │ ├── bucket.py │ │ ├── legacy.py │ │ ├── upload_progress_recorder.py │ │ ├── uploader.py │ │ └── uploaders │ │ ├── __init__.py │ │ ├── _default_retrier.py │ │ ├── abc │ │ ├── __init__.py │ │ ├── resume_uploader_base.py │ │ └── uploader_base.py │ │ ├── form_uploader.py │ │ ├── io_chunked.py │ │ ├── resume_uploader_v1.py │ │ └── resume_uploader_v2.py ├── utils.py └── zone.py ├── setup.py ├── test_qiniu.py └── tests ├── cases ├── __init__.py ├── conftest.py ├── test_auth.py ├── test_http │ ├── __init__.py │ ├── conftest.py │ ├── test_endpoint.py │ ├── test_endpoints_retry_policy.py │ ├── test_middleware.py │ ├── test_qiniu_conf.py │ ├── test_region.py │ ├── test_regions_provider.py │ ├── test_regions_retry_policy.py │ ├── test_resp.py │ └── test_single_flight.py ├── test_retry │ ├── __init__.py │ └── test_retrier.py ├── test_services │ ├── __init__.py │ ├── test_processing │ │ ├── __init__.py │ │ └── test_pfop.py │ └── test_storage │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_bucket_manager.py │ │ ├── test_upload_pfop.py │ │ ├── test_uploader.py │ │ └── test_uploaders_default_retrier.py ├── test_utils.py └── test_zone │ ├── __init__.py │ ├── test_lagacy_region.py │ └── test_qiniu_conf.py └── mock_server ├── main.py └── routes ├── __init__.py ├── echo.py ├── retry_me.py └── timeout.py /.github/workflows/ci-test.yml: -------------------------------------------------------------------------------- 1 | on: [push] 2 | name: Run Test Cases 3 | jobs: 4 | test: 5 | strategy: 6 | fail-fast: false 7 | max-parallel: 1 8 | matrix: 9 | python_version: ['2.7', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9'] 10 | runs-on: ubuntu-20.04 11 | steps: 12 | - name: Checkout repo 13 | uses: actions/checkout@v2 14 | with: 15 | ref: ${{ github.ref }} 16 | - name: Setup miniconda 17 | uses: conda-incubator/setup-miniconda@v2 18 | with: 19 | auto-update-conda: true 20 | channels: conda-forge 21 | python-version: ${{ matrix.python_version }} 22 | activate-environment: qiniu-sdk 23 | auto-activate-base: false 24 | - name: Setup pip 25 | shell: bash -l {0} 26 | env: 27 | PYTHON_VERSION: ${{ matrix.python_version }} 28 | PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip 29 | run: | 30 | MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1) 31 | MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2) 32 | # reinstall pip by some python(<3.7) not compatible 33 | if ! [[ $MAJOR -ge 3 && $MINOR -ge 7 ]]; then 34 | cd /tmp 35 | wget -qLO get-pip.py "$PIP_BOOTSTRAP_SCRIPT_PREFIX/$MAJOR.$MINOR/get-pip.py" 36 | python get-pip.py --user 37 | fi 38 | - name: Setup mock server 39 | shell: bash -el {0} 40 | run: | 41 | conda create -y -n mock-server python=3.10 42 | conda activate mock-server 43 | python3 --version 44 | nohup python3 tests/mock_server/main.py --port 9000 > py-mock-server.log & 45 | echo $! > mock-server.pid 46 | conda deactivate 47 | - name: Install dependencies 48 | shell: bash -l {0} 49 | run: | 50 | python -m pip install --upgrade pip 51 | python -m pip install -I -e ".[dev]" 52 | - name: Run cases 53 | shell: bash -el {0} 54 | env: 55 | QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} 56 | QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} 57 | QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} 58 | QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }} 59 | QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} 60 | QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} 61 | QINIU_TEST_ENV: "travis" 62 | MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" 63 | run: | 64 | flake8 --show-source --max-line-length=160 ./qiniu 65 | python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml 66 | - name: Post Setup mock server 67 | if: ${{ always() }} 68 | shell: bash 69 | run: | 70 | set +e 71 | cat mock-server.pid | xargs kill 72 | rm mock-server.pid 73 | - name: Print mock server log 74 | if: ${{ failure() }} 75 | run: | 76 | cat py-mock-server.log 77 | - name: Upload results to Codecov 78 | uses: codecov/codecov-action@v4 79 | with: 80 | token: ${{ secrets.CODECOV_TOKEN }} 81 | test-win: 82 | strategy: 83 | fail-fast: false 84 | max-parallel: 1 85 | matrix: 86 | python_version: ['2.7', '3.5', '3.9'] 87 | runs-on: windows-2019 88 | # make sure only one test running, 89 | # remove this when cases could run in parallel. 90 | needs: test 91 | steps: 92 | - name: Checkout repo 93 | uses: actions/checkout@v2 94 | with: 95 | ref: ${{ github.ref }} 96 | - name: Setup miniconda 97 | uses: conda-incubator/setup-miniconda@v2 98 | with: 99 | auto-update-conda: true 100 | channels: conda-forge 101 | python-version: ${{ matrix.python_version }} 102 | activate-environment: qiniu-sdk 103 | auto-activate-base: false 104 | - name: Setup pip 105 | env: 106 | PYTHON_VERSION: ${{ matrix.python_version }} 107 | PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip 108 | run: | 109 | # reinstall pip by some python(<3.7) not compatible 110 | $pyversion = [Version]"$ENV:PYTHON_VERSION" 111 | if ($pyversion -lt [Version]"3.7") { 112 | Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py" 113 | python $ENV:TEMP\get-pip.py --user 114 | Remove-Item -Path "$ENV:TEMP\get-pip.py" 115 | } 116 | - name: Install dependencies 117 | run: | 118 | python -m pip install --upgrade pip 119 | python -m pip install -I -e ".[dev]" 120 | - name: Run cases 121 | env: 122 | QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} 123 | QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} 124 | QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} 125 | QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }} 126 | QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} 127 | QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} 128 | QINIU_TEST_ENV: "github" 129 | MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" 130 | PYTHONPATH: "$PYTHONPATH:." 131 | run: | 132 | Write-Host "======== Setup Mock Server =========" 133 | conda create -y -n mock-server python=3.10 134 | conda activate mock-server 135 | python --version 136 | $processOptions = @{ 137 | FilePath="python" 138 | ArgumentList="tests\mock_server\main.py", "--port", "9000" 139 | PassThru=$true 140 | RedirectStandardOutput="py-mock-server.log" 141 | } 142 | $mocksrvp = Start-Process @processOptions 143 | $mocksrvp.Id | Out-File -FilePath "mock-server.pid" 144 | conda deactivate 145 | Sleep 3 146 | Write-Host "======== Running Test =========" 147 | python --version 148 | python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml 149 | - name: Post Setup mock server 150 | if: ${{ always() }} 151 | run: | 152 | Try { 153 | $mocksrvpid = Get-Content -Path "mock-server.pid" 154 | Stop-Process -Id $mocksrvpid 155 | Remove-Item -Path "mock-server.pid" 156 | } Catch { 157 | Write-Host -Object $_ 158 | } 159 | - name: Print mock server log 160 | if: ${{ failure() }} 161 | run: | 162 | Get-Content -Path "py-mock-server.log" | Write-Host 163 | - name: Upload results to Codecov 164 | uses: codecov/codecov-action@v4 165 | with: 166 | token: ${{ secrets.CODECOV_TOKEN }} 167 | -------------------------------------------------------------------------------- /.github/workflows/version-check.yml: -------------------------------------------------------------------------------- 1 | name: Python SDK Version Check 2 | on: 3 | push: 4 | tags: 5 | - "v[0-9]+.[0-9]+.[0-9]+" 6 | jobs: 7 | linux: 8 | name: Version Check 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout code 12 | uses: actions/checkout@v2 13 | - name: Set env 14 | run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV 15 | - name: Check 16 | run: | 17 | set -e 18 | grep -qF "## ${RELEASE_VERSION}" CHANGELOG.md 19 | grep -qF "__version__ = '${RELEASE_VERSION}'" qiniu/__init__.py 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.swp 3 | *.pyc 4 | 5 | *.py[cod] 6 | 7 | my-test-env.sh 8 | 9 | ## 10 | ## from https://github.com/github/gitignore/blob/master/Python.gitignore 11 | ## 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Packages 17 | *.egg 18 | *.egg-info 19 | dist 20 | build 21 | eggs 22 | parts 23 | bin 24 | var 25 | sdist 26 | develop-eggs 27 | .installed.cfg 28 | lib 29 | lib64 30 | __pycache__ 31 | 32 | # Installer logs 33 | pip-log.txt 34 | 35 | # Unit test / coverage reports 36 | .coverage 37 | .tox 38 | nosetests.xml 39 | coverage.xml 40 | 41 | # Translations 42 | *.mo 43 | 44 | # Mr Developer 45 | .mr.developer.cfg 46 | .project 47 | .pydevproject 48 | /.idea 49 | /.venv* 50 | -------------------------------------------------------------------------------- /.scrutinizer.yml: -------------------------------------------------------------------------------- 1 | 2 | checks: 3 | python: 4 | code_rating: true 5 | duplicate_code: true 6 | variables_redefined_outer_name: true 7 | 8 | tools: 9 | external_code_coverage: 10 | timeout: 12000 # Timeout in seconds. 11 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | ## 7.16.0 3 | * 对象存储，优化并发场景的区域查询 4 | * CDN，查询域名带宽，支持 `data_type` 参数 5 | 6 | ## 7.15.0 7 | * 对象存储，持久化处理支持工作流模版 8 | * 对象存储，修复 Windows 平台兼容性问题 9 | 10 | ## 7.14.0 11 | * 对象存储，空间管理、上传文件新增备用域名重试逻辑 12 | * 对象存储，调整查询区域主备域名 13 | * 对象存储，支持空间级别加速域名开关 14 | * 对象存储，回调签名验证函数新增兼容 Qiniu 签名 15 | * 对象存储，持久化处理支持闲时任务 16 | 17 | ## 7.13.2（2024-05-28） 18 | * 对象存储，修复上传回调设置自定义变量失效（v7.12.0 引入） 19 | 20 | ## 7.13.1（2024-02-21） 21 | * 对象存储，修复上传部分配置项的兼容 22 | * 对象存储，添加上传策略部分字段 23 | 24 | ## 7.13.0（2023-12-11） 25 | * 对象存储，新增支持归档直读存储 26 | * 对象存储，批量操作支持自动查询 rs 服务域名 27 | 28 | ## 7.12.1（2023-11-20） 29 | * 修复 CDN 删除域名代码问题 30 | 31 | ## 7.12.0（2023-10-08） 32 | * 对象存储，分片上传支持并发上传 33 | 34 | ## 7.11.1（2023-08-16） 35 | * 修复 setup.py 打包丢失部分包（v7.11.0 引入） 36 | 37 | ## 7.11.0（2023-03-28） 38 | * 对象存储，更新 api 默认域名 39 | * 对象存储，新增 api 域名的配置与获取 40 | * 对象存储，修复获取区域域名后无法按照预期进行过期处理 41 | * 对象存储，更新获取区域域名的接口 42 | * 对象存储，bucket_domains 修改为 list_domains 的别名 43 | * 对象存储，新增请求中间件逻辑，方便拓展请求逻辑 44 | * 对象存储，新增备用 UC 域名用于查询区域域名 45 | 46 | ## 7.10.0（2022-11-15） 47 | * 对象存储，修复通过 set_default 设置 rs, rsf 不生效，而 SDK 自动获取的问题（v7.9.0） 48 | * 对象存储，支持直接从 qiniu 导入 UploadProgressRecorder 49 | * 对象存储，优化分片上传 ctx 超时检测 50 | * 文档，更新注释中文档链接 51 | 52 | ## 7.9.0（2022-07-20） 53 | * 对象存储，支持使用时不配置区域信息，SDK 自动获取； 54 | * 对象存储，新增 list_domains API 用于查询空间绑定的域名 55 | * 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API 56 | * 解决部分已知问题 57 | 58 | ## 7.8.0（2022-06-08） 59 | * 对象存储，管理类 API 发送请求时增加 [X-Qiniu-Date](https://developer.qiniu.com/kodo/3924/common-request-headers) （生成请求的时间） header 60 | 61 | ## 7.7.1 (2022-05-11) 62 | * 对象存储，修复上传不制定 key 部分情况下会上传失败问题。 63 | 64 | ## 7.7.0 (2022-04-29) 65 | * 对象存储，新增 set_object_lifecycle （设置 Object 生命周期） API 66 | 67 | ## 7.6.0 (2022-03-28) 68 | * 优化了错误处理机制 69 | * 支持 [Qiniu](https://developer.qiniu.com/kodo/1201/access-token) 签名算法 70 | 71 | ## 7.5.0 (2021-09-23) 72 | * 上传策略新增对部分字段支持 73 | 74 | ## 7.4.1 (2021-05-25) 75 | * 分片上传 v2 方法不再强制要求 bucket_name 参数 76 | 77 | ## 7.4.0 (2021-05-21) 78 | * 支持分片上传 v2 79 | 80 | ## 7.3.1 (2021-01-06) 81 | * 修复 ResponseInfo 对扩展码错误处理问题 82 | * 增加 python v3.7，v3.8，v3.9 版本 CI 测试 83 | 84 | ## 7.3.0 (2020-09-23) 85 | 新增 86 | * sms[云短信]：新增查询短信发送记录方法：get_messages_info 87 | * cdn: 新增上线域名 domain_online 方法、下线域名 domain_offline 方法和删除域名 delete_domain 方法 88 | * 对象存储：新增批量解冻build_batch_restoreAr方法、获取空间列表bucket_domain方法和修改空间访问权限change_bucket_permission方法 89 | 90 | ## 7.2.10 （2020-08-21） 91 | * 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 92 | ## 7.2.9 (2020-08-07) 93 | * 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 94 | * 更正接口返回描述docstring 95 | * 修复接口对非json response 处理 96 | * ci 覆盖增加python 3.6 3.7 97 | * 修复获取域名列方法 98 | * 修复python3 环境下，二进制对象上传问题 99 | 100 | 101 | ## 7.2.8(2020-03-27) 102 | * add restoreAr 103 | 104 | ## 7.2.7(2020-03-10) 105 | * fix bucket_info 106 | 107 | ## 7.2.6(2019-06-26) 108 | * 添加sms 109 | 110 | ## 7.2.5 (2019-06-06) 111 | * 添加sms 112 | 113 | ## 7.2.4 (2019-04-01) 114 | * 默认导入region类 115 | 116 | ## 7.2.3 (2019-02-25) 117 | * 新增region类，zone继承 118 | * 上传可以指定上传域名 119 | * 新增上传指定上传空间和qvm指定上传内网的例子 120 | * 新增列举账号空间，创建空间，查询空间信息，改变文件状态接口，并提供例子 121 | 122 | ## 7.2.2 (2018-05-10) 123 | * 增加连麦rtc服务端API功能 124 | 125 | ## 7.2.0（2017-11-23） 126 | * 修复put_data不支持file like object的问题 127 | * 增加空间写错时，抛出异常提示客户的功能 128 | * 增加创建空间的接口功能 129 | 130 | ## 7.1.9（2017-11-01） 131 | * 修复python2情况下，中文文件名上传失败的问题 132 | * 修复python2环境下，中文文件使用分片上传时失败的问题 133 | 134 | ## 7.1.8 （2017-10-18） 135 | * 恢复kirk的API为原来的状态 136 | 137 | ## 7.1.7 (2017-09-27) 138 | 139 | * 修复从时间戳获取rfc http格式的时间字符串问题 140 | 141 | ## 7.1.6 (2017-09-26) 142 | 143 | * 给 `put_file` 功能增加保持本地文件Last Modified功能，以支持切换源站的客户CDN不回源 144 | 145 | ## 7.1.5 (2017-08-26) 146 | 147 | * 设置表单上传默认校验crc32 148 | * 增加PutPolicy新参数isPrefixalScope 149 | * 修复手动指定的zone无效的问题 150 | 151 | ## 7.1.4 (2017-06-05) 152 | ### 修正 153 | * cdn功能中获取域名日志列表的参数错误 154 | 155 | ## 7.1.2 (2017-03-24) 156 | ### 增加 157 | * 增加设置文件生命周期的接口 158 | 159 | ## 7.1.1 (2017-02-03) 160 | ### 增加 161 | * 增加cdn刷新，预取，日志获取，时间戳防盗链生成功能 162 | 163 | ### 修正 164 | * 修复分片上传的upload record path遇到中文时的问题，现在使用md5来计算文件名 165 | 166 | ## 7.1.0 (2016-12-08) 167 | ### 增加 168 | * 通用计算支持 169 | 170 | ## 7.0.10 (2016-11-29) 171 | ### 修正 172 | * 去掉homedir 173 | 174 | ## 7.0.9 (2016-10-09) 175 | ### 增加 176 | * 多机房接口调用支持 177 | 178 | ## 7.0.8 (2016-07-05) 179 | ### 修正 180 | * 修复表单上传大于20M文件的400错误 181 | 182 | ### 增加 183 | * copy 和 move 操作增加 force 字段，允许强制覆盖 copy 和 move 184 | * 增加上传策略 deleteAfterDays 字段 185 | * 一些 demo 186 | 187 | ## 7.0.7 (2016-05-05) 188 | ### 修正 189 | * 修复大于4M的文件hash计算错误的问题 190 | * add fname 191 | 192 | ### 增加 193 | * 一些demo 194 | * travis 直接发布 195 | 196 | ## 7.0.6 (2015-12-05) 197 | ### 修正 198 | * 2.x unicode 问题 by @hunter007 199 | * 上传重试判断 200 | * 上传时 dns劫持处理 201 | 202 | ### 增加 203 | * fsizeMin 上传策略 204 | * 断点上传记录 by @hokein 205 | * 计算stream etag 206 | * 3.5 ci 支持 207 | 208 | ## 7.0.5 (2015-06-25) 209 | ### 变更 210 | * 配置up_host 改为配置zone 211 | 212 | ### 增加 213 | * fectch 支持不指定key 214 | 215 | ## 7.0.4 (2015-05-04) 216 | ### 修正 217 | * 上传重试为空文件 218 | * 回调应该只对form data 签名。 219 | 220 | ## 7.0.3 (2015-03-11) 221 | ### 增加 222 | * 可以配置 io/rs/api/rsf host 223 | 224 | ## 7.0.2 (2014-12-24) 225 | ### 修正 226 | * 内部http get当没有auth会出错 227 | * python3下的qiniupy 没有参数时 arg parse会抛异常 228 | * 增加callback policy 229 | 230 | ## 7.0.1 (2014-11-26) 231 | ### 增加 232 | * setup.py从文件中读取版本号，而不是用导入方式 233 | * 补充及修正了一些单元测试 234 | 235 | ## 7.0.0 (2014-11-13) 236 | 237 | ### 增加 238 | * 简化上传接口 239 | * 自动选择断点续上传还是直传 240 | * 重构代码，接口和内部结构更清晰 241 | * 同时支持python 2.x 和 3.x 242 | * 支持pfop 243 | * 支持verify callback 244 | * 改变mime 245 | * 代码覆盖度报告 246 | * policy改为dict, 便于灵活增加，并加入过期字段检查 247 | * 文件列表支持目录形式 248 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # 贡献代码指南 2 | 3 | 我们非常欢迎大家来贡献代码，我们会向贡献者致以最诚挚的敬意。 4 | 5 | 一般可以通过在Github上提交[Pull Request](https://github.com/qiniu/python-sdk)来贡献代码。 6 | 7 | ## Pull Request要求 8 | 9 | - **代码规范** 遵从pep8，pythonic。 10 | 11 | - **代码格式** 提交前 请按 pep8 进行格式化。 12 | 13 | - **必须添加测试！** - 如果没有测试（单元测试、集成测试都可以），那么提交的补丁是不会通过的。 14 | 15 | - **记得更新文档** - 保证`README.md`以及其他相关文档及时更新，和代码的变更保持一致性。 16 | 17 | - **考虑我们的发布周期** - 我们的版本号会服从[SemVer v2.0.0](http://semver.org/)，我们绝对不会随意变更对外的API。 18 | 19 | - **创建feature分支** - 最好不要从你的master分支提交 pull request。 20 | 21 | - **一个feature提交一个pull请求** - 如果你的代码变更了多个操作，那就提交多个pull请求吧。 22 | 23 | - **清晰的commit历史** - 保证你的pull请求的每次commit操作都是有意义的。如果你开发中需要执行多次的即时commit操作，那么请把它们放到一起再提交pull请求。 24 | 25 | ## 运行测试 26 | 27 | ``` bash 28 | py.test 29 | 30 | ``` 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Qiniu, Ltd. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Qiniu Cloud SDK for Python 2 | 3 | [![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) 4 | [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) 5 | [![Build Status](https://github.com/qiniu/python-sdk/actions/workflows/ci-test.yml/badge.svg)](https://travis-ci.org/qiniu/python-sdk) 6 | [![GitHub release](https://img.shields.io/github/v/tag/qiniu/python-sdk.svg?label=release)](https://github.com/qiniu/python-sdk/releases) 7 | [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) 8 | [![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) 9 | [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/qiniu/python-sdk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/qiniu/python-sdk/?branch=master) 10 | [![Coverage Status](https://codecov.io/gh/qiniu/python-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/python-sdk) 11 | 12 | ## 安装 13 | 14 | 通过pip 15 | 16 | ```bash 17 | $ pip install qiniu 18 | ``` 19 | 20 | ## 运行环境 21 | 22 | | Qiniu SDK版本 | Python 版本 | 23 | | :-----------: | :------------------------------------: | 24 | | 7.x | 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 | 25 | | 6.x | 2.7 | 26 | 27 | ## 使用方法 28 | 29 | ### 上传 30 | ```python 31 | import qiniu 32 | 33 | ... 34 | q = qiniu.Auth(access_key, secret_key) 35 | key = 'hello' 36 | data = 'hello qiniu!' 37 | token = q.upload_token(bucket_name) 38 | ret, info = qiniu.put_data(token, key, data) 39 | if ret is not None: 40 | print('All is OK') 41 | else: 42 | print(info) # error message in info 43 | ... 44 | 45 | ``` 46 | 更多参见SDK使用指南： https://developer.qiniu.com/kodo/sdk/python 47 | ``` 48 | 49 | ## 测试 50 | 51 | ``` bash 52 | $ py.test 53 | ``` 54 | 55 | ## 常见问题 56 | 57 | - 第二个参数info保留了请求响应的信息，失败情况下ret 为none, 将info可以打印出来，提交给我们。 58 | - API 的使用 demo 可以参考 [examples示例](https://github.com/qiniu/python-sdk/tree/master/examples)。 59 | - 如果碰到`ImportError: No module named requests.auth` 请安装 `requests` 。 60 | 61 | ## 代码贡献 62 | 63 | 详情参考[代码提交指南](https://github.com/qiniu/python-sdk/blob/master/CONTRIBUTING.md)。 64 | 65 | ## 贡献记录 66 | 67 | - [所有贡献者](https://github.com/qiniu/python-sdk/contributors) 68 | 69 | ## 联系我们 70 | 71 | - 如果需要帮助，请提交工单（在portal右侧点击咨询和建议提交工单，或者直接向 support@qiniu.com 发送邮件） 72 | - 如果有什么问题，可以到问答社区提问，[问答社区](http://qiniu.segmentfault.com/) 73 | - 更详细的文档，见[官方文档站](http://developer.qiniu.com/) 74 | - 如果发现了bug， 欢迎提交 [issue](https://github.com/qiniu/python-sdk/issues) 75 | - 如果有功能需求，欢迎提交 [issue](https://github.com/qiniu/python-sdk/issues) 76 | - 如果要提交代码，欢迎提交 pull request 77 | - 欢迎关注我们的[微信](http://www.qiniu.com/#weixin) [微博](http://weibo.com/qiniutek)，及时获取动态信息。 78 | 79 | ## 代码许可 80 | 81 | The MIT License (MIT).详情见 [License文件](https://github.com/qiniu/python-sdk/blob/master/LICENSE). 82 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | ci: 3 | - prow.qiniu.io # prow need this. seems useless 4 | require_ci_to_pass: no # `no` means the bot will comment on the PR even before all ci passed 5 | 6 | github_checks: # close github checks 7 | annotations: false 8 | 9 | comment: 10 | layout: "reach, diff, flags, files" 11 | behavior: new # `new` means the bot will comment a new message instead of edit the old one 12 | require_changes: false # if true: only post the comment if coverage changes 13 | require_base: no # [yes :: must have a base report to post] 14 | require_head: yes # [yes :: must have a head report to post] 15 | branches: # branch names that can post comment 16 | - "master" 17 | 18 | coverage: 19 | status: # check coverage status to pass or fail 20 | patch: off 21 | project: # project analyze all code in the project 22 | default: 23 | # basic 24 | target: 73.5% # the minimum coverage ratio that the commit must meet 25 | threshold: 3% # allow the coverage to drop 26 | base: auto 27 | if_not_found: success 28 | if_ci_failed: error 29 | -------------------------------------------------------------------------------- /examples/.qiniu_pythonsdk_hostscache.json: -------------------------------------------------------------------------------- 1 | {"http:wxCLv4yl_5saIuOHbbZbkP-Ef3kFFFeCDYmwTdg3:upload30": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1598428478}} -------------------------------------------------------------------------------- /examples/batch.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | from qiniu import build_batch_copy 7 | from qiniu import build_batch_move, build_batch_rename 8 | 9 | access_key = '...' 10 | secret_key = '...' 11 | 12 | # 初始化Auth状态 13 | q = Auth(access_key, secret_key) 14 | 15 | # 初始化BucketManager 16 | bucket = BucketManager(q) 17 | keys = {'123.jpg': '123.jpg'} 18 | 19 | # ops = build_batch_copy( 'teest', keys, 'teest',force='true') 20 | # ops = build_batch_move('teest', keys, 'teest', force='true') 21 | ops = build_batch_rename('teest', keys, force='true') 22 | 23 | ret, info = bucket.batch(ops) 24 | print(ret) 25 | print(info) 26 | assert ret == {} 27 | -------------------------------------------------------------------------------- /examples/batch_copy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | 5 | """ 6 | 批量拷贝文件 7 | 8 | https://developer.qiniu.com/kodo/api/1250/batch 9 | """ 10 | 11 | 12 | from qiniu import build_batch_copy, Auth, BucketManager 13 | 14 | access_key = '' 15 | 16 | secret_key = '' 17 | 18 | q = Auth(access_key, secret_key) 19 | 20 | bucket = BucketManager(q) 21 | 22 | src_bucket_name = '' 23 | 24 | target_bucket_name = '' 25 | 26 | # force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 27 | ops = build_batch_copy(src_bucket_name, 28 | {'src_key1': 'target_key1', 29 | 'src_key2': 'target_key2'}, 30 | target_bucket_name, 31 | force='true') 32 | ret, info = bucket.batch(ops) 33 | print(info) 34 | -------------------------------------------------------------------------------- /examples/batch_delete.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 批量删除文件 6 | 7 | https://developer.qiniu.com/kodo/api/1250/batch 8 | """ 9 | 10 | 11 | from qiniu import build_batch_delete, Auth, BucketManager 12 | 13 | access_key = '' 14 | 15 | secret_key = '' 16 | 17 | q = Auth(access_key, secret_key) 18 | 19 | bucket = BucketManager(q) 20 | 21 | bucket_name = '' 22 | 23 | keys = ['1.gif', '2.txt', '3.png', '4.html'] 24 | 25 | ops = build_batch_delete(bucket_name, keys) 26 | ret, info = bucket.batch(ops) 27 | print(info) 28 | -------------------------------------------------------------------------------- /examples/batch_move.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 批量移动文件 6 | 7 | https://developer.qiniu.com/kodo/api/1250/batch 8 | """ 9 | 10 | 11 | from qiniu import build_batch_move, Auth, BucketManager 12 | 13 | access_key = '' 14 | 15 | secret_key = '' 16 | 17 | 18 | q = Auth(access_key, secret_key) 19 | 20 | bucket = BucketManager(q) 21 | 22 | src_bucket_name = '' 23 | 24 | target_bucket_name = '' 25 | 26 | # force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 27 | ops = build_batch_move(src_bucket_name, 28 | {'src_key1': 'target_key1', 29 | 'src_key2': 'target_key2'}, 30 | target_bucket_name, 31 | force='true') 32 | ret, info = bucket.batch(ops) 33 | print(info) 34 | -------------------------------------------------------------------------------- /examples/batch_rename.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 批量重命名文件 6 | 7 | https://developer.qiniu.com/kodo/api/1250/batch 8 | """ 9 | 10 | 11 | from qiniu import build_batch_rename, Auth, BucketManager 12 | 13 | access_key = '' 14 | 15 | secret_key = '' 16 | 17 | 18 | q = Auth(access_key, secret_key) 19 | 20 | bucket = BucketManager(q) 21 | 22 | bucket_name = '' 23 | 24 | 25 | # force为true时强制同名覆盖, 字典的键为原文件，值为目标文件 26 | ops = build_batch_rename( 27 | bucket_name, { 28 | 'src_key1': 'target_key1', 'src_key2': 'target_key2'}, force='true') 29 | ret, info = bucket.batch(ops) 30 | print(info) 31 | -------------------------------------------------------------------------------- /examples/batch_restoreAr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 批量解冻文件 6 | https://developer.qiniu.com/kodo/api/1250/batch 7 | """ 8 | 9 | from qiniu import build_batch_restoreAr, Auth, BucketManager 10 | 11 | # 七牛账号的公钥和私钥 12 | access_key = '' 13 | secret_key = '' 14 | 15 | q = Auth(access_key, secret_key) 16 | 17 | bucket = BucketManager(q) 18 | 19 | # 存储空间 20 | bucket_name = "空间名" 21 | 22 | # 字典的键为需要解冻的文件，值为解冻有效期1-7 23 | ops = build_batch_restoreAr(bucket_name, 24 | {"test00.png": 1, 25 | "test01.jpeg": 2, 26 | "test02.mp4": 3 27 | } 28 | ) 29 | 30 | ret, info = bucket.batch(ops) 31 | print(info) 32 | -------------------------------------------------------------------------------- /examples/batch_stat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 批量查询文件信息 6 | 7 | https://developer.qiniu.com/kodo/api/1250/batch 8 | """ 9 | 10 | from qiniu import build_batch_stat, Auth, BucketManager 11 | 12 | access_key = '' 13 | secret_key = '' 14 | 15 | q = Auth(access_key, secret_key) 16 | 17 | bucket = BucketManager(q) 18 | 19 | bucket_name = '' 20 | 21 | # 需要查询的文件名 22 | keys = ['1.gif', '2.txt', '3.png', '4.html'] 23 | 24 | ops = build_batch_stat(bucket_name, keys) 25 | ret, info = bucket.batch(ops) 26 | print(info) 27 | -------------------------------------------------------------------------------- /examples/bucket_domain.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | """ 8 | 获取空间绑定的加速域名 9 | https://developer.qiniu.com/kodo/api/3949/get-the-bucket-space-domain 10 | """ 11 | 12 | # 七牛账号的 公钥和私钥 13 | access_key = '' 14 | secret_key = '' 15 | 16 | # 空间名 17 | bucket_name = '' 18 | 19 | q = Auth(access_key, secret_key) 20 | 21 | bucket = BucketManager(q) 22 | 23 | ret, info = bucket.bucket_domain(bucket_name) 24 | print(info) 25 | -------------------------------------------------------------------------------- /examples/bucket_info.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | # 需要填写你的 Access Key 和 Secret Key 8 | access_key = '' 9 | secret_key = '' 10 | 11 | # 空间名 12 | bucket_name = 'bucket_name' 13 | 14 | q = Auth(access_key, secret_key) 15 | 16 | bucket = BucketManager(q) 17 | 18 | ret, info = bucket.bucket_info(bucket_name) 19 | print(info) 20 | -------------------------------------------------------------------------------- /examples/cdn_bandwidth.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 查询指定域名指定时间段内的带宽 6 | """ 7 | import qiniu 8 | from qiniu import CdnManager, DataType 9 | 10 | 11 | # 账户ak，sk 12 | access_key = '' 13 | secret_key = '' 14 | 15 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 16 | cdn_manager = CdnManager(auth) 17 | 18 | startDate = '2017-07-20' 19 | 20 | endDate = '2017-08-20' 21 | 22 | granularity = 'day' 23 | 24 | urls = [ 25 | 'a.example.com', 26 | 'b.example.com' 27 | ] 28 | 29 | ret, info = cdn_manager.get_bandwidth_data( 30 | urls, startDate, endDate, granularity) 31 | 32 | print(ret) 33 | print(info) 34 | 35 | ret, info = cdn_manager.get_bandwidth_data( 36 | urls, startDate, endDate, granularity, data_type=DataType.BANDWIDTH) 37 | 38 | print(ret) 39 | print(info) 40 | -------------------------------------------------------------------------------- /examples/cdn_flux.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 查询指定域名指定时间段内的流量 6 | """ 7 | import qiniu 8 | from qiniu import CdnManager 9 | 10 | 11 | # 账户ak，sk 12 | access_key = '' 13 | secret_key = '' 14 | 15 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 16 | cdn_manager = CdnManager(auth) 17 | 18 | startDate = '2017-07-20' 19 | 20 | endDate = '2017-08-20' 21 | 22 | granularity = 'day' 23 | 24 | urls = [ 25 | 'a.example.com', 26 | 'b.example.com' 27 | ] 28 | 29 | # 获得指定域名流量 30 | ret, info = cdn_manager.get_flux_data(urls, startDate, endDate, granularity) 31 | 32 | print(ret) 33 | print(info) 34 | -------------------------------------------------------------------------------- /examples/cdn_log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 获取指定域名指定时间内的日志链接 6 | """ 7 | import qiniu 8 | from qiniu import CdnManager 9 | 10 | 11 | # 账户ak，sk 12 | access_key = '' 13 | secret_key = '' 14 | 15 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 16 | cdn_manager = CdnManager(auth) 17 | 18 | log_date = '2017-07-20' 19 | 20 | urls = [ 21 | 'a.example.com', 22 | 'b.example.com' 23 | ] 24 | 25 | 26 | ret, info = cdn_manager.get_log_list_data(urls, log_date) 27 | 28 | print(ret) 29 | print(info) 30 | -------------------------------------------------------------------------------- /examples/change_bucket_permission.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | # 需要填写七牛账号的 公钥和私钥 8 | access_key = '' 9 | secret_key = '' 10 | 11 | # 空间名 12 | bucket_name = "" 13 | 14 | # private 参数必须是str类型，0表示公有空间，1表示私有空间 15 | private = "0" 16 | 17 | q = Auth(access_key, secret_key) 18 | 19 | bucket = BucketManager(q) 20 | 21 | ret, info = bucket.change_bucket_permission(bucket_name, private) 22 | print(info) 23 | -------------------------------------------------------------------------------- /examples/change_mime.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 改变文件的mimeType 6 | """ 7 | from qiniu import Auth 8 | from qiniu import BucketManager 9 | 10 | access_key = '...' 11 | secret_key = '...' 12 | 13 | q = Auth(access_key, secret_key) 14 | 15 | bucket = BucketManager(q) 16 | 17 | bucket_name = 'Bucket_Name' 18 | 19 | key = '...' 20 | 21 | ret, info = bucket.change_mime(bucket_name, key, 'image/jpg') 22 | print(info) 23 | assert info.status_code == 200 24 | -------------------------------------------------------------------------------- /examples/change_status.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 改变文件状态，可用或不可用 6 | """ 7 | from qiniu import Auth 8 | from qiniu import BucketManager 9 | 10 | # 需要填写你的 Access Key 和 Secret Key 11 | access_key = '' 12 | secret_key = '' 13 | 14 | q = Auth(access_key, secret_key) 15 | 16 | bucket = BucketManager(q) 17 | 18 | # 空间名 19 | bucket_name = 'bernie' 20 | 21 | # 文件名 22 | key = '233.jpg' 23 | 24 | # 条件匹配，只有匹配上才会执行修改操作 25 | # cond可以填空，一个或多个 26 | cond = {"fsize": "186371", 27 | "putTime": "14899798962573916", 28 | "hash": "FiRxWzeeD6ofGTpwTZub5Fx1ozvi", 29 | "mime": "image/png"} 30 | 31 | ret, info = bucket.change_status(bucket_name, key, '1', cond) 32 | print(info) 33 | -------------------------------------------------------------------------------- /examples/change_type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | access_key = '...' 8 | secret_key = '...' 9 | 10 | q = Auth(access_key, secret_key) 11 | 12 | bucket = BucketManager(q) 13 | 14 | bucket_name = 'Bucket_Name' 15 | 16 | key = '...' 17 | 18 | # 1表示低频存储，0是标准存储 19 | ret, info = bucket.change_type(bucket_name, key, 1) 20 | 21 | print(info) 22 | -------------------------------------------------------------------------------- /examples/copy_to.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | access_key = '...' 8 | secret_key = '...' 9 | 10 | # 初始化Auth状态 11 | q = Auth(access_key, secret_key) 12 | 13 | # 初始化BucketManager 14 | bucket = BucketManager(q) 15 | 16 | # 你要测试的空间， 并且这个key在你空间中存在 17 | bucket_name = 'Bucket_Name' 18 | key = 'python-logo.png' 19 | 20 | # 将文件从文件key 复制到文件key2。 可以在不同bucket复制 21 | key2 = 'python-logo2.png' 22 | 23 | ret, info = bucket.copy(bucket_name, key, bucket_name, key2) 24 | print(info) 25 | assert ret == {} 26 | -------------------------------------------------------------------------------- /examples/create_bucket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 创建存储空间 6 | """ 7 | 8 | from qiniu import Auth 9 | from qiniu import BucketManager 10 | 11 | 12 | access_key = '...' 13 | secret_key = '...' 14 | 15 | q = Auth(access_key, secret_key) 16 | 17 | bucket = BucketManager(q) 18 | 19 | bucket_name = 'Bucket_Name' 20 | 21 | # "填写存储区域代号 z0:华东, z1:华北, z2:华南, na0:北美" 22 | region = 'z0' 23 | 24 | ret, info = bucket.mkbucketv2(bucket_name, region) 25 | print(info) 26 | print(ret) 27 | assert info.status_code == 200 28 | -------------------------------------------------------------------------------- /examples/delete.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | access_key = '...' 8 | secret_key = '...' 9 | 10 | # 初始化Auth状态 11 | q = Auth(access_key, secret_key) 12 | 13 | # 初始化BucketManager 14 | bucket = BucketManager(q) 15 | 16 | # 你要测试的空间， 并且这个key在你空间中存在 17 | bucket_name = 'Bucket_Name' 18 | key = 'python-logo.png' 19 | 20 | # 删除bucket_name 中的文件 key 21 | ret, info = bucket.delete(bucket_name, key) 22 | print(info) 23 | assert ret == {} 24 | -------------------------------------------------------------------------------- /examples/delete_afte_days.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | access_key = '...' 8 | secret_key = '...' 9 | 10 | # 初始化Auth状态 11 | q = Auth(access_key, secret_key) 12 | 13 | # 初始化BucketManager 14 | bucket = BucketManager(q) 15 | 16 | # 你要测试的空间， 并且这个key在你空间中存在 17 | bucket_name = 'Bucket_Name' 18 | key = 'python-test.png' 19 | 20 | # 您要更新的生命周期，单位为天 21 | days = '5' 22 | 23 | ret, info = bucket.delete_after_days(bucket_name, key, days) 24 | print(info) 25 | -------------------------------------------------------------------------------- /examples/domain_relevant.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from qiniu import QiniuMacAuth, DomainManager 3 | import json 4 | 5 | """域名上线""" 6 | 7 | # 七牛账号的 公钥和私钥 8 | access_key = "" 9 | secret_key = "" 10 | 11 | auth = QiniuMacAuth(access_key, secret_key) 12 | 13 | manager = DomainManager(auth) 14 | 15 | # 域名 16 | name = "zhuchangzhao2.peterpy.cn" 17 | 18 | ret, res = manager.domain_online(name) 19 | 20 | headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} 21 | print(json.dumps(headers, indent=4, ensure_ascii=False)) 22 | print(json.dumps(ret, indent=4, ensure_ascii=False)) 23 | 24 | """域名下线""" 25 | 26 | # 七牛账号的 公钥和私钥 27 | access_key = "" 28 | secret_key = "" 29 | 30 | auth = QiniuMacAuth(access_key, secret_key) 31 | 32 | manager = DomainManager(auth) 33 | 34 | # 域名 35 | name = "" 36 | 37 | ret, res = manager.domain_offline(name) 38 | 39 | headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} 40 | print(json.dumps(headers, indent=4, ensure_ascii=False)) 41 | print(json.dumps(ret, indent=4, ensure_ascii=False)) 42 | 43 | """删除域名""" 44 | 45 | # 七牛账号的 公钥和私钥 46 | access_key = "" 47 | secret_key = "" 48 | 49 | auth = QiniuMacAuth(access_key, secret_key) 50 | 51 | manager = DomainManager(auth) 52 | 53 | # 域名 54 | name = "" 55 | 56 | ret, res = manager.delete_domain(name) 57 | 58 | headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} 59 | print(json.dumps(headers, indent=4, ensure_ascii=False)) 60 | print(json.dumps(ret, indent=4, ensure_ascii=False)) 61 | -------------------------------------------------------------------------------- /examples/download.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | import requests 5 | from qiniu import Auth 6 | 7 | access_key = '...' 8 | secret_key = '...' 9 | 10 | q = Auth(access_key, secret_key) 11 | bucket_domain = "..." 12 | key = "..." 13 | 14 | # 有两种方式构造base_url的形式 15 | base_url = 'http://%s/%s' % (bucket_domain, key) 16 | 17 | # 或者直接输入url的方式下载 18 | # base_url = 'http://domain/key' 19 | 20 | # 可以设置token过期时间 21 | private_url = q.private_download_url(base_url, expires=3600) 22 | 23 | print(private_url) 24 | r = requests.get(private_url) 25 | assert r.status_code == 200 26 | -------------------------------------------------------------------------------- /examples/fetch.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | access_key = '...' 8 | secret_key = '...' 9 | 10 | bucket_name = 'Bucket_Name' 11 | 12 | q = Auth(access_key, secret_key) 13 | 14 | bucket = BucketManager(q) 15 | 16 | url = 'http://aaa.example.com/test.jpg' 17 | 18 | key = 'test.jpg' 19 | 20 | ret, info = bucket.fetch(url, bucket_name, key) 21 | print(info) 22 | assert ret['key'] == key 23 | -------------------------------------------------------------------------------- /examples/fops.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth, PersistentFop, urlsafe_base64_encode 4 | 5 | # 对已经上传到七牛的视频发起异步转码操作 6 | access_key = '...' 7 | secret_key = '...' 8 | q = Auth(access_key, secret_key) 9 | 10 | # 要转码的文件所在的空间和文件名。 11 | bucket_name = 'Bucket_Name' 12 | key = '1.mp4' 13 | 14 | # 转码是使用的队列名称。 15 | pipeline = 'your_pipeline' 16 | 17 | # 要进行转码的转码操作，下面是一个例子。 18 | fops = 'avthumb/mp4/s/640x360/vb/1.25m' 19 | 20 | # 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 21 | saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') 22 | fops = fops + '|saveas/' + saveas_key 23 | ops = [] 24 | pfop = PersistentFop(q, bucket_name, pipeline) 25 | 26 | ops.append(fops) 27 | ret, info = pfop.execute(key, ops, 1) 28 | print(info) 29 | assert ret['persistentId'] is not None 30 | -------------------------------------------------------------------------------- /examples/get_domaininfo.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 获取指定域名指定时间内的日志链接 6 | """ 7 | import qiniu 8 | from qiniu import DomainManager 9 | 10 | 11 | # 账户ak，sk 12 | access_key = '' 13 | secret_key = '' 14 | 15 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 16 | domain_manager = DomainManager(auth) 17 | domain = '' 18 | ret, info = domain_manager.get_domain(domain) 19 | print(ret) 20 | print(info) -------------------------------------------------------------------------------- /examples/kirk/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | ``` 4 | $ python list_apps.py 5 | ``` 6 | -------------------------------------------------------------------------------- /examples/kirk/list_apps.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | import sys 5 | from qiniu import QiniuMacAuth 6 | from qiniu import AccountClient 7 | 8 | access_key = sys.argv[1] 9 | secret_key = sys.argv[2] 10 | 11 | acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) 12 | 13 | ret, info = acc_client.list_apps() 14 | 15 | print(ret) 16 | print(info) 17 | 18 | assert len(ret) is not None 19 | -------------------------------------------------------------------------------- /examples/kirk/list_services.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | import sys 5 | from qiniu import QiniuMacAuth 6 | from qiniu import AccountClient 7 | 8 | access_key = sys.argv[1] 9 | secret_key = sys.argv[2] 10 | 11 | acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) 12 | apps, info = acc_client.list_apps() 13 | 14 | for app in apps: 15 | if app.get('runMode') == 'Private': 16 | uri = app.get('uri') 17 | qcos = acc_client.get_qcos_client(uri) 18 | if qcos != None: 19 | stacks, info = qcos.list_stacks() 20 | for stack in stacks: 21 | stack_name = stack.get('name') 22 | services, info = qcos.list_services(stack_name) 23 | print("list_services of '%s : %s':"%(uri, stack_name)) 24 | print(services) 25 | print(info) 26 | assert len(services) is not None 27 | -------------------------------------------------------------------------------- /examples/kirk/list_stacks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | import sys 5 | from qiniu import QiniuMacAuth 6 | from qiniu import AccountClient 7 | 8 | access_key = sys.argv[1] 9 | secret_key = sys.argv[2] 10 | 11 | acc_client = AccountClient(QiniuMacAuth(access_key, secret_key)) 12 | apps, info = acc_client.list_apps() 13 | 14 | for app in apps: 15 | if app.get('runMode') == 'Private': 16 | uri = app.get('uri') 17 | qcos = acc_client.get_qcos_client(uri) 18 | if qcos != None: 19 | stacks, info = qcos.list_stacks() 20 | print("list_stacks of '%s':"%uri) 21 | print(stacks) 22 | print(info) 23 | assert len(stacks) is not None 24 | -------------------------------------------------------------------------------- /examples/list.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth 4 | from qiniu import BucketManager 5 | 6 | access_key = '...' 7 | secret_key = '...' 8 | 9 | q = Auth(access_key, secret_key) 10 | bucket = BucketManager(q) 11 | 12 | bucket_name = 'Bucket_Name' 13 | # 前缀 14 | prefix = None 15 | # 列举条目 16 | limit = 10 17 | # 列举出除'/'的所有文件以及以'/'为分隔的所有前缀 18 | delimiter = None 19 | # 标记 20 | marker = None 21 | 22 | ret, eof, info = bucket.list(bucket_name, prefix, marker, limit, delimiter) 23 | 24 | print(info) 25 | 26 | assert len(ret.get('items')) is not None 27 | -------------------------------------------------------------------------------- /examples/list_buckets.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 列举账号下的空间 6 | """ 7 | from qiniu import Auth 8 | from qiniu import BucketManager 9 | 10 | # 需要填写你的 Access Key 和 Secret Key 11 | access_key = '' 12 | secret_key = '' 13 | 14 | q = Auth(access_key, secret_key) 15 | 16 | bucket = BucketManager(q) 17 | 18 | # 指定需要列举的区域，填空字符串返回全部空间，为减少响应时间建议不为空 19 | # z0:只返回华东区域的空间 20 | # z1:只返回华北区域的空间 21 | # z2:只返回华南区域的空间 22 | # na0:只返回北美区域的空间 23 | # as0:只返回东南亚区域的空间 24 | region = "as0" 25 | 26 | ret, info = bucket.list_bucket(region) 27 | print(info) 28 | print(ret) 29 | -------------------------------------------------------------------------------- /examples/list_domains.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth 4 | from qiniu import BucketManager 5 | 6 | access_key = '...' 7 | secret_key = '...' 8 | 9 | # 初始化Auth状态 10 | q = Auth(access_key, secret_key) 11 | 12 | # 初始化BucketManager 13 | bucket = BucketManager(q) 14 | 15 | # 要获取域名的空间名 16 | bucket_name = 'Bucket_Name' 17 | 18 | # 获取空间绑定的域名列表 19 | ret, info = bucket.list_domains(bucket_name) 20 | print(ret) 21 | print(info) 22 | -------------------------------------------------------------------------------- /examples/mk_bucket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | # 需要填写你的 Access Key 和 Secret Key 8 | access_key = '...' 9 | secret_key = '...' 10 | 11 | bucket_name = 'Bucket_Name' 12 | 13 | q = Auth(access_key, secret_key) 14 | 15 | bucket = BucketManager(q) 16 | 17 | region = "z0" 18 | 19 | ret, info = bucket.mkbucketv2(bucket_name, region) 20 | print(info) 21 | -------------------------------------------------------------------------------- /examples/move_to.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth 4 | from qiniu import BucketManager 5 | 6 | access_key = '...' 7 | secret_key = '...' 8 | 9 | # 初始化Auth状态 10 | q = Auth(access_key, secret_key) 11 | 12 | # 初始化BucketManager 13 | bucket = BucketManager(q) 14 | 15 | # 你要测试的空间， 并且这个key在你空间中存在 16 | bucket_name = 'Bucket_Name' 17 | key = 'python-logo.png' 18 | 19 | # 将文件从文件key 移动到文件key2，可以实现文件的重命名 可以在不同bucket移动 20 | key2 = 'python-logo2.png' 21 | 22 | ret, info = bucket.move(bucket_name, key, bucket_name, key2) 23 | print(info) 24 | assert ret == {} 25 | -------------------------------------------------------------------------------- /examples/pfop_vframe.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth, PersistentFop, urlsafe_base64_encode 4 | 5 | # 对已经上传到七牛的视频发起异步转码操作 6 | access_key = 'Access_Key' 7 | secret_key = 'Secret_Key' 8 | q = Auth(access_key, secret_key) 9 | 10 | # 要转码的文件所在的空间和文件名。 11 | bucket = 'Bucket_Name' 12 | key = '1.mp4' 13 | 14 | # 转码是使用的队列名称。 15 | pipeline = 'pipeline_name' 16 | 17 | # 要进行视频截图操作。 18 | fops = 'vframe/jpg/offset/1/w/480/h/360/rotate/90' 19 | 20 | # 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 21 | saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') 22 | fops = fops + '|saveas/' + saveas_key 23 | 24 | pfop = PersistentFop(q, bucket, pipeline) 25 | ops = [] 26 | ops.append(fops) 27 | ret, info = pfop.execute(key, ops, 1) 28 | print(info) 29 | assert ret['persistentId'] is not None 30 | -------------------------------------------------------------------------------- /examples/pfop_watermark.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth, PersistentFop, urlsafe_base64_encode 4 | 5 | # 对已经上传到七牛的视频发起异步转码操作 6 | access_key = 'Access_Key' 7 | secret_key = 'Secret_Key' 8 | q = Auth(access_key, secret_key) 9 | 10 | # 要转码的文件所在的空间和文件名。 11 | bucket = 'Bucket_Name' 12 | key = '1.mp4' 13 | 14 | # 转码是使用的队列名称。 15 | pipeline = 'pipeline_name' 16 | 17 | # 需要添加水印的图片UrlSafeBase64,可以参考 https://developer.qiniu.com/dora/api/video-watermarking 18 | base64URL = urlsafe_base64_encode( 19 | 'http://developer.qiniu.com/resource/logo-2.jpg') 20 | 21 | # 视频水印参数 22 | fops = 'avthumb/mp4/wmImage/'+base64URL 23 | 24 | # 可以对转码后的文件进行使用saveas参数自定义命名，当然也可以不指定文件会默认命名并保存在当前空间 25 | saveas_key = urlsafe_base64_encode('目标Bucket_Name:自定义文件key') 26 | fops = fops + '|saveas/' + saveas_key 27 | ops = [] 28 | pfop = PersistentFop(q, bucket, pipeline) 29 | ops.append(fops) 30 | ret, info = pfop.execute(key, ops, 1) 31 | print(info) 32 | assert ret['persistentId'] 33 | 34 | -------------------------------------------------------------------------------- /examples/prefetch_to_bucket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 拉取镜像源资源到空间 6 | 7 | https://developer.qiniu.com/kodo/api/1293/prefetch 8 | """ 9 | 10 | from qiniu import Auth 11 | from qiniu import BucketManager 12 | 13 | access_key = '...' 14 | secret_key = '...' 15 | 16 | 17 | bucket_name = 'Bucket_Name' 18 | 19 | q = Auth(access_key, secret_key) 20 | 21 | bucket = BucketManager(q) 22 | 23 | # 要拉取的文件名 24 | key = 'test.jpg' 25 | 26 | ret, info = bucket.prefetch(bucket_name, key) 27 | print(info) 28 | assert ret['key'] == key 29 | -------------------------------------------------------------------------------- /examples/prefetch_to_cdn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 预取资源到cdn节点 6 | 7 | https://developer.qiniu.com/fusion/api/1227/file-prefetching 8 | """ 9 | 10 | 11 | import qiniu 12 | from qiniu import CdnManager 13 | 14 | 15 | # 账户ak，sk 16 | access_key = '...' 17 | secret_key = '...' 18 | 19 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 20 | cdn_manager = CdnManager(auth) 21 | 22 | # 需要刷新的文件链接 23 | urls = [ 24 | 'http://aaa.example.com/doc/img/', 25 | 'http://bbb.example.com/doc/video/' 26 | ] 27 | 28 | 29 | # 刷新链接 30 | refresh_dir_result = cdn_manager.prefetch_urls(urls) 31 | -------------------------------------------------------------------------------- /examples/refresh_dirs.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | import qiniu 5 | from qiniu import CdnManager 6 | 7 | 8 | # 账户ak，sk 9 | access_key = '...' 10 | secret_key = '...' 11 | 12 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 13 | cdn_manager = CdnManager(auth) 14 | 15 | # 需要刷新的目录链接 16 | dirs = [ 17 | 'http://aaa.example.com/doc/img/', 18 | 'http://bbb.example.com/doc/video/' 19 | ] 20 | 21 | 22 | # 刷新链接 23 | refresh_dir_result = cdn_manager.refresh_dirs(dirs) 24 | -------------------------------------------------------------------------------- /examples/refresh_urls.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | import qiniu 5 | from qiniu import CdnManager 6 | 7 | # 账户ak，sk 8 | access_key = '...' 9 | secret_key = '...' 10 | 11 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 12 | cdn_manager = CdnManager(auth) 13 | 14 | # 需要刷新的文件链接 15 | urls = [ 16 | 'http://aaa.example.com/a.gif', 17 | 'http://bbb.example.com/b.jpg' 18 | ] 19 | 20 | # 刷新链接 21 | refresh_url_result = cdn_manager.refresh_urls(urls) 22 | print(refresh_url_result) 23 | -------------------------------------------------------------------------------- /examples/restorear.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth 4 | from qiniu import BucketManager 5 | 6 | 7 | access_key = '' 8 | secret_key = '' 9 | 10 | q = Auth(access_key, secret_key) 11 | bucket = BucketManager(q) 12 | bucket_name = '13' 13 | key = 'fb8539c39f65d74b4e70db9133c1e9d5.mp4' 14 | ret,info = bucket.restoreAr(bucket_name,key,3) 15 | print(ret) 16 | print(info) 17 | 18 | -------------------------------------------------------------------------------- /examples/rtc_server.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import QiniuMacAuth 5 | from qiniu import RtcServer, get_room_token 6 | import time 7 | 8 | # 需要填写你的 Access Key 和 Secret Key 9 | access_key = 'xxx' 10 | secret_key = 'xxx' 11 | 12 | # 构建鉴权对象 13 | q = QiniuMacAuth(access_key, secret_key) 14 | 15 | # 构建直播连麦管理对象 16 | rtc = RtcServer(q) 17 | 18 | # 创建一个APP 19 | # 首先需要写好创建APP的各个参数。参数如下 20 | create_data = { 21 | "hub": 'python_test_hub', # Hub: 绑定的直播 hub，可选，使用此 hub 的资源进行推流等业务功能，hub与app 必须属于同一个七牛账户。 22 | "title": 'python_test_app', # Title: app 的名称，可选，注意，Title 不是唯一标识，重复 create 动作将生成多个 app。 23 | # "maxUsers": MaxUsers, # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 24 | # "noAutoKickUser": NoAutoKickUser # NoAutoKickUser: bool 类型，可选，禁止自动踢人（抢流）。默认为 false ， 25 | # 即同一个身份的 client (app/room/user) ，新的连麦请求可以成功，旧连接被关闭。 26 | } 27 | # 然后运行 rtc.CreateApp(<创建APP相关参数的字典变量>) 28 | print(rtc.create_app(create_data)) 29 | 30 | # 查询一个APP 31 | # 查询某一个具体的APP的相关信息的方法为 print( rtc.GetApp() ) ，其中 app_id 是类似 'desls83s2' 32 | # 这样在创建时由七牛自动生成的数字字母乱序组合的字符串 33 | # 如果不指定具体的app_id，直接运行 print( rtc.GetApp() ) ，那么就会列举出该账号下所有的APP 34 | print(rtc.get_app(':可选填')) 35 | 36 | # 删除一个APP 37 | # 使用方法为：rtc.DeleteApp()，例如： rtc.DeleteApp('desls83s2') 38 | print(rtc.delete_app(':必填')) 39 | 40 | # 更新一个APP的相关参数 41 | # 首先需要写好更新的APP的各个参数。参数如下： 42 | update_data = { 43 | "hub": "python_new_hub", # Hub: 绑定的直播 hub，可选，用于合流后 rtmp 推流。 44 | "title": "python_new_app", # Title: app 的名称， 可选。 45 | # "maxUsers": , # MaxUsers: int 类型，可选，连麦房间支持的最大在线人数。 46 | # "noAutoKickUser": , # NoAutoKickUser: bool 类型，可选，禁止自动踢人。 47 | # "mergePublishRtmp": { # MergePublishRtmp: 连麦合流转推 RTMP 的配置，可选择。其详细配置包括如下 48 | # "enable": , # Enable: 布尔类型，用于开启和关闭所有房间的合流功能。 49 | # "audioOnly": , # AudioOnly: 布尔类型，可选，指定是否只合成音频。 50 | # "height": , # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 51 | # "width": , # Height, Width: int64，可选，指定合流输出的高和宽，默认为 640 x 480。 52 | # "fps": , # OutputFps: int64，可选，指定合流输出的帧率，默认为 25 fps 。 53 | # "kbps": , # OutputKbps: int64，可选，指定合流输出的码率，默认为 1000 。 54 | # "url": "", # URL: 合流后转推旁路直播的地址，可选，支持魔法变量配置按照连麦房间号生成不同 55 | # 的推流地址。如果是转推到七牛直播云，不建议使用该配置。 56 | 57 | # "streamTitle": "" # StreamTitle: 转推七牛直播云的流名，可选，支持魔法变量配置按照连麦房间号 58 | # 生成不同的流名。例如，配置 Hub 为 qn-zhibo ，配置 StreamTitle 为 $(roomName) ， 59 | # 则房间 meeting-001 的合流将会被转推到 rtmp://pili-publish.qn-zhibo.***.com/qn-zhibo/meeting-001地址。 60 | # 详细配置细则，请咨询七牛技术支持。 61 | # } 62 | } 63 | # 使用方法为：rtc.UpdateApp(':必填', update_data)，例如：app_id 是形如 desmfnkw5 的字符串 64 | print(rtc.update_app(':必填', update_data)) 65 | 66 | # 列举一个APP下面，某个房间的所有用户 67 | print(rtc.list_user(':必填', '<房间名>:必填')) 68 | 69 | # 踢出一个APP下面，某个房间的某个用户 70 | print(rtc.kick_user(':必填', '<房间名>:必填', '<客户ID>:必填')) 71 | 72 | # 列举一个APP下面，所有的房间 73 | print(rtc.list_active_rooms(':必填')) 74 | 75 | # 计算房间管理鉴权。连麦用户终端通过房间管理鉴权获取七牛连麦服务 76 | # 首先需要写好房间鉴权的各个参数。参数如下： 77 | roomAccess = { 78 | "appId": ":必填", # AppID: 房间所属帐号的 app 。 79 | "roomName": "<房间名>:必填", # RoomName: 房间名称，需满足规格 ^[a-zA-Z0-9_-]{3,64}$ 80 | "userId": "<用户名>:必填", # UserID: 请求加入房间的用户 ID，需满足规格 ^[a-zA-Z0-9_-]{3,50}$ 81 | # ExpireAt: int64 类型，鉴权的有效时间，传入以秒为单位的64位Unix绝对时间， 82 | "expireAt": int(time.time()) + 3600, 83 | # token 将在该时间后失效。 84 | "permission": "user" # 该用户的房间管理权限，"admin" 或 "user"，默认为 "user" 。当权限角色为 "admin" 时， 85 | # 拥有将其他用户移除出房间等特权. 86 | } 87 | # 获得房间管理鉴权的方法：print(RtcRoomToken ( access_key, secret_key, roomAccess ) ) 88 | print(get_room_token(access_key, secret_key, roomAccess)) 89 | -------------------------------------------------------------------------------- /examples/set_object_lifecycle.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | from qiniu import BucketManager 6 | 7 | access_key = 'your_ak' 8 | secret_key = 'your_sk' 9 | 10 | # 初始化 Auth 11 | q = Auth(access_key, secret_key) 12 | 13 | # 初始化 BucketManager 14 | bucket = BucketManager(q) 15 | 16 | # 目标空间 17 | bucket_name = 'your_bucket_name' 18 | # 目标 key 19 | key = 'path/to/key' 20 | 21 | # bucket_name 更新 rule 22 | ret, info = bucket.set_object_lifecycle( 23 | bucket=bucket_name, 24 | key=key, 25 | to_line_after_days=10, 26 | to_archive_after_days=20, 27 | to_deep_archive_after_days=30, 28 | delete_after_days=40, 29 | cond={ 30 | 'hash': 'object_hash' 31 | } 32 | ) 33 | print(ret) 34 | print(info) 35 | -------------------------------------------------------------------------------- /examples/sms_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import QiniuMacAuth 5 | from qiniu import Sms 6 | import os 7 | 8 | access_key = os.getenv('QINIU_ACCESS_KEY') 9 | secret_key = os.getenv('QINIU_SECRET_KEY') 10 | 11 | # 初始化Auth状态 12 | q = QiniuMacAuth(access_key, secret_key) 13 | 14 | # 初始化Sms 15 | sms = Sms(q) 16 | 17 | """ 18 | #创建签名 19 | signature = 'abs' 20 | source = 'website' 21 | req, info = sms.createSignature(signature, source) 22 | print(req,info) 23 | """ 24 | 25 | """ 26 | #查询签名 27 | audit_status = '' 28 | page = 1 29 | page_size = 20 30 | req, info = sms.querySignature(audit_status, page, page_size) 31 | print(req, info) 32 | """ 33 | 34 | """ 35 | 编辑签名 36 | id = 1136530250662940672 37 | signature = 'sssss' 38 | req, info = sms.updateSignature(id, signature) 39 | print(req, info) 40 | """ 41 | 42 | """ 43 | #删除签名 44 | signature_id= 1136530250662940672 45 | req, info = sms.deleteSignature(signature_id) 46 | print(req, info) 47 | """ 48 | 49 | """ 50 | #创建模版 51 | name = '06-062-test' 52 | template = '${test}' 53 | type = 'notification' 54 | description = '就测试啊' 55 | signature_id = '1131464448834277376' 56 | req, info = sms.createTemplate(name, template, type, description, signature_id) 57 | print(req, info) 58 | """ 59 | 60 | """ 61 | #查询模版 62 | audit_status = '' 63 | page = 1 64 | page_size = 20 65 | req, info = sms.queryTemplate(audit_status, page, page_size) 66 | print(req, info) 67 | """ 68 | 69 | """ 70 | #编辑模版 71 | template_id = '1136589777022226432' 72 | name = '06-06-test' 73 | template = 'hi,你好' 74 | description = '就测试啊' 75 | signature_id = '1131464448834277376' 76 | req, info = sms.updateTemplate(template_id, name, template, description, signature_id) 77 | print(info) 78 | """ 79 | 80 | """ 81 | #删除模版 82 | template_id = '1136589777022226432' 83 | req, info = sms.deleteTemplate(template_id) 84 | print(req, info) 85 | """ 86 | 87 | """ 88 | # 查询短信发送记录 89 | req, info = sms.get_messages_info() 90 | print(req, info) 91 | """ 92 | 93 | """ 94 | #发送短信 95 | """ 96 | template_id = '' 97 | mobiles = [] 98 | parameters = {} 99 | req, info = sms.sendMessage(template_id, mobiles, parameters) 100 | print(req, info) 101 | -------------------------------------------------------------------------------- /examples/stat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth 4 | from qiniu import BucketManager 5 | 6 | access_key = '...' 7 | secret_key = '...' 8 | 9 | # 初始化Auth状态 10 | q = Auth(access_key, secret_key) 11 | 12 | # 初始化BucketManager 13 | bucket = BucketManager(q) 14 | 15 | # 你要测试的空间， 并且这个key在你空间中存在 16 | bucket_name = 'Bucket_Name' 17 | key = 'python-logo.png' 18 | 19 | # 获取文件的状态信息 20 | ret, info = bucket.stat(bucket_name, key) 21 | print(info) 22 | assert 'hash' in ret 23 | -------------------------------------------------------------------------------- /examples/timestamp_url.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 获取一个配置时间戳防盗链的url 6 | """ 7 | 8 | from qiniu.services.cdn.manager import create_timestamp_anti_leech_url 9 | import time 10 | 11 | host = 'http://a.example.com' 12 | 13 | # 配置时间戳时指定的key 14 | encrypt_key = '' 15 | 16 | # 资源路径 17 | file_name = 'a/b/c/example.jpeg' 18 | 19 | # 查询字符串,不需加? 20 | query_string = '' 21 | 22 | # 截止日期的时间戳,秒为单位，3600为当前时间一小时之后过期 23 | deadline = int(time.time()) + 3600 24 | 25 | 26 | timestamp_url = create_timestamp_anti_leech_url( 27 | host, file_name, query_string, encrypt_key, deadline) 28 | 29 | print(timestamp_url) 30 | -------------------------------------------------------------------------------- /examples/update_cdn_sslcert.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | """ 5 | 更新cdn证书(可配合let's encrypt 等完成自动证书更新) 6 | """ 7 | import qiniu 8 | from qiniu import DomainManager 9 | 10 | # 账户ak，sk 11 | access_key = '' 12 | secret_key = '' 13 | 14 | auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) 15 | domain_manager = DomainManager(auth) 16 | 17 | privatekey = "ssl/www.qiniu.com/privkey.pem" 18 | ca = "ssl/www.qiniu.com/fullchain.pem" 19 | domain_name = 'www.qiniu.com' 20 | 21 | with open(privatekey, 'r') as f: 22 | privatekey_str = f.read() 23 | 24 | with open(ca, 'r') as f: 25 | ca_str = f.read() 26 | 27 | ret, info = domain_manager.create_sslcert( 28 | domain_name, domain_name, privatekey_str, ca_str) 29 | print(ret['certID']) 30 | 31 | ret, info = domain_manager.put_httpsconf(domain_name, ret['certID'], False) 32 | print(info) 33 | -------------------------------------------------------------------------------- /examples/upload.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | # import hashlib 4 | 5 | from qiniu import Auth, put_file, urlsafe_base64_encode 6 | import qiniu.config 7 | from qiniu.compat import is_py2, is_py3 8 | 9 | # 需要填写你的 Access Key 和 Secret Key 10 | access_key = '...' 11 | secret_key = '...' 12 | 13 | # 构建鉴权对象 14 | q = Auth(access_key, secret_key) 15 | 16 | # 要上传的空间 17 | bucket_name = '' 18 | 19 | # 上传到七牛后保存的文件名 20 | key = 'my-python-七牛.png' 21 | 22 | # 生成上传 Token，可以指定过期时间等 23 | token = q.upload_token(bucket_name, key, 3600) 24 | 25 | # 要上传文件的本地路径 26 | localfile = '/Users/jemy/Documents/qiniu.png' 27 | 28 | # 上传时，sdk 会自动计算文件 hash 作为参数传递给服务端确保上传完整性 29 | # （若不一致，服务端会拒绝完成上传） 30 | # 但在访问文件时，服务端可能不会提供 MD5 或者编码格式不是期望的 31 | # 因此若有需有，请通过元数据功能自定义 MD5 或其他 hash 字段 32 | # hasher = hashlib.md5() 33 | # with open(localfile, 'rb') as f: 34 | # for d in f: 35 | # hasher.update(d) 36 | # object_metadata = { 37 | # 'x-qn-meta-md5': hasher.hexdigest() 38 | # } 39 | 40 | ret, info = put_file( 41 | token, 42 | key, 43 | localfile 44 | # metadata=object_metadata 45 | ) 46 | print(ret) 47 | print(info) 48 | 49 | if is_py2: 50 | assert ret['key'].encode('utf-8') == key 51 | elif is_py3: 52 | assert ret['key'] == key 53 | -------------------------------------------------------------------------------- /examples/upload_callback.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth, put_file 5 | 6 | access_key = '...' 7 | secret_key = '...' 8 | 9 | q = Auth(access_key, secret_key) 10 | 11 | bucket_name = 'Bucket_Name' 12 | 13 | key = 'my-python-logo.png' 14 | 15 | # 上传文件到七牛后， 七牛将文件名和文件大小回调给业务服务器。 16 | policy = { 17 | 'callbackUrl': 'http://your.domain.com/callback.php', 18 | 'callbackBody': 'filename=$(fname)&filesize=$(fsize)' 19 | } 20 | 21 | token = q.upload_token(bucket_name, key, 3600, policy) 22 | 23 | localfile = './sync/bbb.jpg' 24 | 25 | ret, info = put_file(token, key, localfile) 26 | print(info) 27 | assert ret['key'] == key 28 | -------------------------------------------------------------------------------- /examples/upload_pfops.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | from qiniu import Auth, put_file, urlsafe_base64_encode 4 | 5 | access_key = '...' 6 | secret_key = '...' 7 | 8 | # 初始化Auth状态 9 | q = Auth(access_key, secret_key) 10 | 11 | # 你要测试的空间， 并且这个key在你空间中存在 12 | bucket_name = 'Bucket_Name' 13 | key = 'python_video.flv' 14 | 15 | # 指定转码使用的队列名称 16 | pipeline = 'your_pipeline' 17 | 18 | # 设置转码参数（以视频转码为例） 19 | fops = 'avthumb/mp4/vcodec/libx264' 20 | 21 | # 通过添加'|saveas'参数，指定处理后的文件保存的bucket和key，不指定默认保存在当前空间，bucket_saved为目标bucket，bucket_saved为目标key 22 | saveas_key = urlsafe_base64_encode('bucket_saved:bucket_saved') 23 | 24 | fops = fops + '|saveas/' + saveas_key 25 | 26 | # 在上传策略中指定fobs和pipeline 27 | policy = { 28 | 'persistentOps': fops, 29 | 'persistentPipeline': pipeline 30 | } 31 | 32 | token = q.upload_token(bucket_name, key, 3600, policy) 33 | 34 | localfile = './python_video.flv' 35 | 36 | ret, info = put_file(token, key, localfile) 37 | print(info) 38 | assert ret['key'] == key 39 | -------------------------------------------------------------------------------- /examples/upload_token.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth 5 | 6 | # 需要填写你的 Access Key 和 Secret Key 7 | access_key = '' 8 | secret_key = '' 9 | 10 | # 构建鉴权对象 11 | q = Auth(access_key, secret_key) 12 | 13 | # 要上传的空间 14 | bucket_name = '' 15 | 16 | # 上传到七牛后保存的文件名 17 | key = '' 18 | 19 | # 生成上传 Token，可以指定过期时间等 20 | 21 | # 上传策略示例 22 | # https://developer.qiniu.com/kodo/manual/1206/put-policy 23 | policy = { 24 | # 'callbackUrl':'https://requestb.in/1c7q2d31', 25 | # 'callbackBody':'filename=$(fname)&filesize=$(fsize)' 26 | # 'persistentOps':'imageView2/1/w/200/h/200' 27 | } 28 | 29 | token = q.upload_token(bucket_name, key, 3600, policy) 30 | 31 | print(token) 32 | -------------------------------------------------------------------------------- /examples/upload_with_qvmzone.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth, put_file, urlsafe_base64_encode 5 | import qiniu.config 6 | from qiniu import Zone, set_default 7 | 8 | # 需要填写你的 Access Key 和 Secret Key 9 | access_key = '...' 10 | secret_key = '...' 11 | 12 | # 构建鉴权对象 13 | q = Auth(access_key, secret_key) 14 | 15 | # 要上传的空间 16 | bucket_name = 'Bucket_Name' 17 | 18 | # 上传到七牛后保存的文件名 19 | key = 'my-python-logo.png' 20 | 21 | # 生成上传 Token，可以指定过期时间等 22 | token = q.upload_token(bucket_name, key, 3600) 23 | 24 | # 要上传文件的本地路径 25 | localfile = 'stat.py' 26 | 27 | # up_host, 指定上传域名，注意不同区域的qvm上传域名不同 28 | # https://developer.qiniu.com/qvm/manual/4269/qvm-kodo 29 | 30 | zone = Zone( 31 | up_host='free-qvm-z1-zz.qiniup.com', 32 | up_host_backup='free-qvm-z1-zz.qiniup.com', 33 | io_host='iovip.qbox.me', 34 | scheme='http') 35 | set_default(default_zone=zone) 36 | 37 | ret, info = put_file(token, key, localfile) 38 | print(info) 39 | assert ret['key'] == key 40 | -------------------------------------------------------------------------------- /examples/upload_with_zone.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # flake8: noqa 3 | 4 | from qiniu import Auth, put_file 5 | from qiniu import Zone, set_default 6 | 7 | # 需要填写你的 Access Key 和 Secret Key 8 | access_key = '' 9 | secret_key = '' 10 | 11 | # 构建鉴权对象 12 | q = Auth(access_key, secret_key) 13 | 14 | # 要上传的空间 15 | bucket_name = 'bucket_name' 16 | 17 | # 上传到七牛后保存的文件名 18 | key = 'a.jpg' 19 | 20 | # 生成上传 Token，可以指定过期时间等 21 | token = q.upload_token(bucket_name, key, 3600) 22 | 23 | # 要上传文件的本地路径 24 | localfile = '/Users/abc/Documents/a.jpg' 25 | 26 | # 指定固定域名的zone,不同区域uphost域名见下文档 27 | # https://developer.qiniu.com/kodo/manual/1671/region-endpoint 28 | # 未指定或上传错误，sdk会根据token自动查询对应的上传域名 29 | # *.qiniup.com 支持https上传 30 | # 备用*.qiniu.com域名 不支持https上传 31 | # 要求https上传时，如果客户指定的两个host都错误，且sdk自动查询的第一个*.qiniup.com上传域名因意外不可用导致访问到备用*.qiniu.com会报ssl错误 32 | # 建议https上传时查看上面文档，指定正确的host 33 | 34 | zone = Zone( 35 | up_host='https://up.qiniup.com', 36 | up_host_backup='https://upload.qiniup.com', 37 | io_host='http://iovip.qbox.me', 38 | scheme='https') 39 | set_default(default_zone=zone) 40 | 41 | ret, info = put_file(token, key, localfile) 42 | print(info) 43 | -------------------------------------------------------------------------------- /qiniu/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | Qiniu Resource Storage SDK for Python 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | For detailed document, please see: 7 | 8 | ''' 9 | 10 | # flake8: noqa 11 | 12 | __version__ = '7.16.0' 13 | 14 | from .auth import Auth, QiniuMacAuth 15 | 16 | from .config import set_default 17 | from .zone import Zone 18 | from .region import LegacyRegion as Region 19 | 20 | from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ 21 | build_batch_stat, build_batch_delete, build_batch_restoreAr, build_batch_restore_ar 22 | from .services.storage.uploader import put_data, put_file, put_stream 23 | from .services.storage.upload_progress_recorder import UploadProgressRecorder 24 | from .services.cdn.manager import CdnManager, DataType, create_timestamp_anti_leech_url, DomainManager 25 | from .services.processing.pfop import PersistentFop 26 | from .services.processing.cmd import build_op, pipe_cmd, op_save 27 | from .services.compute.app import AccountClient 28 | from .services.compute.qcos_api import QcosClient 29 | from .services.sms.sms import Sms 30 | from .services.pili.rtc_server_manager import RtcServer, get_room_token 31 | from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, decode_entry, canonical_mime_header_key 32 | -------------------------------------------------------------------------------- /qiniu/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | pythoncompat 5 | """ 6 | 7 | import os 8 | import sys 9 | 10 | try: 11 | import simplejson as json 12 | except (ImportError, SyntaxError): 13 | # simplejson does not support Python 3.2, it thows a SyntaxError 14 | # because of u'...' Unicode literals. 15 | import json # noqa 16 | 17 | # ------- 18 | # Platform 19 | # ------- 20 | 21 | is_windows = sys.platform == 'win32' 22 | is_linux = sys.platform == 'linux' 23 | is_macos = sys.platform == 'darwin' 24 | 25 | # ------- 26 | # Pythons 27 | # ------- 28 | 29 | _ver = sys.version_info 30 | 31 | #: Python 2.x? 32 | is_py2 = (_ver[0] == 2) 33 | 34 | #: Python 3.x? 35 | is_py3 = (_ver[0] == 3) 36 | 37 | 38 | # --------- 39 | # Specifics 40 | # --------- 41 | 42 | if is_py2: 43 | from urllib import urlencode # noqa 44 | from urlparse import urlparse # noqa 45 | import StringIO 46 | StringIO = BytesIO = StringIO.StringIO 47 | 48 | builtin_str = str 49 | bytes = str 50 | str = unicode # noqa 51 | basestring = basestring # noqa 52 | numeric_types = (int, long, float) # noqa 53 | 54 | def b(data): 55 | return bytes(data) 56 | 57 | def s(data): 58 | return bytes(data) 59 | 60 | def u(data): 61 | return unicode(data, 'unicode_escape') # noqa 62 | 63 | def is_seekable(data): 64 | try: 65 | data.seek(0, os.SEEK_CUR) 66 | return True 67 | except (AttributeError, IOError): 68 | return False 69 | 70 | elif is_py3: 71 | from urllib.parse import urlparse, urlencode # noqa 72 | import io 73 | StringIO = io.StringIO 74 | BytesIO = io.BytesIO 75 | 76 | builtin_str = str 77 | str = str 78 | bytes = bytes 79 | basestring = (str, bytes) 80 | numeric_types = (int, float) 81 | 82 | def b(data): 83 | if isinstance(data, str): 84 | return data.encode('utf-8') 85 | return data 86 | 87 | def s(data): 88 | if isinstance(data, bytes): 89 | data = data.decode('utf-8') 90 | return data 91 | 92 | def u(data): 93 | return data 94 | 95 | def is_seekable(data): 96 | return data.seekable() 97 | -------------------------------------------------------------------------------- /qiniu/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | RS_HOST = 'http://rs.qiniu.com' # 管理操作Host 3 | RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host 4 | API_HOST = 'http://api.qiniuapi.com' # 数据处理操作Host 5 | QUERY_REGION_HOST = 'https://uc.qiniuapi.com' 6 | QUERY_REGION_BACKUP_HOSTS = [ 7 | 'kodo-config.qiniuapi.com', 8 | 'uc.qbox.me' 9 | ] 10 | UC_HOST = QUERY_REGION_HOST # 获取空间信息Host 11 | UC_BACKUP_HOSTS = QUERY_REGION_BACKUP_HOSTS 12 | 13 | _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 14 | 15 | _config = { 16 | 'default_zone': None, 17 | 'default_rs_host': RS_HOST, 18 | 'default_rsf_host': RSF_HOST, 19 | 'default_api_host': API_HOST, 20 | 'default_uc_host': UC_HOST, 21 | 'default_uc_backup_hosts': UC_BACKUP_HOSTS, 22 | 'default_query_region_host': QUERY_REGION_HOST, 23 | 'default_query_region_backup_hosts': QUERY_REGION_BACKUP_HOSTS, 24 | 'default_backup_hosts_retry_times': 3, # 仅控制旧区域 LegacyRegion 查询 Hosts 的重试次数 25 | 'connection_timeout': 30, # 链接超时为时间为30s 26 | 'connection_retries': 3, # 链接重试次数为3次 27 | 'connection_pool': 10, # 链接池个数为10 28 | 'default_upload_threshold': 2 * _BLOCK_SIZE # put_file上传方式的临界默认值 29 | } 30 | 31 | _is_customized_default = { 32 | k: False 33 | for k in _config.keys() 34 | } 35 | 36 | 37 | def is_customized_default(key): 38 | return _is_customized_default[key] 39 | 40 | 41 | def get_default(key): 42 | if key == 'default_zone' and not _is_customized_default[key]: 43 | # prevent circle import 44 | from .region import LegacyRegion 45 | return LegacyRegion() 46 | return _config[key] 47 | 48 | 49 | def set_default( 50 | default_zone=None, connection_retries=None, connection_pool=None, 51 | connection_timeout=None, default_rs_host=None, default_uc_host=None, 52 | default_rsf_host=None, default_api_host=None, default_upload_threshold=None, 53 | default_query_region_host=None, default_query_region_backup_hosts=None, 54 | default_backup_hosts_retry_times=None, default_uc_backup_hosts=None): 55 | if default_zone: 56 | _config['default_zone'] = default_zone 57 | _is_customized_default['default_zone'] = True 58 | if default_rs_host: 59 | _config['default_rs_host'] = default_rs_host 60 | _is_customized_default['default_rs_host'] = True 61 | if default_rsf_host: 62 | _config['default_rsf_host'] = default_rsf_host 63 | _is_customized_default['default_rsf_host'] = True 64 | if default_api_host: 65 | _config['default_api_host'] = default_api_host 66 | _is_customized_default['default_api_host'] = True 67 | if default_uc_host: 68 | _config['default_uc_host'] = default_uc_host 69 | _is_customized_default['default_uc_host'] = True 70 | _config['default_uc_backup_hosts'] = [] 71 | _is_customized_default['default_uc_backup_hosts'] = True 72 | _config['default_query_region_host'] = default_uc_host 73 | _is_customized_default['default_query_region_host'] = True 74 | _config['default_query_region_backup_hosts'] = [] 75 | _is_customized_default['default_query_region_backup_hosts'] = True 76 | if default_uc_backup_hosts is not None: 77 | _config['default_uc_backup_hosts'] = default_uc_backup_hosts 78 | _is_customized_default['default_uc_backup_hosts'] = True 79 | _config['default_query_region_backup_hosts'] = default_uc_backup_hosts 80 | _is_customized_default['default_query_region_backup_hosts'] = True 81 | if default_query_region_host: 82 | _config['default_query_region_host'] = default_query_region_host 83 | _is_customized_default['default_query_region_host'] = True 84 | _config['default_query_region_backup_hosts'] = [] 85 | _is_customized_default['default_query_region_backup_hosts'] = True 86 | if default_query_region_backup_hosts is not None: 87 | _config['default_query_region_backup_hosts'] = default_query_region_backup_hosts 88 | _is_customized_default['default_query_region_backup_hosts'] = True 89 | if default_backup_hosts_retry_times: 90 | _config['default_backup_hosts_retry_times'] = default_backup_hosts_retry_times 91 | _is_customized_default['default_backup_hosts_retry_times'] = True 92 | if connection_retries: 93 | _config['connection_retries'] = connection_retries 94 | _is_customized_default['connection_retries'] = True 95 | if connection_pool: 96 | _config['connection_pool'] = connection_pool 97 | _is_customized_default['connection_pool'] = True 98 | if connection_timeout: 99 | _config['connection_timeout'] = connection_timeout 100 | _is_customized_default['connection_timeout'] = True 101 | if default_upload_threshold: 102 | _config['default_upload_threshold'] = default_upload_threshold 103 | _is_customized_default['default_upload_threshold'] = True 104 | -------------------------------------------------------------------------------- /qiniu/http/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | 4 | import requests 5 | 6 | from qiniu.config import get_default 7 | from .response import ResponseInfo 8 | from .middleware import compose_middleware 9 | 10 | 11 | class HTTPClient: 12 | def __init__(self, middlewares=None, send_opts=None): 13 | self.session = requests.Session() 14 | self.middlewares = [] if middlewares is None else middlewares 15 | self.send_opts = {} if send_opts is None else send_opts 16 | 17 | def _wrap_send(self, req, **kwargs): 18 | # compatibility with setting timeout by qiniu.config.set_default 19 | kwargs.setdefault('timeout', get_default('connection_timeout')) 20 | 21 | resp = self.session.send(req.prepare(), **kwargs) 22 | return ResponseInfo(resp, None) 23 | 24 | def send_request(self, request, middlewares=None, **kwargs): 25 | """ 26 | 27 | Args: 28 | request (requests.Request): 29 | requests.Request 对象 30 | 31 | middlewares (list[qiniu.http.middleware.Middleware] or (list[qiniu.http.middleware.Middleware]) -> list[qiniu.http.middleware.Middleware]): 32 | 仅对本次请求生效的中间件。 33 | 34 | 如果传入的是列表，那么会作为追加的中间件拼接到 Client 中间件的后面。 35 | 36 | 也可传入函数，获得 Client 中间件的一个副本来做更细的控制。 37 | 例如拼接到 Client 中间件的前面，可以这样使用： 38 | 39 | c.send_request(my_req, middlewares=lambda mws: my_mws + mws) 40 | 41 | kwargs: 42 | 将作为其他参数直接透传给 session.send 方法 43 | 44 | 45 | Returns: 46 | (dict, ResponseInfo): 可拆包的一个元组。 47 | 第一个元素为响应体的 dict，若响应体为 json 的话。 48 | 第二个元素为包装过的响应内容，包括了更多的响应内容。 49 | 50 | """ 51 | 52 | # set default values 53 | middlewares = [] if middlewares is None else middlewares 54 | 55 | # join middlewares and client middlewares 56 | mw_ls = [] 57 | if callable(middlewares): 58 | mw_ls = middlewares(self.middlewares.copy()) 59 | elif isinstance(middlewares, list): 60 | mw_ls = self.middlewares + middlewares 61 | 62 | # send request 63 | try: 64 | handle = compose_middleware( 65 | mw_ls, 66 | lambda req: self._wrap_send(req, **kwargs) 67 | ) 68 | resp_info = handle(request) 69 | except Exception as e: 70 | return None, ResponseInfo(None, e) 71 | 72 | # if ok try dump response info to dict from json 73 | if not resp_info.ok(): 74 | return None, resp_info 75 | 76 | try: 77 | ret = resp_info.json() 78 | except ValueError: 79 | logging.debug("response body decode error: %s" % resp_info.text_body) 80 | ret = {} 81 | return ret, resp_info 82 | 83 | def get( 84 | self, 85 | url, 86 | params=None, 87 | auth=None, 88 | headers=None, 89 | middlewares=None, 90 | **kwargs 91 | ): 92 | req = requests.Request( 93 | method='get', 94 | url=url, 95 | params=params, 96 | auth=auth, 97 | headers=headers 98 | ) 99 | send_opts = self.send_opts.copy() 100 | send_opts.update(kwargs) 101 | send_opts.setdefault("allow_redirects", True) 102 | return self.send_request( 103 | req, 104 | middlewares=middlewares, 105 | **send_opts 106 | ) 107 | 108 | def post( 109 | self, 110 | url, 111 | data, 112 | files, 113 | auth=None, 114 | headers=None, 115 | middlewares=None, 116 | **kwargs 117 | ): 118 | req = requests.Request( 119 | method='post', 120 | url=url, 121 | data=data, 122 | files=files, 123 | auth=auth, 124 | headers=headers 125 | ) 126 | send_opts = self.send_opts.copy() 127 | send_opts.update(kwargs) 128 | return self.send_request( 129 | req, 130 | middlewares=middlewares, 131 | **send_opts 132 | ) 133 | 134 | def put( 135 | self, 136 | url, 137 | data, 138 | files, 139 | auth=None, 140 | headers=None, 141 | middlewares=None, 142 | **kwargs 143 | ): 144 | req = requests.Request( 145 | method='put', 146 | url=url, 147 | data=data, 148 | files=files, 149 | auth=auth, 150 | headers=headers 151 | ) 152 | send_opts = self.send_opts.copy() 153 | send_opts.update(kwargs) 154 | return self.send_request( 155 | req, 156 | middlewares=middlewares, 157 | **send_opts 158 | ) 159 | 160 | def delete( 161 | self, 162 | url, 163 | params, 164 | auth=None, 165 | headers=None, 166 | middlewares=None, 167 | **kwargs 168 | ): 169 | req = requests.Request( 170 | method='delete', 171 | url=url, 172 | params=params, 173 | auth=auth, 174 | headers=headers 175 | ) 176 | send_opts = self.send_opts.copy() 177 | send_opts.update(kwargs) 178 | return self.send_request( 179 | req, 180 | middlewares=middlewares, 181 | **send_opts 182 | ) 183 | -------------------------------------------------------------------------------- /qiniu/http/default_client.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | from requests.adapters import HTTPAdapter 4 | 5 | from qiniu import config, __version__ 6 | 7 | from .client import HTTPClient 8 | from .middleware import UserAgentMiddleware 9 | 10 | qn_http_client = HTTPClient( 11 | middlewares=[ 12 | UserAgentMiddleware(__version__) 13 | ] 14 | ) 15 | 16 | 17 | # compatibility with some config from qiniu.config 18 | def _before_send(func): 19 | @functools.wraps(func) 20 | def wrapper(self, *args, **kwargs): 21 | _init_http_adapter() 22 | return func(self, *args, **kwargs) 23 | 24 | return wrapper 25 | 26 | 27 | qn_http_client.send_request = _before_send(qn_http_client.send_request) 28 | 29 | 30 | def _init_http_adapter(): 31 | # may be optimized: 32 | # only called when config changed, not every time before send request 33 | adapter = HTTPAdapter( 34 | pool_connections=config.get_default('connection_pool'), 35 | pool_maxsize=config.get_default('connection_pool'), 36 | max_retries=config.get_default('connection_retries')) 37 | qn_http_client.session.mount('http://', adapter) 38 | -------------------------------------------------------------------------------- /qiniu/http/endpoint.py: -------------------------------------------------------------------------------- 1 | class Endpoint: 2 | @staticmethod 3 | def from_host(host): 4 | """ 5 | Autodetect scheme from host string 6 | 7 | Parameters 8 | ---------- 9 | host: str 10 | 11 | Returns 12 | ------- 13 | Endpoint 14 | """ 15 | if '://' in host: 16 | scheme, host = host.split('://') 17 | return Endpoint(host=host, default_scheme=scheme) 18 | else: 19 | return Endpoint(host=host) 20 | 21 | def __init__(self, host, default_scheme='https'): 22 | """ 23 | Parameters 24 | ---------- 25 | host: str 26 | default_scheme: str 27 | """ 28 | self.host = host 29 | self.default_scheme = default_scheme 30 | 31 | def __str__(self): 32 | return 'Endpoint(host:\'{0}\',default_scheme:\'{1}\')'.format( 33 | self.host, 34 | self.default_scheme 35 | ) 36 | 37 | def __repr__(self): 38 | return self.__str__() 39 | 40 | def __eq__(self, other): 41 | if not isinstance(other, Endpoint): 42 | raise TypeError('Cannot compare Endpoint with {0}'.format(type(other))) 43 | 44 | return self.host == other.host and self.default_scheme == other.default_scheme 45 | 46 | def get_value(self, scheme=None): 47 | """ 48 | Parameters 49 | ---------- 50 | scheme: str 51 | 52 | Returns 53 | ------- 54 | str 55 | """ 56 | scheme = scheme if scheme is not None else self.default_scheme 57 | return ''.join([scheme, '://', self.host]) 58 | 59 | def clone(self): 60 | """ 61 | Returns 62 | ------- 63 | Endpoint 64 | """ 65 | return Endpoint( 66 | host=self.host, 67 | default_scheme=self.default_scheme 68 | ) 69 | -------------------------------------------------------------------------------- /qiniu/http/endpoints_provider.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class EndpointsProvider: 5 | __metaclass__ = abc.ABCMeta 6 | 7 | @abc.abstractmethod 8 | def __iter__(self): 9 | """ 10 | Returns 11 | ------- 12 | list[Endpoint] 13 | """ 14 | -------------------------------------------------------------------------------- /qiniu/http/endpoints_retry_policy.py: -------------------------------------------------------------------------------- 1 | from qiniu.retry.abc import RetryPolicy 2 | 3 | 4 | class EndpointsRetryPolicy(RetryPolicy): 5 | def __init__(self, endpoints_provider=None, skip_init_context=False): 6 | """ 7 | Parameters 8 | ---------- 9 | endpoints_provider: Iterable[Endpoint] 10 | skip_init_context: bool 11 | """ 12 | self.endpoints_provider = endpoints_provider if endpoints_provider else [] 13 | self.skip_init_context = skip_init_context 14 | 15 | def init_context(self, context): 16 | """ 17 | Parameters 18 | ---------- 19 | context: dict 20 | 21 | Returns 22 | ------- 23 | None 24 | """ 25 | if self.skip_init_context: 26 | return 27 | context['alternative_endpoints'] = list(self.endpoints_provider) 28 | if not context['alternative_endpoints']: 29 | raise ValueError('There isn\'t available endpoint') 30 | context['endpoint'] = context['alternative_endpoints'].pop(0) 31 | 32 | def should_retry(self, attempt): 33 | """ 34 | Parameters 35 | ---------- 36 | attempt: qiniu.retry.Attempt 37 | 38 | Returns 39 | ------- 40 | bool 41 | """ 42 | return len(attempt.context['alternative_endpoints']) > 0 43 | 44 | def prepare_retry(self, attempt): 45 | """ 46 | Parameters 47 | ---------- 48 | attempt: qiniu.retry.Attempt 49 | 50 | Returns 51 | ------- 52 | None 53 | """ 54 | if not attempt.context['alternative_endpoints']: 55 | raise Exception('There isn\'t available endpoint for next try') 56 | attempt.context['endpoint'] = attempt.context['alternative_endpoints'].pop(0) 57 | -------------------------------------------------------------------------------- /qiniu/http/middleware/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Middleware, compose_middleware 2 | from .ua import UserAgentMiddleware 3 | from .retry_domains import RetryDomainsMiddleware 4 | 5 | __all__ = [ 6 | 'Middleware', 'compose_middleware', 7 | 'UserAgentMiddleware', 8 | 'RetryDomainsMiddleware' 9 | ] 10 | -------------------------------------------------------------------------------- /qiniu/http/middleware/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from functools import reduce 3 | 4 | 5 | def compose_middleware(middlewares, handle): 6 | """ 7 | Args: 8 | middlewares (list[Middleware]): Middlewares 9 | handle ((requests.Request) -> qiniu.http.response.ResponseInfo): The send request handle 10 | 11 | Returns: 12 | (requests.Request) -> qiniu.http.response.ResponseInfo: Composed handle 13 | 14 | """ 15 | middlewares.reverse() 16 | 17 | return reduce( 18 | lambda h, mw: 19 | lambda req: mw(req, h), 20 | middlewares, 21 | handle 22 | ) 23 | 24 | 25 | class Middleware: 26 | def __call__(self, request, nxt): 27 | """ 28 | Args: 29 | request (requests.Request): 30 | nxt ((requests.Request) -> qiniu.http.response.ResponseInfo): 31 | 32 | Returns: 33 | requests.Response: 34 | 35 | """ 36 | raise NotImplementedError('{0}.__call__ method is not implemented yet'.format(type(self))) 37 | -------------------------------------------------------------------------------- /qiniu/http/middleware/retry_domains.py: -------------------------------------------------------------------------------- 1 | from qiniu.compat import urlparse 2 | 3 | from .base import Middleware 4 | 5 | 6 | class RetryDomainsMiddleware(Middleware): 7 | def __init__(self, backup_domains, max_retry_times=2, retry_condition=None): 8 | """ 9 | Args: 10 | backup_domains (list[str]): 11 | max_retry_times (int): 12 | retry_condition ((requests.Response or None, requests.Request)->bool): 13 | """ 14 | self.backup_domains = backup_domains 15 | self.max_retry_times = max_retry_times 16 | self.retry_condition = retry_condition 17 | 18 | self.retried_times = 0 19 | 20 | @staticmethod 21 | def _get_changed_url(url, domain): 22 | url_parse_result = urlparse(url) 23 | 24 | backup_netloc = '' 25 | has_user = False 26 | if url_parse_result.username is not None: 27 | backup_netloc += url_parse_result.username 28 | has_user = True 29 | if url_parse_result.password is not None: 30 | backup_netloc += url_parse_result.password 31 | has_user = True 32 | if has_user: 33 | backup_netloc += '@' 34 | backup_netloc += domain 35 | if url_parse_result.port is not None: 36 | backup_netloc += ':' + str(url_parse_result.port) 37 | 38 | # the _replace is a public method. start with `_` just to prevent conflicts with field names 39 | # see namedtuple docs 40 | url_parse_result = url_parse_result._replace( 41 | netloc=backup_netloc 42 | ) 43 | 44 | return url_parse_result.geturl() 45 | 46 | @staticmethod 47 | def _try_nxt(request, nxt): 48 | resp = None 49 | err = None 50 | try: 51 | resp = nxt(request) 52 | except Exception as e: 53 | err = e 54 | return resp, err 55 | 56 | def _should_retry(self, resp, req): 57 | if callable(self.retry_condition): 58 | return self.retry_condition(resp, req) 59 | 60 | return resp is None or resp.need_retry() 61 | 62 | def __call__(self, request, nxt): 63 | resp_info, err = None, None 64 | url_parse_result = urlparse(request.url) 65 | 66 | for backup_domain in [str(url_parse_result.hostname)] + self.backup_domains: 67 | request.url = RetryDomainsMiddleware._get_changed_url(request.url, backup_domain) 68 | self.retried_times = 0 69 | 70 | while self.retried_times < self.max_retry_times: 71 | resp_info, err = RetryDomainsMiddleware._try_nxt(request, nxt) 72 | self.retried_times += 1 73 | if not self._should_retry(resp_info, request): 74 | if err is not None: 75 | raise err 76 | return resp_info 77 | 78 | if err is not None: 79 | raise err 80 | 81 | return resp_info 82 | -------------------------------------------------------------------------------- /qiniu/http/middleware/ua.py: -------------------------------------------------------------------------------- 1 | import platform as _platform 2 | 3 | from .base import Middleware 4 | 5 | 6 | class UserAgentMiddleware(Middleware): 7 | def __init__(self, sdk_version): 8 | sys_info = '{0}; {1}'.format(_platform.system(), _platform.machine()) 9 | python_ver = _platform.python_version() 10 | 11 | user_agent = 'QiniuPython/{0} ({1}; ) Python/{2}'.format( 12 | sdk_version, sys_info, python_ver) 13 | 14 | self.user_agent = user_agent 15 | 16 | def __call__(self, request, nxt): 17 | if not request.headers: 18 | request.headers = {} 19 | request.headers['User-Agent'] = self.user_agent 20 | return nxt(request) 21 | -------------------------------------------------------------------------------- /qiniu/http/region.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from enum import Enum 4 | 5 | from .endpoint import Endpoint 6 | 7 | 8 | # Use StrEnum when min version of python update to >= 3.11 9 | # to make the json stringify more readable, 10 | # or find another way to simple the json stringify 11 | class ServiceName(Enum): 12 | UC = 'uc' 13 | UP = 'up' 14 | UP_ACC = 'up_acc' 15 | IO = 'io' 16 | # IO_SRC = 'io_src' 17 | RS = 'rs' 18 | RSF = 'rsf' 19 | API = 'api' 20 | 21 | 22 | class Region: 23 | @staticmethod 24 | def merge(*args): 25 | """ 26 | Parameters 27 | ---------- 28 | args: list[list[Region]] 29 | 30 | Returns 31 | ------- 32 | 33 | """ 34 | if not args: 35 | raise TypeError('There aren\'ta any regions to merge') 36 | source, rest = args[0], args[1:] 37 | target = source.clone() 38 | for r in rest: 39 | for sn, el in r.services.items(): 40 | if sn not in target.services: 41 | target.services[sn] = [e.clone() for e in el] 42 | else: 43 | target_values = [e.get_value() for e in target.services[sn]] 44 | target.services[sn] += [ 45 | e.clone() 46 | for e in el 47 | if e.get_value() not in target_values 48 | ] 49 | 50 | return target 51 | 52 | @staticmethod 53 | def from_region_id(region_id, **kwargs): 54 | """ 55 | Parameters 56 | ---------- 57 | region_id: str 58 | kwargs: dict 59 | s3_region_id: str 60 | ttl: int 61 | create_time: datetime 62 | extended_services: dict[str, list[Region]] 63 | preferred_scheme: str 64 | 65 | Returns 66 | ------- 67 | Region 68 | """ 69 | # create services endpoints 70 | endpoint_kwargs = { 71 | } 72 | if 'preferred_scheme' in kwargs: 73 | endpoint_kwargs['default_scheme'] = kwargs.get('preferred_scheme') 74 | 75 | is_z0 = region_id == 'z0' 76 | services_hosts = { 77 | ServiceName.UC: ['uc.qiniuapi.com'], 78 | ServiceName.UP: [ 79 | 'upload-{0}.qiniup.com'.format(region_id), 80 | 'up-{0}.qiniup.com'.format(region_id) 81 | ] if not is_z0 else [ 82 | 'upload.qiniup.com', 83 | 'up.qiniup.com' 84 | ], 85 | ServiceName.IO: [ 86 | 'iovip-{0}.qiniuio.com'.format(region_id), 87 | ] if not is_z0 else [ 88 | 'iovip.qiniuio.com', 89 | ], 90 | ServiceName.RS: [ 91 | 'rs-{0}.qiniuapi.com'.format(region_id), 92 | ], 93 | ServiceName.RSF: [ 94 | 'rsf-{0}.qiniuapi.com'.format(region_id), 95 | ], 96 | ServiceName.API: [ 97 | 'api-{0}.qiniuapi.com'.format(region_id), 98 | ] 99 | } 100 | services = { 101 | k: [ 102 | Endpoint(h, **endpoint_kwargs) for h in v 103 | ] 104 | for k, v in services_hosts.items() 105 | } 106 | services.update(kwargs.get('extended_services', {})) 107 | 108 | # create region 109 | region_kwargs = { 110 | k: kwargs.get(k) 111 | for k in [ 112 | 's3_region_id', 113 | 'ttl', 114 | 'create_time' 115 | ] if k in kwargs 116 | } 117 | region_kwargs['region_id'] = region_id 118 | region_kwargs.setdefault('s3_region_id', region_id) 119 | region_kwargs['services'] = services 120 | 121 | return Region(**region_kwargs) 122 | 123 | def __init__( 124 | self, 125 | region_id=None, 126 | s3_region_id=None, 127 | services=None, 128 | ttl=86400, 129 | create_time=None 130 | ): 131 | """ 132 | Parameters 133 | ---------- 134 | region_id: str 135 | s3_region_id: str 136 | services: dict[ServiceName or str, list[Endpoint]] 137 | ttl: int, default 86400 138 | create_time: datetime, default datetime.now() 139 | """ 140 | self.region_id = region_id 141 | self.s3_region_id = s3_region_id if s3_region_id else region_id 142 | 143 | self.services = services if services else {} 144 | self.services.update( 145 | { 146 | k: [] 147 | for k in ServiceName 148 | if 149 | k not in self.services or 150 | not isinstance(self.services[k], list) 151 | } 152 | ) 153 | 154 | self.ttl = ttl 155 | self.create_time = create_time if create_time else datetime.now() 156 | 157 | @property 158 | def is_live(self): 159 | """ 160 | Returns 161 | ------- 162 | bool 163 | """ 164 | if self.ttl < 0: 165 | return True 166 | live_time = datetime.now() - self.create_time 167 | return live_time < timedelta(seconds=self.ttl) 168 | 169 | def clone(self): 170 | """ 171 | Returns 172 | ------- 173 | Region 174 | """ 175 | return Region( 176 | region_id=self.region_id, 177 | s3_region_id=self.s3_region_id, 178 | services={ 179 | k: [endpoint.clone() for endpoint in self.services[k]] 180 | for k in self.services 181 | }, 182 | ttl=self.ttl, 183 | create_time=self.create_time 184 | ) 185 | -------------------------------------------------------------------------------- /qiniu/http/response.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from qiniu.compat import is_py2, is_py3 3 | 4 | 5 | class ResponseInfo(object): 6 | """七牛HTTP请求返回信息类 7 | 8 | 该类主要是用于获取和解析对七牛发起各种请求后的响应包的header和body。 9 | 10 | Attributes: 11 | status_code (int): 整数变量，响应状态码 12 | text_body (str): 字符串变量，响应的body 13 | req_id (str): 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers 14 | x_log (str): 字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers 15 | error (str): 字符串变量，响应的错误内容 16 | """ 17 | 18 | def __init__(self, response, exception=None): 19 | """用响应包和异常信息初始化ResponseInfo类""" 20 | self.__response = response 21 | self.exception = exception 22 | if response is None: 23 | self.url = None 24 | self.status_code = -1 25 | self.text_body = None 26 | self.req_id = None 27 | self.x_log = None 28 | self.error = str(exception) 29 | else: 30 | self.url = response.url 31 | self.status_code = response.status_code 32 | self.text_body = response.text 33 | self.req_id = response.headers.get('X-Reqid') 34 | self.x_log = response.headers.get('X-Log') 35 | if self.status_code >= 400: 36 | if self.__check_json(response): 37 | ret = response.json() if response.text != '' else None 38 | if ret is None: 39 | self.error = 'unknown' 40 | else: 41 | self.error = response.text 42 | else: 43 | self.error = response.text 44 | if self.req_id is None and self.status_code == 200: 45 | self.error = 'server is not qiniu' 46 | 47 | def ok(self): 48 | return self.status_code // 100 == 2 49 | 50 | def need_retry(self): 51 | if 100 <= self.status_code < 500: 52 | return False 53 | if all([ 54 | self.status_code < 0, 55 | self.exception is not None, 56 | 'BadStatusLine' in str(self.exception) 57 | ]): 58 | return False 59 | # https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code 60 | # https://developer.qiniu.com/kodo/3928/error-responses 61 | if self.status_code in [ 62 | 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 63 | ]: 64 | return False 65 | return True 66 | 67 | def connect_failed(self): 68 | return self.__response is None or self.req_id is None 69 | 70 | def json(self): 71 | try: 72 | self.__response.encoding = "utf-8" 73 | return self.__response.json() 74 | except Exception: 75 | return {} 76 | 77 | def __str__(self): 78 | if is_py2: 79 | return ', '.join( 80 | ['%s:%s' % item for item in self.__dict__.items()]).encode('utf-8') 81 | elif is_py3: 82 | return ', '.join( 83 | ['%s:%s' % item for item in self.__dict__.items()]) 84 | 85 | def __repr__(self): 86 | return self.__str__() 87 | 88 | def __check_json(self, response): 89 | try: 90 | response.json() 91 | return True 92 | except Exception: 93 | return False 94 | -------------------------------------------------------------------------------- /qiniu/http/single_flight.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | 4 | class _FlightLock: 5 | """ 6 | Do not use dataclass which caused the event created only once 7 | """ 8 | def __init__(self): 9 | self.event = threading.Event() 10 | self.result = None 11 | self.error = None 12 | 13 | 14 | class SingleFlight: 15 | def __init__(self): 16 | self._locks = {} 17 | self._lock = threading.Lock() 18 | 19 | def do(self, key, fn, *args, **kwargs): 20 | # here does not use `with` statement 21 | # because need to wait by another object if it exists, 22 | # and reduce the `acquire` times if it not exists 23 | self._lock.acquire() 24 | if key in self._locks: 25 | flight_lock = self._locks[key] 26 | 27 | self._lock.release() 28 | flight_lock.event.wait() 29 | 30 | if flight_lock.error: 31 | raise flight_lock.error 32 | return flight_lock.result 33 | 34 | flight_lock = _FlightLock() 35 | self._locks[key] = flight_lock 36 | self._lock.release() 37 | 38 | try: 39 | flight_lock.result = fn(*args, **kwargs) 40 | except Exception as e: 41 | flight_lock.error = e 42 | finally: 43 | flight_lock.event.set() 44 | 45 | with self._lock: 46 | del self._locks[key] 47 | 48 | if flight_lock.error: 49 | raise flight_lock.error 50 | return flight_lock.result 51 | -------------------------------------------------------------------------------- /qiniu/main.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | 6 | from qiniu import etag 7 | 8 | 9 | def main(): 10 | parser = argparse.ArgumentParser(prog='qiniu') 11 | sub_parsers = parser.add_subparsers() 12 | 13 | parser_etag = sub_parsers.add_parser( 14 | 'etag', 15 | description='calculate the etag of the file', 16 | help='etag [file...]') 17 | parser_etag.add_argument( 18 | 'etag_files', 19 | metavar='N', 20 | nargs='+', 21 | help='the file list for calculate') 22 | 23 | args = parser.parse_args() 24 | 25 | try: 26 | etag_files = args.etag_files 27 | 28 | except AttributeError: 29 | etag_files = None 30 | 31 | if etag_files: 32 | r = [etag(file) for file in etag_files] 33 | if len(r) == 1: 34 | print(r[0]) 35 | else: 36 | print(' '.join(r)) 37 | 38 | 39 | if __name__ == '__main__': 40 | main() 41 | -------------------------------------------------------------------------------- /qiniu/retry/__init__.py: -------------------------------------------------------------------------------- 1 | from .attempt import Attempt 2 | from .retrier import Retrier 3 | 4 | __all__ = [ 5 | 'Attempt', 6 | 'Retrier' 7 | ] 8 | -------------------------------------------------------------------------------- /qiniu/retry/abc/__init__.py: -------------------------------------------------------------------------------- 1 | from .policy import RetryPolicy 2 | 3 | __all__ = [ 4 | 'RetryPolicy' 5 | ] 6 | -------------------------------------------------------------------------------- /qiniu/retry/abc/policy.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class RetryPolicy(object): 5 | __metaclass__ = abc.ABCMeta 6 | 7 | @abc.abstractmethod 8 | def init_context(self, context): 9 | """ 10 | initial context values the policy required 11 | 12 | Parameters 13 | ---------- 14 | context: dict 15 | """ 16 | 17 | @abc.abstractmethod 18 | def should_retry(self, attempt): 19 | """ 20 | if returns True, this policy will be applied 21 | 22 | Parameters 23 | ---------- 24 | attempt: qiniu.retry.attempt.Attempt 25 | 26 | Returns 27 | ------- 28 | bool 29 | """ 30 | 31 | @abc.abstractmethod 32 | def prepare_retry(self, attempt): 33 | """ 34 | apply this policy to change the context values for next attempt 35 | 36 | Parameters 37 | ---------- 38 | attempt: qiniu.retry.attempt.Attempt 39 | """ 40 | 41 | def is_important(self, attempt): 42 | """ 43 | if returns True, this policy will be applied, whether it should retry or not. 44 | this is useful when want to stop retry. 45 | 46 | Parameters 47 | ---------- 48 | attempt: qiniu.retry.attempt.Attempt 49 | 50 | Returns 51 | ------- 52 | bool 53 | """ 54 | 55 | def after_retry(self, attempt, policy): 56 | """ 57 | Parameters 58 | ---------- 59 | attempt: qiniu.retry.attempt.Attempt 60 | policy: RetryPolicy 61 | """ 62 | -------------------------------------------------------------------------------- /qiniu/retry/attempt.py: -------------------------------------------------------------------------------- 1 | class Attempt: 2 | def __init__(self, custom_context=None): 3 | """ 4 | Parameters 5 | ---------- 6 | custom_context: dict or None 7 | """ 8 | self.context = custom_context if custom_context is not None else {} 9 | self.exception = None 10 | self.result = None 11 | 12 | def __enter__(self): 13 | pass 14 | 15 | def __exit__(self, exc_type, exc_val, exc_tb): 16 | if exc_type is not None and exc_val is not None: 17 | self.exception = exc_val 18 | return True # Swallow exception. 19 | -------------------------------------------------------------------------------- /qiniu/retry/retrier.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | from .attempt import Attempt 4 | 5 | 6 | def before_retry_nothing(attempt, policy): 7 | return True 8 | 9 | 10 | class Retrier: 11 | def __init__(self, policies=None, before_retry=None): 12 | """ 13 | Parameters 14 | ---------- 15 | policies: list[qiniu.retry.abc.RetryPolicy] 16 | before_retry: callable 17 | `(attempt: Attempt, policy: qiniu.retry.abc.RetryPolicy) -> bool` 18 | """ 19 | self.policies = policies if policies is not None else [] 20 | self.before_retry = before_retry if before_retry is not None else before_retry_nothing 21 | 22 | def __iter__(self): 23 | retrying = Retrying( 24 | # change to `list.copy` for more readable when min version of python update to >= 3 25 | policies=self.policies[:], 26 | before_retry=self.before_retry 27 | ) 28 | retrying.init_context() 29 | while True: 30 | attempt = Attempt(retrying.context) 31 | yield attempt 32 | if ( 33 | hasattr(attempt.exception, 'no_need_retry') and 34 | attempt.exception.no_need_retry 35 | ): 36 | break 37 | policy = retrying.get_retry_policy(attempt) 38 | if not policy: 39 | break 40 | if not self.before_retry(attempt, policy): 41 | break 42 | policy.prepare_retry(attempt) 43 | retrying.after_retried(attempt, policy) 44 | if attempt.exception: 45 | raise attempt.exception 46 | 47 | def try_do( 48 | self, 49 | func, 50 | *args, 51 | **kwargs 52 | ): 53 | attempt = None 54 | for attempt in self: 55 | with attempt: 56 | if kwargs.get('with_retry_context', False): 57 | # inject retry_context 58 | kwargs['retry_context'] = attempt.context 59 | if 'with_retry_context' in kwargs: 60 | del kwargs['with_retry_context'] 61 | 62 | # store result 63 | attempt.result = func(*args, **kwargs) 64 | 65 | if attempt is None: 66 | raise RuntimeError('attempt is none') 67 | 68 | return attempt.result 69 | 70 | def _wrap(self, with_retry_context=False): 71 | def decorator(func): 72 | @functools.wraps(func) 73 | def wrapper(*args, **kwargs): 74 | return self.try_do( 75 | func, 76 | with_retry_context=with_retry_context, 77 | *args, 78 | **kwargs 79 | ) 80 | 81 | return wrapper 82 | 83 | return decorator 84 | 85 | def retry(self, *args, **kwargs): 86 | """ 87 | decorator to retry 88 | """ 89 | if len(args) == 1 and callable(args[0]): 90 | return self.retry()(args[0]) 91 | else: 92 | return self._wrap(**kwargs) 93 | 94 | 95 | class Retrying: 96 | def __init__(self, policies, before_retry): 97 | """ 98 | Parameters 99 | ---------- 100 | policies: list[qiniu.retry.abc.RetryPolicy] 101 | before_retry: callable 102 | `(attempt: Attempt, policy: qiniu.retry.abc.RetryPolicy) -> bool` 103 | """ 104 | self.policies = policies 105 | self.before_retry = before_retry 106 | self.context = {} 107 | 108 | def init_context(self): 109 | for policy in self.policies: 110 | policy.init_context(self.context) 111 | 112 | def get_retry_policy(self, attempt): 113 | """ 114 | 115 | Parameters 116 | ---------- 117 | attempt: Attempt 118 | 119 | Returns 120 | ------- 121 | qiniu.retry.abc.RetryPolicy 122 | 123 | """ 124 | policy = None 125 | 126 | # find important policy 127 | for p in self.policies: 128 | if p.is_important(attempt): 129 | policy = p 130 | break 131 | if policy and policy.should_retry(attempt): 132 | return policy 133 | else: 134 | policy = None 135 | 136 | # find retry policy 137 | for p in self.policies: 138 | if p.should_retry(attempt): 139 | policy = p 140 | break 141 | 142 | return policy 143 | 144 | def after_retried(self, attempt, policy): 145 | for p in self.policies: 146 | p.after_retry(attempt, policy) 147 | 148 | 149 | """ 150 | Examples 151 | -------- 152 | retrier = Retrier() 153 | result = None 154 | for attempt in retrier: 155 | with attempt: 156 | endpoint = attempt.context.get('endpoint') 157 | result = upload(endpoint) 158 | attempt.result = result 159 | return result 160 | """ 161 | 162 | """ 163 | Examples 164 | -------- 165 | def foo(): 166 | print('hi') 167 | 168 | retrier = Retrier() 169 | retrier.try_do(foo) 170 | """ 171 | 172 | """ 173 | Examples 174 | -------- 175 | retrier = Retrier() 176 | 177 | 178 | @retrier.retry 179 | def foo(): 180 | print('hi') 181 | 182 | foo() 183 | """ 184 | -------------------------------------------------------------------------------- /qiniu/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/qiniu/services/__init__.py -------------------------------------------------------------------------------- /qiniu/services/cdn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/qiniu/services/cdn/__init__.py -------------------------------------------------------------------------------- /qiniu/services/compute/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/qiniu/services/compute/__init__.py -------------------------------------------------------------------------------- /qiniu/services/compute/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from qiniu import http, QiniuMacAuth 3 | from .config import KIRK_HOST 4 | from .qcos_api import QcosClient 5 | 6 | 7 | class AccountClient(object): 8 | """客户端入口 9 | 10 | 使用账号密钥生成账号客户端，可以进一步： 11 | 1、获取和操作账号数据 12 | 2、获得部署的应用的客户端 13 | 14 | 属性： 15 | auth: 账号管理密钥对，QiniuMacAuth对象 16 | host: API host，在『内网模式』下使用时，auth=None，会自动使用 apiproxy 服务 17 | 18 | 接口： 19 | get_qcos_client(app_uri) 20 | create_qcos_client(app_uri) 21 | get_app_keys(app_uri) 22 | get_valid_app_auth(app_uri) 23 | get_account_info() 24 | get_app_region_products(app_uri) 25 | get_region_products(region) 26 | list_regions() 27 | list_apps() 28 | create_app(args) 29 | delete_app(app_uri) 30 | 31 | """ 32 | 33 | def __init__(self, auth, host=None): 34 | self.auth = auth 35 | self.qcos_clients = {} 36 | if (auth is None): 37 | self.host = KIRK_HOST['APPPROXY'] 38 | else: 39 | self.host = host or KIRK_HOST['APPGLOBAL'] 40 | acc, info = self.get_account_info() 41 | self.uri = acc.get('name') 42 | 43 | def get_qcos_client(self, app_uri): 44 | """获得资源管理客户端 45 | 缓存，但不是线程安全的 46 | """ 47 | 48 | client = self.qcos_clients.get(app_uri) 49 | if (client is None): 50 | client = self.create_qcos_client(app_uri) 51 | self.qcos_clients[app_uri] = client 52 | 53 | return client 54 | 55 | def create_qcos_client(self, app_uri): 56 | """创建资源管理客户端 57 | 58 | """ 59 | 60 | if (self.auth is None): 61 | return QcosClient(None) 62 | 63 | products = self.get_app_region_products(app_uri) 64 | auth = self.get_valid_app_auth(app_uri) 65 | 66 | if products is None or auth is None: 67 | return None 68 | 69 | return QcosClient(auth, products.get('api')) 70 | 71 | def get_app_keys(self, app_uri): 72 | """获得账号下应用的密钥 73 | 74 | 列出指定应用的密钥，仅当访问者对指定应用有管理权限时有效： 75 | 用户对创建的应用有管理权限。 76 | 用户对使用的第三方应用没有管理权限，第三方应用的运维方有管理权限。 77 | 78 | Args: 79 | - app_uri: 应用的完整标识 80 | 81 | Returns: 82 | 返回一个tuple对象，其格式为(, ) 83 | - result 成功返回秘钥列表，失败返回None 84 | - ResponseInfo 请求的Response信息 85 | """ 86 | 87 | url = '{0}/v3/apps/{1}/keys'.format(self.host, app_uri) 88 | return http._get_with_qiniu_mac(url, None, self.auth) 89 | 90 | def get_valid_app_auth(self, app_uri): 91 | """获得账号下可用的应用的密钥 92 | 93 | 列出指定应用的可用密钥 94 | 95 | Args: 96 | - app_uri: 应用的完整标识 97 | 98 | Returns: 99 | 返回一个tuple对象，其格式为(, ) 100 | - result 成功返回可用秘钥列表，失败返回None 101 | - ResponseInfo 请求的Response信息 102 | """ 103 | 104 | ret, retInfo = self.get_app_keys(app_uri) 105 | 106 | if ret is None: 107 | return None 108 | 109 | for k in ret: 110 | if (k.get('state') == 'enabled'): 111 | return QiniuMacAuth(k.get('ak'), k.get('sk')) 112 | 113 | return None 114 | 115 | def get_account_info(self): 116 | """获得当前账号的信息 117 | 118 | 查看当前请求方（请求鉴权使用的 AccessKey 的属主）的账号信息。 119 | 120 | Returns: 121 | 返回一个tuple对象，其格式为(, ) 122 | - result 成功返回用户信息，失败返回None 123 | - ResponseInfo 请求的Response信息 124 | """ 125 | 126 | url = '{0}/v3/info'.format(self.host) 127 | return http._get_with_qiniu_mac(url, None, self.auth) 128 | 129 | def get_app_region_products(self, app_uri): 130 | """获得指定应用所在区域的产品信息 131 | 132 | Args: 133 | - app_uri: 应用的完整标识 134 | 135 | Returns: 136 | 返回产品信息列表，若失败则返回None 137 | """ 138 | apps, retInfo = self.list_apps() 139 | if apps is None: 140 | return None 141 | 142 | for app in apps: 143 | if (app.get('uri') == app_uri): 144 | return self.get_region_products(app.get('region')) 145 | 146 | return 147 | 148 | def get_region_products(self, region): 149 | """获得指定区域的产品信息 150 | 151 | Args: 152 | - region: 区域，如："nq" 153 | 154 | Returns: 155 | 返回该区域的产品信息，若失败则返回None 156 | """ 157 | 158 | regions, retInfo = self.list_regions() 159 | if regions is None: 160 | return None 161 | 162 | for r in regions: 163 | if r.get('name') == region: 164 | return r.get('products') 165 | 166 | def list_regions(self): 167 | """获得账号可见的区域的信息 168 | 169 | 列出当前用户所有可使用的区域。 170 | 171 | Returns: 172 | 返回一个tuple对象，其格式为(, ) 173 | - result 成功返回区域列表，失败返回None 174 | - ResponseInfo 请求的Response信息 175 | """ 176 | 177 | url = '{0}/v3/regions'.format(self.host) 178 | return http._get_with_qiniu_mac(url, None, self.auth) 179 | 180 | def list_apps(self): 181 | """获得当前账号的应用列表 182 | 183 | 列出所属应用为当前请求方的应用列表。 184 | 185 | Returns: 186 | 返回一个tuple对象，其格式为(, ) 187 | - result 成功返回应用列表，失败返回None 188 | - ResponseInfo 请求的Response信息 189 | """ 190 | 191 | url = '{0}/v3/apps'.format(self.host) 192 | return http._get_with_qiniu_mac(url, None, self.auth) 193 | 194 | def create_app(self, args): 195 | """创建应用 196 | 197 | 在指定区域创建一个新应用，所属应用为当前请求方。 198 | 199 | Args: 200 | - args: 请求参数(json)，参考 http://kirk-docs.qiniu.com/apidocs/ 201 | 202 | Returns: 203 | - result 成功返回所创建的应用信息，若失败则返回None 204 | - ResponseInfo 请求的Response信息 205 | """ 206 | 207 | url = '{0}/v3/apps'.format(self.host) 208 | return http._post_with_qiniu_mac(url, args, self.auth) 209 | 210 | def delete_app(self, app_uri): 211 | """删除应用 212 | 213 | 删除指定标识的应用，当前请求方对该应用应有删除权限。 214 | 215 | Args: 216 | - app_uri: 应用的完整标识 217 | 218 | Returns: 219 | - result 成功返回空dict{}，若失败则返回None 220 | - ResponseInfo 请求的Response信息 221 | """ 222 | 223 | url = '{0}/v3/apps/{1}'.format(self.host, app_uri) 224 | return http._delete_with_qiniu_mac(url, None, self.auth) 225 | -------------------------------------------------------------------------------- /qiniu/services/compute/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | KIRK_HOST = { 4 | 'APPGLOBAL': "https://app-api.qiniu.com", # 公有云 APP API 5 | 'APPPROXY': "http://app.qcos.qiniu", # 内网 APP API 6 | 'APIPROXY': "http://api.qcos.qiniu", # 内网 API 7 | } 8 | 9 | CONTAINER_UINT_TYPE = { 10 | '1U1G': '单核(CPU)，1GB(内存)', 11 | '1U2G': '单核(CPU)，2GB(内存)', 12 | '1U4G': '单核(CPU)，4GB(内存)', 13 | '1U8G': '单核(CPU)，8GB(内存)', 14 | '2U2G': '双核(CPU)，2GB(内存)', 15 | '2U4G': '双核(CPU)，4GB(内存)', 16 | '2U8G': '双核(CPU)，8GB(内存)', 17 | '2U16G': '双核(CPU)，16GB(内存)', 18 | '4U8G': '四核(CPU)，8GB(内存)', 19 | '4U16G': '四核(CPU)，16GB(内存)', 20 | '8U16G': '八核(CPU)，16GB(内存)', 21 | } 22 | -------------------------------------------------------------------------------- /qiniu/services/pili/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/qiniu/services/pili/__init__.py -------------------------------------------------------------------------------- /qiniu/services/pili/rtc_server_manager.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from qiniu import http, Auth 3 | import json 4 | 5 | 6 | class RtcServer(object): 7 | """ 8 | 直播连麦管理类 9 | 主要涉及了直播连麦管理及操作接口的实现，具体的接口规格可以参考官方文档 https://developer.qiniu.com 10 | Attributes: 11 | auth: 账号管理密钥对，Auth对象 12 | 13 | """ 14 | 15 | def __init__(self, auth): 16 | self.auth = auth 17 | self.host = 'http://rtc.qiniuapi.com' 18 | 19 | def create_app(self, data): 20 | return self.__post(self.host + '/v3/apps', data) 21 | 22 | def get_app(self, app_id=None): 23 | if app_id: 24 | return self.__get(self.host + '/v3/apps/%s' % app_id) 25 | else: 26 | return self.__get(self.host + '/v3/apps') 27 | 28 | def delete_app(self, app_id): 29 | return self.__delete(self.host + '/v3/apps/%s' % app_id) 30 | 31 | def update_app(self, app_id, data): 32 | return self.__post(self.host + '/v3/apps/%s' % app_id, data) 33 | 34 | def list_user(self, app_id, room_name): 35 | return self.__get(self.host + '/v3/apps/%s/rooms/%s/users' % (app_id, room_name)) 36 | 37 | def kick_user(self, app_id, room_name, user_id): 38 | return self.__delete(self.host + '/v3/apps/%s/rooms/%s/users/%s' % (app_id, room_name, user_id)) 39 | 40 | def list_active_rooms(self, app_id, room_name_prefix=None): 41 | if room_name_prefix: 42 | return self.__get(self.host + '/v3/apps/%s/rooms?prefix=%s' % (app_id, room_name_prefix)) 43 | else: 44 | return self.__get(self.host + '/v3/apps/%s/rooms' % app_id) 45 | 46 | def __post(self, url, data=None): 47 | return http._post_with_qiniu_mac(url, data, self.auth) 48 | 49 | def __get(self, url, params=None): 50 | return http._get_with_qiniu_mac(url, params, self.auth) 51 | 52 | def __delete(self, url, params=None): 53 | return http._delete_with_qiniu_mac(url, params, self.auth) 54 | 55 | 56 | def get_room_token(access_key, secret_key, room_access): 57 | auth = Auth(access_key, secret_key) 58 | room_access_str = json.dumps(room_access) 59 | room_token = auth.token_with_data(room_access_str) 60 | return room_token 61 | -------------------------------------------------------------------------------- /qiniu/services/processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/qiniu/services/processing/__init__.py -------------------------------------------------------------------------------- /qiniu/services/processing/cmd.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from qiniu.utils import entry 4 | 5 | 6 | def build_op(cmd, first_arg, **kwargs): 7 | op = [cmd] 8 | if first_arg is not None: 9 | op.append(first_arg) 10 | 11 | for k, v in kwargs.items(): 12 | op.append('{0}/{1}'.format(k, v)) 13 | 14 | return '/'.join(op) 15 | 16 | 17 | def pipe_cmd(*cmds): 18 | return '|'.join(cmds) 19 | 20 | 21 | def op_save(op, bucket, key): 22 | return pipe_cmd(op, 'saveas/' + entry(bucket, key)) 23 | -------------------------------------------------------------------------------- /qiniu/services/processing/pfop.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from qiniu import config 4 | from qiniu import http 5 | 6 | 7 | class PersistentFop(object): 8 | """持久化处理类 9 | 10 | 该类用于主动触发异步持久化操作，具体规格参考： 11 | https://developer.qiniu.com/dora/api/persistent-data-processing-pfop 12 | 13 | Attributes: 14 | auth: 账号管理密钥对，Auth对象 15 | bucket: 操作资源所在空间 16 | pipeline: 多媒体处理队列，详见 https://developer.qiniu.com/dora/6499/tasks-and-workflows 17 | notify_url: 持久化处理结果通知URL 18 | """ 19 | 20 | def __init__(self, auth, bucket, pipeline=None, notify_url=None): 21 | """初始化持久化处理类""" 22 | self.auth = auth 23 | self.bucket = bucket 24 | self.pipeline = pipeline 25 | self.notify_url = notify_url 26 | 27 | def execute(self, key, fops=None, force=None, persistent_type=None, workflow_template_id=None): 28 | """ 29 | 执行持久化处理 30 | 31 | Parameters 32 | ---------- 33 | key: str 34 | 待处理的源文件 35 | fops: list[str], optional 36 | 处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop 37 | 与 template_id 二选一 38 | force: int or str, optional 39 | 强制执行持久化处理开关 40 | persistent_type: int or str, optional 41 | 持久化处理类型，为 '1' 时开启闲时任务 42 | template_id: str, optional 43 | 与 fops 二选一 44 | Returns 45 | ------- 46 | ret: dict 47 | 持久化处理的 persistentId，类似 {"persistentId": 5476bedf7823de4068253bae}; 48 | resp: ResponseInfo 49 | """ 50 | if not fops and not workflow_template_id: 51 | raise ValueError('Must provide one of fops or template_id') 52 | data = { 53 | 'bucket': self.bucket, 54 | 'key': key, 55 | } 56 | if self.pipeline: 57 | data['pipeline'] = self.pipeline 58 | if self.notify_url: 59 | data['notifyURL'] = self.notify_url 60 | if fops: 61 | data['fops'] = ';'.join(fops) 62 | if force == 1 or force == '1': 63 | data['force'] = str(force) 64 | if persistent_type and type(int(persistent_type)) is int: 65 | data['type'] = str(persistent_type) 66 | if workflow_template_id: 67 | data['workflowTemplateID'] = workflow_template_id 68 | 69 | url = '{0}/pfop'.format(config.get_default('default_api_host')) 70 | return http._post_with_auth(url, data, self.auth) 71 | 72 | def get_status(self, persistent_id): 73 | """ 74 | 获取持久化处理状态 75 | 76 | Parameters 77 | ---------- 78 | persistent_id: str 79 | 80 | Returns 81 | ------- 82 | ret: dict 83 | 持久化处理的状态，详见 https://developer.qiniu.com/dora/1294/persistent-processing-status-query-prefop 84 | resp: ResponseInfo 85 | """ 86 | url = '{0}/status/get/prefop'.format(config.get_default('default_api_host')) 87 | data = { 88 | 'id': persistent_id 89 | } 90 | return http._get_with_auth(url, data, self.auth) 91 | -------------------------------------------------------------------------------- /qiniu/services/sms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/qiniu/services/sms/__init__.py -------------------------------------------------------------------------------- /qiniu/services/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/qiniu/services/storage/__init__.py -------------------------------------------------------------------------------- /qiniu/services/storage/_bucket_default_retrier.py: -------------------------------------------------------------------------------- 1 | from qiniu.http.endpoints_retry_policy import EndpointsRetryPolicy 2 | from qiniu.http.regions_retry_policy import RegionsRetryPolicy 3 | from qiniu.retry import Retrier 4 | 5 | 6 | def get_default_retrier( 7 | regions_provider, 8 | service_names, 9 | preferred_endpoints_provider=None, 10 | ): 11 | if not service_names: 12 | raise ValueError('service_names should not be empty') 13 | 14 | retry_policies = [ 15 | EndpointsRetryPolicy( 16 | skip_init_context=True 17 | ), 18 | RegionsRetryPolicy( 19 | regions_provider=regions_provider, 20 | service_names=service_names, 21 | preferred_endpoints_provider=preferred_endpoints_provider 22 | ) 23 | ] 24 | 25 | return Retrier(retry_policies) 26 | -------------------------------------------------------------------------------- /qiniu/services/storage/upload_progress_recorder.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import hashlib 3 | import json 4 | import os 5 | import tempfile 6 | from qiniu.compat import is_py2 7 | 8 | 9 | class UploadProgressRecorder(object): 10 | """ 11 | 持久化上传记录类 12 | 13 | 该类默认保存每个文件的上传记录到文件系统中，用于断点续传 14 | 上传记录为json格式 15 | 16 | Attributes: 17 | record_folder: 保存上传记录的目录 18 | """ 19 | 20 | def __init__(self, record_folder=tempfile.gettempdir()): 21 | self.record_folder = record_folder 22 | 23 | def __get_upload_record_file_path(self, file_name, key): 24 | record_key = '{0}/{1}'.format(key, file_name) 25 | if is_py2: 26 | record_file_name = hashlib.md5(record_key).hexdigest() 27 | else: 28 | record_file_name = hashlib.md5(record_key.encode('utf-8')).hexdigest() 29 | return os.path.join(self.record_folder, record_file_name) 30 | 31 | def has_upload_record(self, file_name, key): 32 | upload_record_file_path = self.__get_upload_record_file_path(file_name, key) 33 | return os.path.isfile(upload_record_file_path) 34 | 35 | def get_upload_record(self, file_name, key): 36 | upload_record_file_path = self.__get_upload_record_file_path(file_name, key) 37 | if not self.has_upload_record(file_name, key): 38 | return None 39 | try: 40 | with open(upload_record_file_path, 'r') as f: 41 | json_data = json.load(f) 42 | except (IOError, ValueError): 43 | json_data = None 44 | 45 | return json_data 46 | 47 | def set_upload_record(self, file_name, key, data): 48 | upload_record_file_path = self.__get_upload_record_file_path(file_name, key) 49 | with open(upload_record_file_path, 'w') as f: 50 | json.dump(data, f) 51 | 52 | def delete_upload_record(self, file_name, key): 53 | upload_record_file_path = self.__get_upload_record_file_path(file_name, key) 54 | try: 55 | os.remove(upload_record_file_path) 56 | except OSError: 57 | pass 58 | -------------------------------------------------------------------------------- /qiniu/services/storage/uploaders/__init__.py: -------------------------------------------------------------------------------- 1 | from .form_uploader import FormUploader 2 | from .resume_uploader_v1 import ResumeUploaderV1 3 | from .resume_uploader_v2 import ResumeUploaderV2 4 | 5 | __all__ = [ 6 | 'FormUploader', 7 | 'ResumeUploaderV1', 8 | 'ResumeUploaderV2' 9 | ] 10 | -------------------------------------------------------------------------------- /qiniu/services/storage/uploaders/_default_retrier.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | from qiniu.http.endpoints_retry_policy import EndpointsRetryPolicy 4 | from qiniu.http.region import ServiceName 5 | from qiniu.http.regions_retry_policy import RegionsRetryPolicy 6 | from qiniu.retry.abc import RetryPolicy 7 | from qiniu.retry import Retrier 8 | 9 | 10 | _TokenExpiredRetryState = namedtuple( 11 | 'TokenExpiredRetryState', 12 | [ 13 | 'retried_times', 14 | 'upload_api_version' 15 | ] 16 | ) 17 | 18 | 19 | class TokenExpiredRetryPolicy(RetryPolicy): 20 | def __init__( 21 | self, 22 | upload_api_version, 23 | record_delete_handler, 24 | record_exists_handler, 25 | max_retry_times=1 26 | ): 27 | """ 28 | Parameters 29 | ---------- 30 | upload_api_version: str 31 | record_delete_handler: callable 32 | `() -> None` 33 | record_exists_handler: callable 34 | `() -> bool` 35 | max_retry_times: int 36 | """ 37 | self.upload_api_version = upload_api_version 38 | self.record_delete_handler = record_delete_handler 39 | self.record_exists_handler = record_exists_handler 40 | self.max_retry_times = max_retry_times 41 | 42 | def init_context(self, context): 43 | """ 44 | Parameters 45 | ---------- 46 | context: dict 47 | """ 48 | context[self] = _TokenExpiredRetryState( 49 | retried_times=0, 50 | upload_api_version=self.upload_api_version 51 | ) 52 | 53 | def should_retry(self, attempt): 54 | """ 55 | Parameters 56 | ---------- 57 | attempt: qiniu.retry.Attempt 58 | 59 | Returns 60 | ------- 61 | bool 62 | """ 63 | state = attempt.context[self] 64 | 65 | if ( 66 | state.retried_times >= self.max_retry_times or 67 | not self.record_exists_handler() 68 | ): 69 | return False 70 | 71 | if not attempt.result: 72 | return False 73 | 74 | _ret, resp = attempt.result 75 | 76 | if ( 77 | state.upload_api_version == 'v1' and 78 | resp.status_code == 701 79 | ): 80 | return True 81 | 82 | if ( 83 | state.upload_api_version == 'v2' and 84 | resp.status_code == 612 85 | ): 86 | return True 87 | 88 | return False 89 | 90 | def prepare_retry(self, attempt): 91 | """ 92 | Parameters 93 | ---------- 94 | attempt: qiniu.retry.Attempt 95 | """ 96 | state = attempt.context[self] 97 | attempt.context[self] = state._replace(retried_times=state.retried_times + 1) 98 | 99 | if not self.record_exists_handler(): 100 | return 101 | 102 | self.record_delete_handler() 103 | 104 | 105 | class AccUnavailableRetryPolicy(RetryPolicy): 106 | def __init__(self): 107 | pass 108 | 109 | def init_context(self, context): 110 | pass 111 | 112 | def should_retry(self, attempt): 113 | """ 114 | Parameters 115 | ---------- 116 | attempt: qiniu.retry.Attempt 117 | 118 | Returns 119 | ------- 120 | bool 121 | """ 122 | if not attempt.result: 123 | return False 124 | 125 | region = attempt.context.get('region') 126 | if not region: 127 | return False 128 | 129 | if all( 130 | not region.services[sn] 131 | for sn in attempt.context.get('alternative_service_names') 132 | ): 133 | return False 134 | 135 | _ret, resp = attempt.result 136 | 137 | return resp.status_code == 400 and \ 138 | 'transfer acceleration is not configured on this bucket' in resp.text_body 139 | 140 | def prepare_retry(self, attempt): 141 | """ 142 | Parameters 143 | ---------- 144 | attempt: qiniu.retry.Attempt 145 | """ 146 | endpoints = [] 147 | while not endpoints: 148 | if not attempt.context.get('alternative_service_names'): 149 | raise RuntimeError('No alternative service available') 150 | attempt.context['service_name'] = attempt.context.get('alternative_service_names').pop(0) 151 | # shallow copy list 152 | # change to `list.copy` for more readable when min version of python update to >= 3 153 | endpoints = attempt.context['region'].services.get(attempt.context['service_name'], [])[:] 154 | attempt.context['alternative_endpoints'] = endpoints 155 | attempt.context['endpoint'] = attempt.context['alternative_endpoints'].pop(0) 156 | 157 | 158 | ProgressRecord = namedtuple( 159 | 'ProgressRecorder', 160 | [ 161 | 'upload_api_version', 162 | 'exists', 163 | 'delete' 164 | ] 165 | ) 166 | 167 | 168 | def get_default_retrier( 169 | regions_provider, 170 | preferred_endpoints_provider=None, 171 | progress_record=None, 172 | accelerate_uploading=False 173 | ): 174 | """ 175 | Parameters 176 | ---------- 177 | regions_provider: Iterable[Region] 178 | preferred_endpoints_provider: Iterable[Endpoint] 179 | progress_record: ProgressRecord 180 | accelerate_uploading: bool 181 | 182 | Returns 183 | ------- 184 | Retrier 185 | """ 186 | retry_policies = [] 187 | upload_service_names = [ServiceName.UP] 188 | handle_change_region = None 189 | 190 | if accelerate_uploading: 191 | retry_policies.append(AccUnavailableRetryPolicy()) 192 | upload_service_names.insert(0, ServiceName.UP_ACC) 193 | 194 | if progress_record: 195 | retry_policies.append(TokenExpiredRetryPolicy( 196 | upload_api_version=progress_record.upload_api_version, 197 | record_delete_handler=progress_record.delete, 198 | record_exists_handler=progress_record.exists 199 | )) 200 | 201 | def _handle_change_region(_): 202 | progress_record.delete() 203 | 204 | handle_change_region = _handle_change_region 205 | 206 | retry_policies += [ 207 | EndpointsRetryPolicy(skip_init_context=True), 208 | RegionsRetryPolicy( 209 | regions_provider=regions_provider, 210 | service_names=upload_service_names, 211 | preferred_endpoints_provider=preferred_endpoints_provider, 212 | on_change_region=handle_change_region 213 | ) 214 | ] 215 | 216 | return Retrier(retry_policies) 217 | -------------------------------------------------------------------------------- /qiniu/services/storage/uploaders/abc/__init__.py: -------------------------------------------------------------------------------- 1 | from .uploader_base import UploaderBase 2 | from .resume_uploader_base import ResumeUploaderBase 3 | 4 | __all__ = [ 5 | 'UploaderBase', 6 | 'ResumeUploaderBase' 7 | ] 8 | -------------------------------------------------------------------------------- /qiniu/services/storage/uploaders/abc/resume_uploader_base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from concurrent import futures 3 | 4 | from qiniu.services.storage.uploaders.io_chunked import ChunkInfo 5 | from qiniu.services.storage.uploaders.abc import UploaderBase 6 | 7 | 8 | class ResumeUploaderBase(UploaderBase): 9 | """ 10 | Attributes 11 | ---------- 12 | part_size: int, optional 13 | progress_handler: function, optional 14 | upload_progress_recorder: UploadProgressRecorder, optional 15 | concurrent_executor: futures.Executor, optional 16 | """ 17 | __metaclass__ = abc.ABCMeta 18 | 19 | def __init__( 20 | self, 21 | bucket_name, 22 | **kwargs 23 | ): 24 | """ 25 | Parameters 26 | ---------- 27 | bucket_name 28 | part_size: int 29 | progress_handler: function 30 | upload_progress_recorder: UploadProgressRecorder 31 | max_concurrent_workers: int 32 | concurrent_executor: futures.Executor 33 | kwargs 34 | """ 35 | super(ResumeUploaderBase, self).__init__(bucket_name, **kwargs) 36 | 37 | self.part_size = kwargs.get('part_size', 4 * (1024 ** 2)) 38 | 39 | self.progress_handler = kwargs.get( 40 | 'progress_handler', 41 | None 42 | ) 43 | 44 | self.upload_progress_recorder = kwargs.get( 45 | 'upload_progress_recorder', 46 | None 47 | ) 48 | 49 | max_workers = kwargs.get('max_concurrent_workers', 3) 50 | self.concurrent_executor = kwargs.get( 51 | 'concurrent_executor', 52 | futures.ThreadPoolExecutor(max_workers=max_workers) 53 | ) 54 | 55 | def gen_chunk_list(self, size, chunk_size=None, uploaded_chunk_no_list=None): 56 | """ 57 | Parameters 58 | ---------- 59 | size: int 60 | chunk_size: int 61 | uploaded_chunk_no_list: list[int] 62 | 63 | Yields 64 | ------- 65 | ChunkInfo 66 | """ 67 | if not chunk_size: 68 | chunk_size = self.part_size 69 | if not uploaded_chunk_no_list: 70 | uploaded_chunk_no_list = [] 71 | 72 | for i, chunk_offset in enumerate(range(0, size, chunk_size)): 73 | chunk_no = i + 1 74 | if chunk_no in uploaded_chunk_no_list: 75 | continue 76 | yield ChunkInfo( 77 | chunk_no=chunk_no, 78 | chunk_offset=chunk_offset, 79 | chunk_size=min( 80 | chunk_size, 81 | size - chunk_offset 82 | ) 83 | ) 84 | 85 | @abc.abstractmethod 86 | def _recover_from_record( 87 | self, 88 | file_name, 89 | key, 90 | context 91 | ): 92 | """ 93 | Parameters 94 | ---------- 95 | file_name: str 96 | key: str 97 | context: any 98 | 99 | Returns 100 | ------- 101 | any 102 | """ 103 | 104 | @abc.abstractmethod 105 | def _set_to_record( 106 | self, 107 | file_name, 108 | key, 109 | context 110 | ): 111 | """ 112 | Parameters 113 | ---------- 114 | file_name: str 115 | key: str 116 | context: any 117 | """ 118 | 119 | @abc.abstractmethod 120 | def _progress_handler( 121 | self, 122 | file_name, 123 | key, 124 | context, 125 | uploaded_size, 126 | total_size 127 | ): 128 | """ 129 | Parameters 130 | ---------- 131 | file_name: str 132 | key: str 133 | context: any 134 | uploaded_size: int 135 | total_size: int 136 | """ 137 | 138 | @abc.abstractmethod 139 | def initial_parts( 140 | self, 141 | up_token, 142 | key, 143 | file_path, 144 | data, 145 | data_size, 146 | modify_time, 147 | part_size, 148 | file_name, 149 | **kwargs 150 | ): 151 | """ 152 | Parameters 153 | ---------- 154 | up_token: str 155 | key: str 156 | file_path: str 157 | data: IOBase 158 | data_size: int 159 | modify_time: int 160 | part_size: int 161 | file_name: str 162 | kwargs: dict 163 | 164 | Returns 165 | ------- 166 | ret: dict 167 | resp: ResponseInfo 168 | """ 169 | 170 | @abc.abstractmethod 171 | def upload_parts( 172 | self, 173 | up_token, 174 | data, 175 | data_size, 176 | context, 177 | **kwargs 178 | ): 179 | """ 180 | Parameters 181 | ---------- 182 | up_token: str 183 | data: IOBase 184 | data_size: int 185 | context: any 186 | kwargs: dict 187 | 188 | Returns 189 | ------- 190 | ret: dict 191 | resp: ResponseInfo 192 | """ 193 | 194 | @abc.abstractmethod 195 | def complete_parts( 196 | self, 197 | up_token, 198 | data_size, 199 | context, 200 | **kwargs 201 | ): 202 | """ 203 | Parameters 204 | ---------- 205 | up_token: str 206 | data_size: int 207 | context: any 208 | kwargs: dict 209 | 210 | Returns 211 | ------- 212 | ret: dict 213 | resp: ResponseInfo 214 | """ 215 | -------------------------------------------------------------------------------- /qiniu/services/storage/uploaders/io_chunked.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | from collections import namedtuple 4 | 5 | from qiniu.compat import is_seekable 6 | 7 | 8 | ChunkInfo = namedtuple( 9 | 'ChunkInfo', 10 | [ 11 | 'chunk_no', 12 | 'chunk_offset', 13 | 'chunk_size' 14 | ] 15 | ) 16 | 17 | 18 | class IOChunked(io.IOBase): 19 | def __init__( 20 | self, 21 | base_io, 22 | chunk_offset, 23 | chunk_size, 24 | lock, 25 | buffer_size=4 * (1024 ** 2) # 4MB just for demo 26 | ): 27 | if not is_seekable(base_io): 28 | raise TypeError('"base_io" must be seekable') 29 | self.__base_io = base_io 30 | self.__chunk_start = chunk_offset 31 | self.__chunk_size = chunk_size 32 | self.__chunk_end = chunk_offset + chunk_size 33 | self.__lock = lock 34 | self.__chunk_pos = 0 35 | 36 | self.buffer_size = min(buffer_size, chunk_size) 37 | 38 | def readable(self): 39 | return self.__base_io.readable() 40 | 41 | def seekable(self): 42 | return True 43 | 44 | def seek(self, offset, whence=0): 45 | if not self.seekable(): 46 | raise io.UnsupportedOperation('does not support seek') 47 | if whence == os.SEEK_SET: 48 | if offset < 0: 49 | raise ValueError('offset should be zero or positive if whence is 0') 50 | self.__chunk_pos = offset 51 | elif whence == os.SEEK_CUR: 52 | self.__chunk_pos += offset 53 | elif whence == os.SEEK_END: 54 | if offset > 0: 55 | raise ValueError('offset should be zero or negative if whence is 2') 56 | self.__chunk_pos = self.__chunk_size + offset 57 | else: 58 | raise ValueError('whence should be 0, 1 or 2') 59 | self.__chunk_pos = max( 60 | 0, 61 | min(self.__chunk_size, self.__chunk_pos) 62 | ) 63 | 64 | def tell(self): 65 | return self.__chunk_pos 66 | 67 | def read(self, size): 68 | if self.__curr_base_pos >= self.__chunk_end: 69 | return b'' 70 | read_size = max(self.buffer_size, size) 71 | read_size = min(self.__rest_chunk_size, read_size) 72 | 73 | # -- ignore size argument -- 74 | with self.__lock: 75 | self.__base_io.seek(self.__curr_base_pos) 76 | data = self.__base_io.read(read_size) 77 | 78 | self.__chunk_pos += len(data) 79 | return data 80 | 81 | def __len__(self): 82 | return self.__chunk_size 83 | 84 | @property 85 | def __curr_base_pos(self): 86 | return self.__chunk_start + self.__chunk_pos 87 | 88 | @property 89 | def __rest_chunk_size(self): 90 | return self.__chunk_end - self.__curr_base_pos 91 | -------------------------------------------------------------------------------- /qiniu/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from hashlib import sha1, new as hashlib_new 3 | from base64 import urlsafe_b64encode, urlsafe_b64decode 4 | from datetime import datetime, tzinfo, timedelta 5 | 6 | from .compat import b, s 7 | 8 | try: 9 | import zlib 10 | 11 | binascii = zlib 12 | except ImportError: 13 | zlib = None 14 | import binascii 15 | 16 | _BLOCK_SIZE = 1024 * 1024 * 4 17 | 18 | 19 | def urlsafe_base64_encode(data): 20 | """urlsafe的base64编码: 21 | 22 | 对提供的数据进行urlsafe的base64编码。规格参考： 23 | https://developer.qiniu.com/kodo/manual/1231/appendix#1 24 | 25 | Args: 26 | data: 待编码的数据，一般为字符串 27 | 28 | Returns: 29 | 编码后的字符串 30 | """ 31 | ret = urlsafe_b64encode(b(data)) 32 | return s(ret) 33 | 34 | 35 | def urlsafe_base64_decode(data): 36 | """urlsafe的base64解码: 37 | 38 | 对提供的urlsafe的base64编码的数据进行解码 39 | 40 | Args: 41 | data: 待解码的数据，一般为字符串 42 | 43 | Returns: 44 | 解码后的字符串。 45 | """ 46 | ret = urlsafe_b64decode(s(data)) 47 | return ret 48 | 49 | 50 | def file_crc32(filePath): 51 | """计算文件的crc32检验码: 52 | 53 | Args: 54 | filePath: 待计算校验码的文件路径 55 | 56 | Returns: 57 | 文件内容的crc32校验码。 58 | """ 59 | crc = 0 60 | with open(filePath, 'rb') as f: 61 | for block in _file_iter(f, _BLOCK_SIZE): 62 | crc = binascii.crc32(block, crc) & 0xFFFFFFFF 63 | return crc 64 | 65 | 66 | def io_crc32(io_data): 67 | result = 0 68 | for d in io_data: 69 | result = binascii.crc32(d, result) & 0xFFFFFFFF 70 | return result 71 | 72 | 73 | def io_md5(io_data): 74 | h = hashlib_new('md5') 75 | for d in io_data: 76 | h.update(d) 77 | return h.hexdigest() 78 | 79 | 80 | def crc32(data): 81 | """计算输入流的crc32检验码: 82 | 83 | Args: 84 | data: 待计算校验码的字符流 85 | 86 | Returns: 87 | 输入流的crc32校验码。 88 | """ 89 | return binascii.crc32(b(data)) & 0xffffffff 90 | 91 | 92 | def _file_iter(input_stream, size, offset=0): 93 | """读取输入流: 94 | 95 | Args: 96 | input_stream: 待读取文件的二进制流 97 | size: 二进制流的大小 98 | 99 | Raises: 100 | IOError: 文件流读取失败 101 | """ 102 | input_stream.seek(offset) 103 | d = input_stream.read(size) 104 | while d: 105 | yield d 106 | d = input_stream.read(size) 107 | input_stream.seek(0) 108 | 109 | 110 | def _sha1(data): 111 | """单块计算hash: 112 | 113 | Args: 114 | data: 待计算hash的数据 115 | 116 | Returns: 117 | 输入数据计算的hash值 118 | """ 119 | h = sha1() 120 | h.update(data) 121 | return h.digest() 122 | 123 | 124 | def etag_stream(input_stream): 125 | """ 126 | 计算输入流的etag 127 | 128 | .. deprecated:: 129 | 在 v2 分片上传使用 4MB 以外分片大小时无法正常工作 130 | 131 | Parameters 132 | ---------- 133 | input_stream: io.IOBase 134 | 支持随机访问的文件型对象 135 | 136 | Returns 137 | ------- 138 | str 139 | 140 | """ 141 | array = [_sha1(block) for block in _file_iter(input_stream, _BLOCK_SIZE)] 142 | if len(array) == 0: 143 | array = [_sha1(b'')] 144 | if len(array) == 1: 145 | data = array[0] 146 | prefix = b'\x16' 147 | else: 148 | sha1_str = b('').join(array) 149 | data = _sha1(sha1_str) 150 | prefix = b'\x96' 151 | return urlsafe_base64_encode(prefix + data) 152 | 153 | 154 | def etag(filePath): 155 | """ 156 | 计算文件的etag: 157 | 158 | .. deprecated:: 159 | 在 v2 分片上传使用 4MB 以外分片大小时无法正常工作 160 | 161 | 162 | Parameters 163 | ---------- 164 | filePath: str 165 | 待计算 etag 的文件路径 166 | 167 | Returns 168 | ------- 169 | str 170 | 输入文件的etag值 171 | """ 172 | with open(filePath, 'rb') as f: 173 | return etag_stream(f) 174 | 175 | 176 | def entry(bucket, key): 177 | """计算七牛API中的数据格式: 178 | 179 | entry规格参考 https://developer.qiniu.com/kodo/api/1276/data-format 180 | 181 | Args: 182 | bucket: 待操作的空间名 183 | key: 待操作的文件名 184 | 185 | Returns: 186 | 符合七牛API规格的数据格式 187 | """ 188 | if key is None: 189 | return urlsafe_base64_encode('{0}'.format(bucket)) 190 | else: 191 | return urlsafe_base64_encode('{0}:{1}'.format(bucket, key)) 192 | 193 | 194 | def decode_entry(e): 195 | return (s(urlsafe_base64_decode(e)).split(':') + [None] * 2)[:2] 196 | 197 | 198 | def rfc_from_timestamp(timestamp): 199 | """将时间戳转换为HTTP RFC格式 200 | 201 | Args: 202 | timestamp: 整型Unix时间戳（单位秒） 203 | """ 204 | last_modified_date = datetime.utcfromtimestamp(timestamp) 205 | last_modified_str = last_modified_date.strftime( 206 | '%a, %d %b %Y %H:%M:%S GMT') 207 | return last_modified_str 208 | 209 | 210 | def _valid_header_key_char(ch): 211 | is_token_table = [ 212 | "!", "#", "$", "%", "&", "\\", "*", "+", "-", ".", 213 | "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", 214 | "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", 215 | "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", 216 | "U", "W", "V", "X", "Y", "Z", 217 | "^", "_", "`", 218 | "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", 219 | "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", 220 | "u", "v", "w", "x", "y", "z", 221 | "|", "~"] 222 | return 0 <= ord(ch) < 128 and ch in is_token_table 223 | 224 | 225 | def canonical_mime_header_key(field_name): 226 | for ch in field_name: 227 | if not _valid_header_key_char(ch): 228 | return field_name 229 | result = "" 230 | upper = True 231 | for ch in field_name: 232 | if upper and "a" <= ch <= "z": 233 | result += ch.upper() 234 | elif not upper and "A" <= ch <= "Z": 235 | result += ch.lower() 236 | else: 237 | result += ch 238 | upper = ch == "-" 239 | return result 240 | 241 | 242 | class _UTC_TZINFO(tzinfo): 243 | def utcoffset(self, dt): 244 | return timedelta(hours=0) 245 | 246 | def tzname(self, dt): 247 | return "UTC" 248 | 249 | def dst(self, dt): 250 | return timedelta(0) 251 | 252 | 253 | def dt2ts(dt): 254 | """ 255 | converte datetime to timestamp 256 | 257 | Parameters 258 | ---------- 259 | dt: datetime.datetime 260 | """ 261 | if not dt.tzinfo: 262 | st = (dt - datetime(1970, 1, 1)).total_seconds() 263 | else: 264 | st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds() 265 | 266 | return int(st) 267 | -------------------------------------------------------------------------------- /qiniu/zone.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from qiniu.region import LegacyRegion 4 | 5 | 6 | class Zone(LegacyRegion): 7 | pass 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import io 5 | import os 6 | import re 7 | 8 | from setuptools import setup, find_packages 9 | 10 | 11 | def read(*names, **kwargs): 12 | return io.open( 13 | os.path.join(os.path.dirname(__file__), *names), 14 | encoding=kwargs.get("encoding", "utf8") 15 | ).read() 16 | 17 | 18 | def find_version(*file_paths): 19 | version_file = read(*file_paths) 20 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 21 | version_file, re.M) 22 | if version_match: 23 | return version_match.group(1) 24 | raise RuntimeError("Unable to find version string.") 25 | 26 | 27 | setup( 28 | name='qiniu', 29 | version=find_version("qiniu/__init__.py"), 30 | description='Qiniu Resource Storage SDK', 31 | long_description='see:\nhttps://github.com/qiniu/python-sdk\n', 32 | author='Shanghai Qiniu Information Technologies Co., Ltd.', 33 | author_email='sdk@qiniu.com', 34 | maintainer_email='support@qiniu.com', 35 | license='MIT', 36 | url='https://github.com/qiniu/python-sdk', 37 | platforms='any', 38 | packages=find_packages(), 39 | classifiers=[ 40 | 'Intended Audience :: Developers', 41 | 'License :: OSI Approved :: MIT License', 42 | 'Operating System :: OS Independent', 43 | 'Programming Language :: Python', 44 | 'Programming Language :: Python :: 2', 45 | 'Programming Language :: Python :: 2.7', 46 | 'Programming Language :: Python :: 3', 47 | 'Programming Language :: Python :: 3.4', 48 | 'Programming Language :: Python :: 3.5', 49 | 'Programming Language :: Python :: 3.6', 50 | 'Programming Language :: Python :: 3.7', 51 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 52 | 'Topic :: Software Development :: Libraries :: Python Modules' 53 | ], 54 | install_requires=[ 55 | 'requests; python_version >= "3.7"', 56 | 'requests<2.28; python_version < "3.7"', 57 | 'futures; python_version == "2.7"', 58 | 'enum34; python_version == "2.7"' 59 | ], 60 | extras_require={ 61 | 'dev': [ 62 | 'coverage<7.2', 63 | 'flake8', 64 | 'pytest', 65 | 'pytest-cov', 66 | 'freezegun', 67 | ] 68 | }, 69 | 70 | entry_points={ 71 | 'console_scripts': [ 72 | 'qiniupy = qiniu.main:main', 73 | ], 74 | } 75 | ) 76 | -------------------------------------------------------------------------------- /tests/cases/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/tests/cases/__init__.py -------------------------------------------------------------------------------- /tests/cases/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import random 4 | import string 5 | 6 | import pytest 7 | 8 | from qiniu import config as qn_config 9 | from qiniu import Auth 10 | 11 | 12 | @pytest.fixture(scope='session') 13 | def access_key(): 14 | yield os.getenv('QINIU_ACCESS_KEY') 15 | 16 | 17 | @pytest.fixture(scope='session') 18 | def secret_key(): 19 | yield os.getenv('QINIU_SECRET_KEY') 20 | 21 | 22 | @pytest.fixture(scope='session') 23 | def bucket_name(): 24 | yield os.getenv('QINIU_TEST_BUCKET') 25 | 26 | 27 | @pytest.fixture(scope='session') 28 | def no_acc_bucket_name(): 29 | yield os.getenv('QINIU_TEST_NO_ACC_BUCKET') 30 | 31 | 32 | @pytest.fixture(scope='session') 33 | def download_domain(): 34 | yield os.getenv('QINIU_TEST_DOMAIN') 35 | 36 | 37 | @pytest.fixture(scope='session') 38 | def upload_callback_url(): 39 | yield os.getenv('QINIU_UPLOAD_CALLBACK_URL') 40 | 41 | 42 | @pytest.fixture(scope='session') 43 | def qn_auth(access_key, secret_key): 44 | yield Auth(access_key, secret_key) 45 | 46 | 47 | @pytest.fixture(scope='session') 48 | def is_travis(): 49 | """ 50 | migrate from old test cases. 51 | seems useless. 52 | """ 53 | yield os.getenv('QINIU_TEST_ENV') == 'travis' 54 | 55 | 56 | @pytest.fixture(scope='function') 57 | def set_conf_default(request): 58 | if hasattr(request, 'param'): 59 | qn_config.set_default(**request.param) 60 | yield 61 | qn_config._config = { 62 | 'default_zone': None, 63 | 'default_rs_host': qn_config.RS_HOST, 64 | 'default_rsf_host': qn_config.RSF_HOST, 65 | 'default_api_host': qn_config.API_HOST, 66 | 'default_uc_host': qn_config.UC_HOST, 67 | 'default_uc_backup_hosts': qn_config.UC_BACKUP_HOSTS, 68 | 'default_query_region_host': qn_config.QUERY_REGION_HOST, 69 | 'default_query_region_backup_hosts': [ 70 | 'uc.qbox.me', 71 | 'api.qiniu.com' 72 | ], 73 | 'default_backup_hosts_retry_times': 2, 74 | 'connection_timeout': 30, # 链接超时为时间为30s 75 | 'connection_retries': 3, # 链接重试次数为3次 76 | 'connection_pool': 10, # 链接池个数为10 77 | 'default_upload_threshold': 2 * qn_config._BLOCK_SIZE # put_file上传方式的临界默认值 78 | } 79 | 80 | _is_customized_default = { 81 | 'default_zone': False, 82 | 'default_rs_host': False, 83 | 'default_rsf_host': False, 84 | 'default_api_host': False, 85 | 'default_uc_host': False, 86 | 'default_query_region_host': False, 87 | 'default_query_region_backup_hosts': False, 88 | 'default_backup_hosts_retry_times': False, 89 | 'connection_timeout': False, 90 | 'connection_retries': False, 91 | 'connection_pool': False, 92 | 'default_upload_threshold': False 93 | } 94 | 95 | 96 | @pytest.fixture(scope='session') 97 | def rand_string(): 98 | def _rand_string(length): 99 | # use random.choices when min version of python >= 3.6 100 | return ''.join( 101 | random.choice(string.ascii_letters + string.digits) 102 | for _ in range(length) 103 | ) 104 | yield _rand_string 105 | 106 | 107 | class Ref: 108 | """ 109 | python2 not support nonlocal keyword 110 | """ 111 | def __init__(self, value=None): 112 | self.value = value 113 | 114 | 115 | @pytest.fixture(scope='session') 116 | def use_ref(): 117 | def _use_ref(value): 118 | return Ref(value) 119 | 120 | yield _use_ref 121 | -------------------------------------------------------------------------------- /tests/cases/test_http/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/tests/cases/test_http/__init__.py -------------------------------------------------------------------------------- /tests/cases/test_http/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from qiniu.compat import urlparse 6 | 7 | 8 | @pytest.fixture(scope='session') 9 | def mock_server_addr(): 10 | addr = os.getenv('MOCK_SERVER_ADDRESS', 'http://localhost:9000') 11 | yield urlparse(addr) 12 | -------------------------------------------------------------------------------- /tests/cases/test_http/test_endpoint.py: -------------------------------------------------------------------------------- 1 | from qiniu.http.endpoint import Endpoint 2 | 3 | 4 | class TestEndpoint: 5 | def test_endpoint_with_default_scheme(self): 6 | endpoint = Endpoint('uc.python-sdk.qiniu.com') 7 | assert endpoint.get_value() == 'https://uc.python-sdk.qiniu.com' 8 | 9 | def test_endpoint_with_custom_scheme(self): 10 | endpoint = Endpoint('uc.python-sdk.qiniu.com', default_scheme='http') 11 | assert endpoint.get_value() == 'http://uc.python-sdk.qiniu.com' 12 | 13 | def test_endpoint_with_get_value_with_custom_scheme(self): 14 | endpoint = Endpoint('uc.python-sdk.qiniu.com', default_scheme='http') 15 | assert endpoint.get_value('https') == 'https://uc.python-sdk.qiniu.com' 16 | 17 | def test_create_endpoint_from_host_with_scheme(self): 18 | endpoint = Endpoint.from_host('http://uc.python-sdk.qiniu.com') 19 | assert endpoint.default_scheme == 'http' 20 | assert endpoint.get_value() == 'http://uc.python-sdk.qiniu.com' 21 | 22 | def test_clone_endpoint(self): 23 | endpoint = Endpoint('uc.python-sdk.qiniu.com') 24 | another_endpoint = endpoint.clone() 25 | another_endpoint.host = 'another-uc.python-sdk.qiniu.com' 26 | assert endpoint.get_value() == 'https://uc.python-sdk.qiniu.com' 27 | assert another_endpoint.get_value() == 'https://another-uc.python-sdk.qiniu.com' 28 | -------------------------------------------------------------------------------- /tests/cases/test_http/test_endpoints_retry_policy.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from qiniu.http.endpoint import Endpoint 4 | from qiniu.http.endpoints_retry_policy import EndpointsRetryPolicy 5 | from qiniu.retry.attempt import Attempt 6 | 7 | 8 | @pytest.fixture(scope='function') 9 | def mocked_endpoints_provider(): 10 | yield [ 11 | Endpoint('a'), 12 | Endpoint('b'), 13 | Endpoint('c') 14 | ] 15 | 16 | 17 | class TestEndpointsRetryPolicy: 18 | def test_init_context(self, mocked_endpoints_provider): 19 | endpoints_retry_policy = EndpointsRetryPolicy( 20 | endpoints_provider=mocked_endpoints_provider 21 | ) 22 | 23 | mocked_context = {} 24 | endpoints_retry_policy.init_context(mocked_context) 25 | 26 | assert mocked_context['endpoint'].get_value() == mocked_endpoints_provider[0].get_value() 27 | assert [ 28 | e.get_value() 29 | for e in mocked_context['alternative_endpoints'] 30 | ] == [ 31 | e.get_value() 32 | for e in mocked_endpoints_provider[1:] 33 | ] 34 | 35 | def test_should_retry(self, mocked_endpoints_provider): 36 | mocked_attempt = Attempt() 37 | 38 | endpoints_retry_policy = EndpointsRetryPolicy( 39 | endpoints_provider=mocked_endpoints_provider 40 | ) 41 | endpoints_retry_policy.init_context(mocked_attempt.context) 42 | assert endpoints_retry_policy.should_retry(mocked_attempt) 43 | 44 | def test_prepare_retry(self, mocked_endpoints_provider): 45 | mocked_attempt = Attempt() 46 | 47 | endpoints_retry_policy = EndpointsRetryPolicy( 48 | endpoints_provider=mocked_endpoints_provider 49 | ) 50 | endpoints_retry_policy.init_context(mocked_attempt.context) 51 | 52 | actual_tried_endpoints = [ 53 | mocked_attempt.context.get('endpoint') 54 | ] 55 | while endpoints_retry_policy.should_retry(mocked_attempt): 56 | endpoints_retry_policy.prepare_retry(mocked_attempt) 57 | actual_tried_endpoints.append(mocked_attempt.context.get('endpoint')) 58 | 59 | assert [ 60 | e.get_value() for e in actual_tried_endpoints 61 | ] == [ 62 | e.get_value() for e in mocked_endpoints_provider 63 | ] 64 | 65 | def test_skip_init_context(self, mocked_endpoints_provider): 66 | endpoints_retry_policy = EndpointsRetryPolicy( 67 | endpoints_provider=mocked_endpoints_provider, 68 | skip_init_context=True 69 | ) 70 | 71 | mocked_context = {} 72 | endpoints_retry_policy.init_context(mocked_context) 73 | 74 | assert not mocked_context.get('endpoint') 75 | assert not mocked_context.get('alternative_endpoints') 76 | -------------------------------------------------------------------------------- /tests/cases/test_http/test_middleware.py: -------------------------------------------------------------------------------- 1 | from qiniu.http.middleware import Middleware, RetryDomainsMiddleware 2 | from qiniu.http import qn_http_client 3 | 4 | 5 | class MiddlewareRecorder(Middleware): 6 | def __init__(self, rec, label): 7 | self.rec = rec 8 | self.label = label 9 | 10 | def __call__(self, request, nxt): 11 | self.rec.append( 12 | 'bef_{0}{1}'.format(self.label, len(self.rec)) 13 | ) 14 | resp = nxt(request) 15 | self.rec.append( 16 | 'aft_{0}{1}'.format(self.label, len(self.rec)) 17 | ) 18 | return resp 19 | 20 | 21 | class TestMiddleware: 22 | def test_middlewares(self, mock_server_addr): 23 | rec_ls = [] 24 | mw_a = MiddlewareRecorder(rec_ls, 'A') 25 | mw_b = MiddlewareRecorder(rec_ls, 'B') 26 | qn_http_client.get( 27 | '{scheme}://{host}/echo?status=200'.format( 28 | scheme=mock_server_addr.scheme, 29 | host=mock_server_addr.netloc 30 | ), 31 | middlewares=[ 32 | mw_a, 33 | mw_b 34 | ] 35 | ) 36 | assert rec_ls == ['bef_A0', 'bef_B1', 'aft_B2', 'aft_A3'] 37 | 38 | def test_retry_domains(self, mock_server_addr): 39 | rec_ls = [] 40 | mw_rec = MiddlewareRecorder(rec_ls, 'rec') 41 | ret, resp = qn_http_client.get( 42 | '{scheme}://fake.pysdk.qiniu.com/echo?status=200'.format( 43 | scheme=mock_server_addr.scheme 44 | ), 45 | middlewares=[ 46 | RetryDomainsMiddleware( 47 | backup_domains=[ 48 | 'unavailable.pysdk.qiniu.com', 49 | mock_server_addr.netloc 50 | ], 51 | max_retry_times=3 52 | ), 53 | mw_rec 54 | ] 55 | ) 56 | # ['bef_rec0', 'bef_rec1', 'bef_rec2'] are 'fake.pysdk.qiniu.com' with retried 3 times 57 | # ['bef_rec3', 'bef_rec4', 'bef_rec5'] are 'unavailable.pysdk.qiniu.com' with retried 3 times 58 | # ['bef_rec6', 'aft_rec7'] are mock_server and it's success 59 | assert rec_ls == [ 60 | 'bef_rec0', 'bef_rec1', 'bef_rec2', 61 | 'bef_rec3', 'bef_rec4', 'bef_rec5', 62 | 'bef_rec6', 'aft_rec7' 63 | ] 64 | assert ret == {} 65 | assert resp.status_code == 200 66 | 67 | def test_retry_domains_fail_fast(self, mock_server_addr): 68 | rec_ls = [] 69 | mw_rec = MiddlewareRecorder(rec_ls, 'rec') 70 | ret, resp = qn_http_client.get( 71 | '{scheme}://fake.pysdk.qiniu.com/echo?status=200'.format( 72 | scheme=mock_server_addr.scheme 73 | ), 74 | middlewares=[ 75 | RetryDomainsMiddleware( 76 | backup_domains=[ 77 | 'unavailable.pysdk.qiniu.com', 78 | mock_server_addr.netloc 79 | ], 80 | retry_condition=lambda _resp, _req: False 81 | ), 82 | mw_rec 83 | ] 84 | ) 85 | # ['bef_rec0'] are 'fake.pysdk.qiniu.com' with fail fast 86 | assert rec_ls == ['bef_rec0'] 87 | assert ret is None 88 | assert resp.status_code == -1 89 | -------------------------------------------------------------------------------- /tests/cases/test_http/test_qiniu_conf.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import requests 3 | 4 | from qiniu.compat import urlencode 5 | import qiniu.http as qiniu_http 6 | 7 | 8 | @pytest.fixture(scope='function') 9 | def retry_id(request, mock_server_addr): 10 | success_times = [] 11 | failure_times = [] 12 | if hasattr(request, 'param'): 13 | success_times = request.param.get('success_times', success_times) 14 | failure_times = request.param.get('failure_times', failure_times) 15 | query_dict = { 16 | 's': success_times, 17 | 'f': failure_times, 18 | } 19 | query_params = urlencode( 20 | query_dict, 21 | doseq=True 22 | ) 23 | request_url = '{scheme}://{host}/retry_me/__mgr__?{query_params}'.format( 24 | scheme=mock_server_addr.scheme, 25 | host=mock_server_addr.netloc, 26 | query_params=query_params 27 | ) 28 | resp = requests.put(request_url) 29 | resp.raise_for_status() 30 | record_id = resp.text 31 | yield record_id 32 | request_url = '{scheme}://{host}/retry_me/__mgr__?id={id}'.format( 33 | scheme=mock_server_addr.scheme, 34 | host=mock_server_addr.netloc, 35 | id=record_id 36 | ) 37 | resp = requests.delete(request_url) 38 | resp.raise_for_status() 39 | 40 | 41 | @pytest.fixture(scope='function') 42 | def reset_session(): 43 | qiniu_http._session = None 44 | yield 45 | 46 | 47 | class TestQiniuConfWithHTTP: 48 | @pytest.mark.usefixtures('reset_session') 49 | @pytest.mark.parametrize( 50 | 'set_conf_default', 51 | [ 52 | { 53 | 'connection_timeout': 0.3, 54 | 'connection_retries': 0 55 | } 56 | ], 57 | indirect=True 58 | ) 59 | @pytest.mark.parametrize( 60 | 'method,opts', 61 | [ 62 | ('get', {}), 63 | ('put', {'data': None, 'files': None}), 64 | ('post', {'data': None, 'files': None}), 65 | ('delete', {'params': None}) 66 | ], 67 | ids=lambda v: v if type(v) is str else 'opts' 68 | ) 69 | def test_timeout_conf(self, mock_server_addr, method, opts, set_conf_default): 70 | request_url = '{scheme}://{host}/timeout?delay=0.5'.format( 71 | scheme=mock_server_addr.scheme, 72 | host=mock_server_addr.netloc 73 | ) 74 | send = getattr(qiniu_http.qn_http_client, method) 75 | _ret, resp = send(request_url, **opts) 76 | assert 'Read timed out' in str(resp.exception) 77 | 78 | @pytest.mark.usefixtures('reset_session') 79 | @pytest.mark.parametrize( 80 | 'retry_id', 81 | [ 82 | { 83 | 'success_times': [0, 1], 84 | 'failure_times': [5, 0], 85 | }, 86 | ], 87 | indirect=True 88 | ) 89 | @pytest.mark.parametrize( 90 | 'set_conf_default', 91 | [ 92 | { 93 | 'connection_retries': 5 94 | } 95 | ], 96 | indirect=True 97 | ) 98 | @pytest.mark.parametrize( 99 | 'method,opts', 100 | [ 101 | # post not retry default, see 102 | # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS 103 | ('get', {}), 104 | ('put', {'data': None, 'files': None}), 105 | ('delete', {'params': None}) 106 | ], 107 | ids=lambda v: v if type(v) is str else 'opts' 108 | ) 109 | def test_retry_times(self, retry_id, mock_server_addr, method, opts, set_conf_default): 110 | request_url = '{scheme}://{host}/retry_me?id={id}'.format( 111 | scheme=mock_server_addr.scheme, 112 | host=mock_server_addr.netloc, 113 | id=retry_id 114 | ) 115 | send = getattr(qiniu_http.qn_http_client, method) 116 | _ret, resp = send(request_url, **opts) 117 | assert resp.status_code == 200 118 | -------------------------------------------------------------------------------- /tests/cases/test_http/test_resp.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from qiniu.http import qn_http_client, __return_wrapper as return_wrapper 4 | 5 | 6 | class TestResponse: 7 | def test_response_need_retry(self, mock_server_addr): 8 | def gen_case(code): 9 | if 0 <= code < 500: 10 | return code, False 11 | if code in [ 12 | 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 13 | ]: 14 | return code, False 15 | return code, True 16 | 17 | cases = [ 18 | gen_case(i) for i in range(-1, 800) 19 | ] 20 | 21 | for test_code, should_retry in cases: 22 | req_url = '{scheme}://{host}/echo?status={status}'.format( 23 | scheme=mock_server_addr.scheme, 24 | host=mock_server_addr.netloc, 25 | status=test_code 26 | ) 27 | if test_code < 0: 28 | req_url = 'http://fake.python-sdk.qiniu.com/' 29 | _ret, resp_info = qn_http_client.get(req_url) 30 | assert_msg = '{code} should{adv} retry'.format( 31 | code=test_code, 32 | adv='' if should_retry else ' NOT' 33 | ) 34 | assert resp_info.need_retry() == should_retry, assert_msg 35 | 36 | def test_json_decode_error(self, mock_server_addr): 37 | req_url = '{scheme}://{host}/echo?status=200'.format( 38 | scheme=mock_server_addr.scheme, 39 | host=mock_server_addr.netloc 40 | ) 41 | ret, resp = qn_http_client.get(req_url) 42 | assert resp.text_body is not None 43 | assert ret == {} 44 | 45 | def test_old_json_decode_error(self): 46 | """ 47 | test old return_wrapper 48 | """ 49 | 50 | def mock_res(): 51 | r = requests.Response() 52 | r.status_code = 200 53 | r.headers.__setitem__('X-Reqid', 'mockedReqid') 54 | 55 | def json_func(): 56 | raise ValueError('%s: line %d column %d (char %d)' % ('Expecting value', 0, 0, 0)) 57 | 58 | r.json = json_func 59 | 60 | return r 61 | 62 | mocked_res = mock_res() 63 | ret, _ = return_wrapper(mocked_res) 64 | assert ret == {} 65 | -------------------------------------------------------------------------------- /tests/cases/test_http/test_single_flight.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import time 3 | from multiprocessing.pool import ThreadPool 4 | 5 | from qiniu.http.single_flight import SingleFlight 6 | 7 | class TestSingleFlight: 8 | def test_single_flight_success(self): 9 | sf = SingleFlight() 10 | 11 | def fn(): 12 | return "result" 13 | 14 | result = sf.do("key1", fn) 15 | assert result == "result" 16 | 17 | def test_single_flight_exception(self): 18 | sf = SingleFlight() 19 | 20 | def fn(): 21 | raise ValueError("error") 22 | 23 | with pytest.raises(ValueError, match="error"): 24 | sf.do("key2", fn) 25 | 26 | def test_single_flight_concurrent(self): 27 | sf = SingleFlight() 28 | share_state = [] 29 | results = [] 30 | 31 | def fn(): 32 | time.sleep(1) 33 | share_state.append('share_state') 34 | return "result" 35 | 36 | def worker(_n): 37 | result = sf.do("key3", fn) 38 | results.append(result) 39 | 40 | ThreadPool(2).map(worker, range(5)) 41 | 42 | assert len(share_state) == 3 43 | assert all(result == "result" for result in results) 44 | 45 | def test_single_flight_different_keys(self): 46 | sf = SingleFlight() 47 | results = [] 48 | 49 | def fn(): 50 | time.sleep(1) 51 | return "result" 52 | 53 | def worker(n): 54 | result = sf.do("key{}".format(n), fn) 55 | results.append(result) 56 | 57 | ThreadPool(2).map(worker, range(2)) 58 | assert len(results) == 2 59 | assert all(result == "result" for result in results) 60 | -------------------------------------------------------------------------------- /tests/cases/test_retry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/tests/cases/test_retry/__init__.py -------------------------------------------------------------------------------- /tests/cases/test_retry/test_retrier.py: -------------------------------------------------------------------------------- 1 | import qiniu.retry 2 | import qiniu.retry.abc 3 | 4 | 5 | class MaxRetryPolicy(qiniu.retry.abc.RetryPolicy): 6 | def __init__(self, max_times): 7 | super(MaxRetryPolicy, self).__init__() 8 | self.max_times = max_times 9 | 10 | def is_important(self, attempt): 11 | return attempt.context[self]['retriedTimes'] >= self.max_times 12 | 13 | def init_context(self, context): 14 | context[self] = { 15 | 'retriedTimes': 0 16 | } 17 | 18 | def should_retry(self, attempt): 19 | if not attempt.exception: 20 | return False 21 | return attempt.context[self]['retriedTimes'] < self.max_times 22 | 23 | def prepare_retry(self, attempt): 24 | pass 25 | 26 | def after_retry(self, attempt, policy): 27 | attempt.context[self]['retriedTimes'] += 1 28 | 29 | 30 | class TestRetry: 31 | def test_retrier_with_code_block(self, use_ref): 32 | retried_times_ref = use_ref(0) 33 | 34 | def handle_before_retry(_attempt, _policy): 35 | retried_times_ref.value += 1 36 | return True 37 | 38 | max_retry_times = 3 39 | retrier = qiniu.retry.Retrier( 40 | policies=[ 41 | MaxRetryPolicy(max_times=max_retry_times) 42 | ], 43 | before_retry=handle_before_retry 44 | ) 45 | 46 | tried_times = 0 47 | try: 48 | for attempt in retrier: 49 | with attempt: 50 | tried_times += 1 51 | raise Exception('mocked error') 52 | except Exception as err: 53 | assert str(err) == 'mocked error' 54 | 55 | assert tried_times == max_retry_times + 1 56 | assert retried_times_ref.value == max_retry_times 57 | 58 | def test_retrier_with_try_do(self, use_ref): 59 | retried_times_ref = use_ref(0) 60 | 61 | def handle_before_retry(_attempt, _policy): 62 | retried_times_ref.value += 1 63 | return True 64 | 65 | max_retry_times = 3 66 | retrier = qiniu.retry.Retrier( 67 | policies=[ 68 | MaxRetryPolicy(max_times=max_retry_times) 69 | ], 70 | before_retry=handle_before_retry 71 | ) 72 | 73 | tried_times_ref = use_ref(0) 74 | 75 | def add_one(n): 76 | tried_times_ref.value += 1 77 | if tried_times_ref.value <= 3: 78 | raise Exception('mock error') 79 | return n + 1 80 | 81 | result = retrier.try_do(add_one, 1) 82 | assert result == 2 83 | assert tried_times_ref.value == max_retry_times + 1 84 | assert retried_times_ref.value == max_retry_times 85 | 86 | def test_retrier_with_decorator(self, use_ref): 87 | retried_times_ref = use_ref(0) 88 | 89 | def handle_before_retry(_attempt, _policy): 90 | retried_times_ref.value += 1 91 | return True 92 | 93 | max_retry_times = 3 94 | retrier = qiniu.retry.Retrier( 95 | policies=[ 96 | MaxRetryPolicy(max_times=max_retry_times) 97 | ], 98 | before_retry=handle_before_retry 99 | ) 100 | 101 | tried_times_ref = use_ref(0) 102 | 103 | @retrier.retry 104 | def add_one(n): 105 | tried_times_ref.value += 1 106 | if tried_times_ref.value <= 3: 107 | raise Exception('mock error') 108 | return n + 1 109 | 110 | result = add_one(1) 111 | assert result == 2 112 | assert tried_times_ref.value == max_retry_times + 1 113 | assert retried_times_ref.value == max_retry_times 114 | 115 | def test_retrier_with_no_need_retry_err(self, use_ref): 116 | retried_times_ref = use_ref(0) 117 | 118 | def handle_before_retry(_attempt, _policy): 119 | retried_times_ref.value += 1 120 | return True 121 | 122 | max_retry_times = 3 123 | retrier = qiniu.retry.Retrier( 124 | policies=[ 125 | MaxRetryPolicy(max_times=max_retry_times) 126 | ], 127 | before_retry=handle_before_retry 128 | ) 129 | 130 | tried_times = 0 131 | try: 132 | for attempt in retrier: 133 | with attempt: 134 | tried_times += 1 135 | err = Exception('mocked error') 136 | err.no_need_retry = True 137 | raise err 138 | except Exception as err: 139 | assert str(err) == 'mocked error' 140 | 141 | assert tried_times == 1 142 | assert retried_times_ref.value == 0 143 | -------------------------------------------------------------------------------- /tests/cases/test_services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/tests/cases/test_services/__init__.py -------------------------------------------------------------------------------- /tests/cases/test_services/test_processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/tests/cases/test_services/test_processing/__init__.py -------------------------------------------------------------------------------- /tests/cases/test_services/test_processing/test_pfop.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from qiniu import PersistentFop, op_save 4 | 5 | 6 | persistent_id = None 7 | 8 | 9 | class TestPersistentFop: 10 | def test_pfop_execute(self, qn_auth): 11 | pfop = PersistentFop(qn_auth, 'testres', 'sdktest') 12 | op = op_save('avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', 'pythonsdk', 'pfoptest') 13 | ops = [ 14 | op 15 | ] 16 | ret, resp = pfop.execute('sintel_trailer.mp4', ops, 1) 17 | assert resp.status_code == 200, resp 18 | assert ret is not None, resp 19 | assert ret['persistentId'] is not None, resp 20 | global persistent_id 21 | persistent_id = ret['persistentId'] 22 | 23 | def test_pfop_get_status(self, qn_auth): 24 | assert persistent_id is not None 25 | pfop = PersistentFop(qn_auth, 'testres', 'sdktest') 26 | ret, resp = pfop.get_status(persistent_id) 27 | assert resp.status_code == 200, resp 28 | assert ret is not None, resp 29 | 30 | @pytest.mark.parametrize( 31 | 'persistent_options', 32 | ( 33 | # included by above test_pfop_execute 34 | # { 35 | # 'persistent_type': None, 36 | # }, 37 | { 38 | 'persistent_type': 0, 39 | }, 40 | { 41 | 'persistent_type': 1, 42 | }, 43 | { 44 | 'workflow_template_id': 'test-workflow', 45 | }, 46 | ) 47 | ) 48 | def test_pfop_idle_time_task( 49 | self, 50 | set_conf_default, 51 | qn_auth, 52 | bucket_name, 53 | persistent_options, 54 | ): 55 | persistent_type = persistent_options.get('persistent_type') 56 | workflow_template_id = persistent_options.get('workflow_template_id', None) 57 | 58 | execute_opts = {} 59 | if workflow_template_id: 60 | execute_opts['workflow_template_id'] = workflow_template_id 61 | else: 62 | persistent_key = '_'.join([ 63 | 'test-pfop/test-pfop-by-api', 64 | 'type', 65 | str(persistent_type) 66 | ]) 67 | execute_opts['fops'] = [ 68 | op_save( 69 | op='avinfo', 70 | bucket=bucket_name, 71 | key=persistent_key 72 | ) 73 | ] 74 | 75 | if persistent_type is not None: 76 | execute_opts['persistent_type'] = persistent_type 77 | 78 | pfop = PersistentFop(qn_auth, bucket_name) 79 | key = 'qiniu.png' 80 | ret, resp = pfop.execute( 81 | key, 82 | **execute_opts 83 | ) 84 | 85 | assert resp.status_code == 200, resp 86 | assert ret is not None 87 | assert 'persistentId' in ret, resp 88 | 89 | ret, resp = pfop.get_status(ret['persistentId']) 90 | assert resp.status_code == 200, resp 91 | assert ret is not None 92 | assert ret['creationDate'] is not None, resp 93 | 94 | if persistent_id == 1: 95 | assert ret['type'] == 1, resp 96 | elif workflow_template_id: 97 | assert workflow_template_id in ret['taskFrom'], resp 98 | -------------------------------------------------------------------------------- /tests/cases/test_services/test_storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/tests/cases/test_services/test_storage/__init__.py -------------------------------------------------------------------------------- /tests/cases/test_services/test_storage/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | from collections import namedtuple 3 | from hashlib import new as hashlib_new 4 | import tempfile 5 | 6 | import pytest 7 | 8 | import requests 9 | 10 | from qiniu import BucketManager 11 | from qiniu.utils import io_md5 12 | from qiniu.config import QUERY_REGION_HOST, QUERY_REGION_BACKUP_HOSTS 13 | from qiniu.http.endpoint import Endpoint 14 | from qiniu.http.regions_provider import Region, ServiceName, get_default_regions_provider 15 | 16 | 17 | @pytest.fixture(scope='session') 18 | def bucket_manager(qn_auth): 19 | yield BucketManager(qn_auth) 20 | 21 | 22 | @pytest.fixture(scope='session') 23 | def get_remote_object_headers_and_md5(download_domain): 24 | def fetch_calc_md5(key=None, scheme=None, url=None): 25 | if not key and not url: 26 | raise TypeError('Must provide key or url') 27 | 28 | scheme = scheme if scheme is not None else 'http' 29 | download_url = '{}://{}/{}'.format(scheme, download_domain, key) 30 | if url: 31 | download_url = url 32 | 33 | resp = requests.get(download_url, stream=True) 34 | resp.raise_for_status() 35 | 36 | return resp.headers, io_md5(resp.iter_content(chunk_size=8192)) 37 | 38 | yield fetch_calc_md5 39 | 40 | 41 | @pytest.fixture(scope='session') 42 | def get_real_regions(): 43 | def _get_real_regions(access_key, bucket_name): 44 | regions = list( 45 | get_default_regions_provider( 46 | query_endpoints_provider=[ 47 | Endpoint.from_host(h) 48 | for h in [QUERY_REGION_HOST] + QUERY_REGION_BACKUP_HOSTS 49 | ], 50 | access_key=access_key, 51 | bucket_name=bucket_name 52 | ) 53 | ) 54 | 55 | if not regions: 56 | raise RuntimeError('No regions found') 57 | 58 | return regions 59 | 60 | yield _get_real_regions 61 | 62 | 63 | @pytest.fixture(scope='function') 64 | def regions_with_real_endpoints(access_key, bucket_name, get_real_regions): 65 | yield get_real_regions(access_key, bucket_name) 66 | 67 | 68 | @pytest.fixture(scope='function') 69 | def regions_with_fake_endpoints(regions_with_real_endpoints): 70 | """ 71 | Returns 72 | ------- 73 | list[Region] 74 | The first element is the fake region with fake endpoints for every service. 75 | The second element is the real region with first fake endpoint for every service. 76 | The rest elements are real regions with real endpoints if exists. 77 | """ 78 | regions = regions_with_real_endpoints 79 | 80 | regions[0].services = { 81 | sn: [ 82 | Endpoint('fake-{0}.python-sdk.qiniu.com'.format(sn.value)) 83 | ] + endpoints 84 | for sn, endpoints in regions[0].services.items() 85 | } 86 | 87 | regions.insert(0, Region( 88 | 'fake-id', 89 | 'fake-s3-id', 90 | services={ 91 | sn: [ 92 | Endpoint('fake-region-{0}.python-sdk.qiniu.com'.format(sn.value)) 93 | ] 94 | for sn in ServiceName 95 | } 96 | )) 97 | 98 | yield regions 99 | 100 | 101 | TempFile = namedtuple( 102 | 'TempFile', 103 | [ 104 | 'path', 105 | 'md5', 106 | 'name', 107 | 'size' 108 | ] 109 | ) 110 | 111 | 112 | @pytest.fixture(scope='function') 113 | def temp_file(request): 114 | size = 4 * 1024 115 | if hasattr(request, 'param'): 116 | size = request.param 117 | 118 | tmp_file_path = tempfile.mktemp() 119 | chunk_size = 4 * 1024 120 | 121 | md5_hasher = hashlib_new('md5') 122 | with open(tmp_file_path, 'wb') as f: 123 | remaining_bytes = size 124 | while remaining_bytes > 0: 125 | chunk = os.urandom(min(chunk_size, remaining_bytes)) 126 | f.write(chunk) 127 | md5_hasher.update(chunk) 128 | remaining_bytes -= len(chunk) 129 | 130 | yield TempFile( 131 | path=tmp_file_path, 132 | md5=md5_hasher.hexdigest(), 133 | name=os.path.basename(tmp_file_path), 134 | size=size 135 | ) 136 | 137 | try: 138 | os.remove(tmp_file_path) 139 | except Exception: 140 | pass 141 | -------------------------------------------------------------------------------- /tests/cases/test_services/test_storage/test_bucket_manager.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from qiniu.services.storage.bucket import BucketManager 4 | from qiniu.region import LegacyRegion 5 | from qiniu import build_batch_restore_ar 6 | 7 | 8 | @pytest.fixture(scope='function') 9 | def object_key(bucket_manager, bucket_name, rand_string): 10 | key_to = 'copyto_' + rand_string(8) 11 | bucket_manager.copy( 12 | bucket=bucket_name, 13 | key='copyfrom', 14 | bucket_to=bucket_name, 15 | key_to=key_to, 16 | force='true' 17 | ) 18 | 19 | yield key_to 20 | 21 | bucket_manager.delete(bucket_name, key_to) 22 | 23 | 24 | class TestBucketManager: 25 | # TODO(lihs): Move other test cases to here from test_qiniu.py 26 | def test_restore_ar(self, bucket_manager, bucket_name, object_key): 27 | ret, resp = bucket_manager.restore_ar(bucket_name, object_key, 7) 28 | assert not resp.ok(), resp 29 | ret, resp = bucket_manager.change_type(bucket_name, object_key, 2) 30 | assert resp.ok(), resp 31 | ret, resp = bucket_manager.restore_ar(bucket_name, object_key, 7) 32 | assert resp.ok(), resp 33 | 34 | @pytest.mark.parametrize( 35 | 'cond,expect_ok', 36 | [ 37 | ( 38 | None, True 39 | ), 40 | ( 41 | { 42 | 'mime': 'text/plain' 43 | }, 44 | True 45 | ), 46 | ( 47 | { 48 | 'mime': 'application/json' 49 | }, 50 | False 51 | ) 52 | ] 53 | ) 54 | def test_change_status( 55 | self, 56 | bucket_manager, 57 | bucket_name, 58 | object_key, 59 | cond, 60 | expect_ok 61 | ): 62 | ret, resp = bucket_manager.change_status(bucket_name, object_key, 1, cond) 63 | assert resp.ok() == expect_ok, resp 64 | 65 | def test_mkbucketv3(self, bucket_manager, rand_string): 66 | # tested manually, no drop bucket API to auto cleanup 67 | # ret, resp = bucket_manager.mkbucketv3('py-test-' + rand_string(8).lower(), 'z0') 68 | # assert resp.ok(), resp 69 | pass 70 | 71 | def test_list_bucket(self, bucket_manager, bucket_name): 72 | ret, resp = bucket_manager.list_bucket('na0') 73 | assert resp.ok(), resp 74 | assert any(b.get('tbl') == bucket_name for b in ret) 75 | 76 | def test_bucket_info(self, bucket_manager, bucket_name): 77 | ret, resp = bucket_manager.bucket_info(bucket_name) 78 | assert resp.ok(), resp 79 | for k in [ 80 | 'protected', 81 | 'private' 82 | ]: 83 | assert k in ret 84 | 85 | def test_change_bucket_permission(self, bucket_manager, bucket_name): 86 | ret, resp = bucket_manager.bucket_info(bucket_name) 87 | assert resp.ok(), resp 88 | original_private = ret['private'] 89 | ret, resp = bucket_manager.change_bucket_permission( 90 | bucket_name, 91 | 1 if original_private == 1 else 0 92 | ) 93 | assert resp.ok(), resp 94 | ret, resp = bucket_manager.change_bucket_permission( 95 | bucket_name, 96 | original_private 97 | ) 98 | assert resp.ok(), resp 99 | 100 | def test_batch_restore_ar( 101 | self, 102 | bucket_manager, 103 | bucket_name, 104 | object_key 105 | ): 106 | bucket_manager.change_type(bucket_name, object_key, 2) 107 | ops = build_batch_restore_ar( 108 | bucket_name, 109 | { 110 | object_key: 7 111 | } 112 | ) 113 | ret, resp = bucket_manager.batch(ops) 114 | assert resp.status_code == 200, resp 115 | assert len(ret) > 0 116 | assert ret[0].get('code') == 200, ret[0] 117 | 118 | def test_compatible_with_zone(self, qn_auth, bucket_name, regions_with_real_endpoints): 119 | r = LegacyRegion( 120 | io_host='https://fake-io.python-sdk.qiniu.com', 121 | rs_host='https://fake-rs.python-sdk.qiniu.com', 122 | rsf_host='https://fake-rsf.python-sdk.qiniu.com', 123 | api_host='https://fake-api.python-sdk.qiniu.com' 124 | ) 125 | bucket_manager = BucketManager( 126 | qn_auth, 127 | zone=r 128 | ) 129 | 130 | # rs host 131 | ret, resp = bucket_manager.stat(bucket_name, 'python-sdk.html') 132 | assert resp.status_code == -1 133 | assert ret is None 134 | 135 | # rsf host 136 | ret, _eof, resp = bucket_manager.list(bucket_name, '', limit=10) 137 | assert resp.status_code == -1 138 | assert ret is None 139 | 140 | # io host 141 | ret, info = bucket_manager.prefetch(bucket_name, 'python-sdk.html') 142 | assert resp.status_code == -1 143 | assert ret is None 144 | 145 | # api host 146 | # no API method to test 147 | 148 | @pytest.mark.parametrize( 149 | 'preferred_scheme', 150 | [ 151 | None, # default 'http' 152 | 'http', 153 | 'https' 154 | ] 155 | ) 156 | def test_preferred_scheme( 157 | self, 158 | qn_auth, 159 | bucket_name, 160 | preferred_scheme 161 | ): 162 | bucket_manager = BucketManager( 163 | auth=qn_auth, 164 | preferred_scheme=preferred_scheme 165 | ) 166 | 167 | ret, resp = bucket_manager.stat(bucket_name, 'python-sdk.html') 168 | 169 | assert ret is not None, resp 170 | assert resp.ok(), resp 171 | 172 | expect_scheme = preferred_scheme if preferred_scheme else 'http' 173 | assert resp.url.startswith(expect_scheme + '://'), resp.url 174 | 175 | def test_operation_with_regions_and_retrier( 176 | self, 177 | qn_auth, 178 | bucket_name, 179 | regions_with_fake_endpoints 180 | ): 181 | bucket_manager = BucketManager( 182 | auth=qn_auth, 183 | regions=regions_with_fake_endpoints, 184 | ) 185 | 186 | ret, resp = bucket_manager.stat(bucket_name, 'python-sdk.html') 187 | 188 | assert ret is not None, resp 189 | assert resp.ok(), resp 190 | 191 | def test_uc_service_with_retrier( 192 | self, 193 | qn_auth, 194 | bucket_name, 195 | regions_with_fake_endpoints 196 | ): 197 | bucket_manager = BucketManager( 198 | auth=qn_auth, 199 | regions=regions_with_fake_endpoints 200 | ) 201 | 202 | ret, resp = bucket_manager.list_bucket('na0') 203 | assert resp.ok(), resp 204 | assert len(ret) > 0, resp 205 | assert any(b.get('tbl') for b in ret), ret 206 | -------------------------------------------------------------------------------- /tests/cases/test_services/test_storage/test_upload_pfop.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import qiniu 4 | 5 | 6 | KB = 1024 7 | MB = 1024 * KB 8 | GB = 1024 * MB 9 | 10 | 11 | # set a bucket lifecycle manually to delete prefix `test-pfop`! 12 | # or this test will continue to occupy bucket space. 13 | class TestPersistentFopByUpload: 14 | @pytest.mark.parametrize('temp_file', [10 * MB], indirect=True) 15 | @pytest.mark.parametrize( 16 | 'persistent_options', 17 | ( 18 | { 19 | 'persistent_type': None, 20 | }, 21 | { 22 | 'persistent_type': 0, 23 | }, 24 | { 25 | 'persistent_type': 1, 26 | }, 27 | { 28 | 'persistent_workflow_template_id': 'test-workflow', 29 | }, 30 | ) 31 | ) 32 | def test_pfop_with_upload( 33 | self, 34 | set_conf_default, 35 | qn_auth, 36 | bucket_name, 37 | temp_file, 38 | persistent_options, 39 | ): 40 | key = 'test-pfop/upload-file' 41 | persistent_type = persistent_options.get('persistent_type') 42 | persistent_workflow_template_id = persistent_options.get('persistent_workflow_template_id') 43 | 44 | upload_policy = {} 45 | 46 | # set pfops or tmplate id 47 | if persistent_workflow_template_id: 48 | upload_policy['persistentWorkflowTemplateID'] = persistent_workflow_template_id 49 | else: 50 | persistent_key = '_'.join([ 51 | 'test-pfop/test-pfop-by-upload', 52 | 'type', 53 | str(persistent_type) 54 | ]) 55 | persistent_ops = ';'.join([ 56 | qiniu.op_save( 57 | op='avinfo', 58 | bucket=bucket_name, 59 | key=persistent_key 60 | ) 61 | ]) 62 | upload_policy['persistentOps'] = persistent_ops 63 | 64 | # set persistent type 65 | if persistent_type is not None: 66 | upload_policy['persistentType'] = persistent_type 67 | 68 | # upload 69 | token = qn_auth.upload_token( 70 | bucket_name, 71 | key, 72 | policy=upload_policy 73 | ) 74 | ret, resp = qiniu.put_file( 75 | token, 76 | key, 77 | temp_file.path, 78 | check_crc=True 79 | ) 80 | 81 | assert ret is not None, resp 82 | assert ret['key'] == key, resp 83 | assert 'persistentId' in ret, resp 84 | 85 | pfop = qiniu.PersistentFop(qn_auth, bucket_name) 86 | ret, resp = pfop.get_status(ret['persistentId']) 87 | assert resp.status_code == 200, resp 88 | assert ret is not None, resp 89 | assert ret['creationDate'] is not None, resp 90 | 91 | if persistent_type == 1: 92 | assert ret['type'] == 1, resp 93 | elif persistent_workflow_template_id: 94 | assert persistent_workflow_template_id in ret['taskFrom'], resp 95 | -------------------------------------------------------------------------------- /tests/cases/test_utils.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta, tzinfo 2 | 3 | from qiniu import utils, compat 4 | 5 | 6 | class _CN_TZINFO(tzinfo): 7 | def utcoffset(self, dt): 8 | return timedelta(hours=8) 9 | 10 | def tzname(self, dt): 11 | return "CST" 12 | 13 | def dst(self, dt): 14 | return timedelta(0) 15 | 16 | 17 | class TestUtils: 18 | def test_urlsafe(self): 19 | a = 'hello\x96' 20 | u = utils.urlsafe_base64_encode(a) 21 | assert compat.b(a) == utils.urlsafe_base64_decode(u) 22 | 23 | def test_canonical_mime_header_key(self): 24 | field_names = [ 25 | ":status", 26 | ":x-test-1", 27 | ":x-Test-2", 28 | "content-type", 29 | "CONTENT-LENGTH", 30 | "oRiGin", 31 | "ReFer", 32 | "Last-Modified", 33 | "acCePt-ChArsEt", 34 | "x-test-3", 35 | "cache-control", 36 | ] 37 | expect_canonical_field_names = [ 38 | ":status", 39 | ":x-test-1", 40 | ":x-Test-2", 41 | "Content-Type", 42 | "Content-Length", 43 | "Origin", 44 | "Refer", 45 | "Last-Modified", 46 | "Accept-Charset", 47 | "X-Test-3", 48 | "Cache-Control", 49 | ] 50 | assert len(field_names) == len(expect_canonical_field_names) 51 | for i in range(len(field_names)): 52 | assert utils.canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] 53 | 54 | def test_entry(self): 55 | case_list = [ 56 | { 57 | 'msg': 'normal', 58 | 'bucket': 'qiniuphotos', 59 | 'key': 'gogopher.jpg', 60 | 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' 61 | }, 62 | { 63 | 'msg': 'key empty', 64 | 'bucket': 'qiniuphotos', 65 | 'key': '', 66 | 'expect': 'cWluaXVwaG90b3M6' 67 | }, 68 | { 69 | 'msg': 'key undefined', 70 | 'bucket': 'qiniuphotos', 71 | 'key': None, 72 | 'expect': 'cWluaXVwaG90b3M=' 73 | }, 74 | { 75 | 'msg': 'key need replace plus symbol', 76 | 'bucket': 'qiniuphotos', 77 | 'key': '012ts>a', 78 | 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ==' 79 | }, 80 | { 81 | 'msg': 'key need replace slash symbol', 82 | 'bucket': 'qiniuphotos', 83 | 'key': '012ts?a', 84 | 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ==' 85 | } 86 | ] 87 | for c in case_list: 88 | assert c.get('expect') == utils.entry(c.get('bucket'), c.get('key')), c.get('msg') 89 | 90 | def test_decode_entry(self): 91 | case_list = [ 92 | { 93 | 'msg': 'normal', 94 | 'expect': { 95 | 'bucket': 'qiniuphotos', 96 | 'key': 'gogopher.jpg' 97 | }, 98 | 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' 99 | }, 100 | { 101 | 'msg': 'key empty', 102 | 'expect': { 103 | 'bucket': 'qiniuphotos', 104 | 'key': '' 105 | }, 106 | 'entry': 'cWluaXVwaG90b3M6' 107 | }, 108 | { 109 | 'msg': 'key undefined', 110 | 'expect': { 111 | 'bucket': 'qiniuphotos', 112 | 'key': None 113 | }, 114 | 'entry': 'cWluaXVwaG90b3M=' 115 | }, 116 | { 117 | 'msg': 'key need replace plus symbol', 118 | 'expect': { 119 | 'bucket': 'qiniuphotos', 120 | 'key': '012ts>a' 121 | }, 122 | 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ==' 123 | }, 124 | { 125 | 'msg': 'key need replace slash symbol', 126 | 'expect': { 127 | 'bucket': 'qiniuphotos', 128 | 'key': '012ts?a' 129 | }, 130 | 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ==' 131 | } 132 | ] 133 | for c in case_list: 134 | bucket, key = utils.decode_entry(c.get('entry')) 135 | assert bucket == c.get('expect', {}).get('bucket'), c.get('msg') 136 | assert key == c.get('expect', {}).get('key'), c.get('msg') 137 | 138 | def test_dt2ts(self): 139 | dt = datetime(year=2011, month=8, day=3, tzinfo=_CN_TZINFO()) 140 | expect = 1312300800 141 | assert utils.dt2ts(dt) == expect 142 | 143 | base_dt = datetime(year=2011, month=8, day=3) 144 | now_dt = datetime.now() 145 | assert int((now_dt - base_dt).total_seconds()) == utils.dt2ts(now_dt) - utils.dt2ts(base_dt) 146 | -------------------------------------------------------------------------------- /tests/cases/test_zone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qiniu/python-sdk/e86ef9844e8a3ca39712604daa61750397150856/tests/cases/test_zone/__init__.py -------------------------------------------------------------------------------- /tests/cases/test_zone/test_lagacy_region.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from qiniu.http.region import Region, ServiceName 4 | from qiniu.region import LegacyRegion 5 | from qiniu.compat import json, is_py2 6 | 7 | 8 | @pytest.fixture 9 | def mocked_hosts(): 10 | mocked_hosts = { 11 | ServiceName.UP: ['https://up.python-example.qiniu.com', 'https://up-2.python-example.qiniu.com'], 12 | ServiceName.IO: ['https://io.python-example.qiniu.com'], 13 | ServiceName.RS: ['https://rs.python-example.qiniu.com'], 14 | ServiceName.RSF: ['https://rsf.python-example.qiniu.com'], 15 | ServiceName.API: ['https://api.python-example.qiniu.com'] 16 | } 17 | yield mocked_hosts 18 | 19 | 20 | @pytest.fixture 21 | def mock_legacy_region(mocked_hosts): 22 | region = LegacyRegion( 23 | up_host=mocked_hosts[ServiceName.UP][0], 24 | up_host_backup=mocked_hosts[ServiceName.UP][1], 25 | io_host=mocked_hosts[ServiceName.IO][0], 26 | rs_host=mocked_hosts[ServiceName.RS][0], 27 | rsf_host=mocked_hosts[ServiceName.RSF][0], 28 | api_host=mocked_hosts[ServiceName.API][0] 29 | ) 30 | yield region 31 | 32 | 33 | class TestLegacyRegion: 34 | def test_get_hosts_from_self(self, mocked_hosts, mock_legacy_region, qn_auth, bucket_name): 35 | cases = [ 36 | # up will always query from the old version, 37 | # which version implements the `get_up_host_*` method 38 | ( 39 | mock_legacy_region.get_io_host(qn_auth.get_access_key(), None), 40 | mocked_hosts[ServiceName.IO][0] 41 | ), 42 | ( 43 | mock_legacy_region.get_rs_host(qn_auth.get_access_key(), None), 44 | mocked_hosts[ServiceName.RS][0] 45 | ), 46 | ( 47 | mock_legacy_region.get_rsf_host(qn_auth.get_access_key(), None), 48 | mocked_hosts[ServiceName.RSF][0] 49 | ), 50 | ( 51 | mock_legacy_region.get_api_host(qn_auth.get_access_key(), None), 52 | mocked_hosts[ServiceName.API][0] 53 | ) 54 | ] 55 | for actual, expect in cases: 56 | assert actual == expect 57 | 58 | def test_get_hosts_from_query(self, qn_auth, bucket_name): 59 | up_token = qn_auth.upload_token(bucket_name) 60 | region = LegacyRegion() 61 | up_host = region.get_up_host_by_token(up_token, None) 62 | up_host_backup = region.get_up_host_backup_by_token(up_token, None) 63 | if is_py2: 64 | up_host = up_host.encode() 65 | up_host_backup = up_host_backup.encode() 66 | assert type(up_host) is str and len(up_host) > 0 67 | assert type(up_host_backup) is str and len(up_host_backup) > 0 68 | assert up_host != up_host_backup 69 | 70 | def test_compatible_with_http_region(self, mocked_hosts, mock_legacy_region): 71 | assert isinstance(mock_legacy_region, Region) 72 | assert mocked_hosts == { 73 | k: [ 74 | e.get_value() 75 | for e in mock_legacy_region.services[k] 76 | ] 77 | for k in mocked_hosts 78 | } 79 | 80 | def test_get_bucket_hosts(self, access_key, bucket_name): 81 | region = LegacyRegion() 82 | bucket_hosts = region.get_bucket_hosts(access_key, bucket_name) 83 | for k in [ 84 | 'upHosts', 85 | 'ioHosts', 86 | 'rsHosts', 87 | 'rsfHosts', 88 | 'apiHosts' 89 | ]: 90 | assert all(h.startswith('http') for h in bucket_hosts[k]), bucket_hosts[k] 91 | 92 | def test_bucket_hosts(self, access_key, bucket_name): 93 | region = LegacyRegion() 94 | bucket_hosts_str = region.bucket_hosts(access_key, bucket_name) 95 | bucket_hosts = json.loads(bucket_hosts_str) 96 | 97 | region_hosts = bucket_hosts.get('hosts', []) 98 | 99 | assert len(region_hosts) > 0 100 | 101 | for r in region_hosts: 102 | for k in [ 103 | 'up', 104 | 'io', 105 | 'rs', 106 | 'rsf', 107 | 'api' 108 | ]: 109 | service_hosts = r[k].get('domains') 110 | assert len(service_hosts) > 0 111 | assert all(len(h) for h in service_hosts) 112 | -------------------------------------------------------------------------------- /tests/cases/test_zone/test_qiniu_conf.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from qiniu import Zone 4 | from qiniu.config import get_default 5 | 6 | TEST_RS_HOST = 'rs.test.region.compatible.config.qiniu.com' 7 | TEST_RSF_HOST = 'rsf.test.region.compatible.config.qiniu.com' 8 | TEST_API_HOST = 'api.test.region.compatible.config.qiniu.com' 9 | 10 | 11 | class TestQiniuConfWithZone: 12 | """ 13 | Test qiniu.conf with Zone(aka LegacyRegion) 14 | """ 15 | 16 | @pytest.mark.parametrize( 17 | 'set_conf_default', 18 | [ 19 | { 20 | 'default_uc_backup_hosts': [], 21 | }, 22 | { 23 | 'default_uc_backup_hosts': [], 24 | 'default_query_region_backup_hosts': [] 25 | } 26 | ], 27 | indirect=True 28 | ) 29 | def test_disable_backup_hosts(self, set_conf_default): 30 | assert get_default('default_uc_backup_hosts') == [] 31 | assert get_default('default_query_region_backup_hosts') == [] 32 | 33 | @pytest.mark.parametrize( 34 | 'set_conf_default', 35 | [ 36 | { 37 | 'default_rs_host': TEST_RS_HOST, 38 | 'default_rsf_host': TEST_RSF_HOST, 39 | 'default_api_host': TEST_API_HOST 40 | } 41 | ], 42 | indirect=True 43 | ) 44 | def test_config_compatible(self, set_conf_default): 45 | zone = Zone() 46 | assert zone.get_rs_host("mock_ak", "mock_bucket") == TEST_RS_HOST 47 | assert zone.get_rsf_host("mock_ak", "mock_bucket") == TEST_RSF_HOST 48 | assert zone.get_api_host("mock_ak", "mock_bucket") == TEST_API_HOST 49 | 50 | @pytest.mark.parametrize( 51 | 'set_conf_default', 52 | [ 53 | { 54 | 'default_query_region_host': 'https://fake-uc.pysdk.qiniu.com' 55 | } 56 | ], 57 | indirect=True 58 | ) 59 | def test_query_region_with_custom_domain(self, access_key, bucket_name, set_conf_default): 60 | with pytest.raises(Exception) as exc: 61 | zone = Zone() 62 | zone.bucket_hosts(access_key, bucket_name) 63 | assert 'HTTP Status Code -1' in str(exc) 64 | 65 | @pytest.mark.parametrize( 66 | 'set_conf_default', 67 | [ 68 | { 69 | 'default_query_region_host': 'https://fake-uc.pysdk.qiniu.com', 70 | 'default_query_region_backup_hosts': [ 71 | 'unavailable-uc.pysdk.qiniu.com', 72 | 'uc.qbox.me' 73 | ] 74 | } 75 | ], 76 | indirect=True 77 | ) 78 | def test_query_region_with_backup_domains(self, access_key, bucket_name, set_conf_default): 79 | zone = Zone() 80 | data = zone.bucket_hosts(access_key, bucket_name) 81 | assert data != 'null' and len(data) > 0 82 | 83 | @pytest.mark.parametrize( 84 | 'set_conf_default', 85 | [ 86 | { 87 | 'default_uc_host': 'https://fake-uc.pysdk.qiniu.com', 88 | 'default_query_region_backup_hosts': [ 89 | 'unavailable-uc.phpsdk.qiniu.com', 90 | 'uc.qbox.me' 91 | ] 92 | } 93 | ], 94 | indirect=True 95 | ) 96 | def test_query_region_with_uc_and_backup_domains(self, access_key, bucket_name, set_conf_default): 97 | zone = Zone() 98 | data = zone.bucket_hosts(access_key, bucket_name) 99 | assert data != 'null' 100 | -------------------------------------------------------------------------------- /tests/mock_server/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import http.server 3 | import http.client 4 | import logging 5 | import sys 6 | from urllib.parse import urlparse 7 | 8 | from routes import routes 9 | 10 | 11 | class MockHandler(http.server.BaseHTTPRequestHandler): 12 | def do_GET(self): 13 | self.handle_request('GET') 14 | 15 | def do_POST(self): 16 | self.handle_request('POST') 17 | 18 | def do_PUT(self): 19 | self.handle_request('PUT') 20 | 21 | def do_DELETE(self): 22 | self.handle_request('DELETE') 23 | 24 | def do_OPTIONS(self): 25 | self.handle_request('OPTIONS') 26 | 27 | def do_HEAD(self): 28 | self.handle_request('HEAD') 29 | 30 | def handle_request(self, method): 31 | parsed_uri = urlparse(self.path) 32 | handle = routes.get(parsed_uri.path) 33 | if callable(handle): 34 | try: 35 | handle(method=method, parsed_uri=parsed_uri, request_handler=self) 36 | except Exception: 37 | logging.exception('Exception while handling.') 38 | else: 39 | self.send_response(404) 40 | self.send_header('Content-type', 'text/html') 41 | self.end_headers() 42 | self.wfile.write(b'404 Not Found') 43 | 44 | 45 | if __name__ == '__main__': 46 | parser = argparse.ArgumentParser() 47 | parser.add_argument( 48 | '--port', 49 | type=int, 50 | default=8000, 51 | ) 52 | args = parser.parse_args() 53 | 54 | logging.basicConfig( 55 | level=logging.INFO, 56 | datefmt='%Y-%m-%d %H:%M:%S', 57 | format='[%(asctime)s %(levelname)s] %(message)s', 58 | handlers=[logging.StreamHandler(sys.stdout)], 59 | ) 60 | 61 | server_address = ('', args.port) 62 | httpd = http.server.HTTPServer(server_address, MockHandler) 63 | logging.info('Mock Server running on port {}...'.format(args.port)) 64 | 65 | httpd.serve_forever() 66 | -------------------------------------------------------------------------------- /tests/mock_server/routes/__init__.py: -------------------------------------------------------------------------------- 1 | from .timeout import * 2 | from .echo import * 3 | from .retry_me import * 4 | 5 | routes = { 6 | '/timeout': handle_timeout, 7 | '/echo': handle_echo, 8 | '/retry_me': handle_retry_me, 9 | '/retry_me/__mgr__': handle_mgr_retry_me, 10 | } 11 | -------------------------------------------------------------------------------- /tests/mock_server/routes/echo.py: -------------------------------------------------------------------------------- 1 | import http 2 | import logging 3 | from urllib.parse import parse_qs 4 | 5 | 6 | def handle_echo(method, parsed_uri, request_handler): 7 | """ 8 | Parameters 9 | ---------- 10 | method: str 11 | HTTP method 12 | parsed_uri: urllib.parse.ParseResult 13 | parsed URI 14 | request_handler: http.server.BaseHTTPRequestHandler 15 | request handler 16 | """ 17 | if method not in []: 18 | # all method allowed 19 | pass 20 | echo_status = parse_qs(parsed_uri.query).get('status') 21 | if not echo_status: 22 | echo_status = http.HTTPStatus.BAD_REQUEST 23 | logging.error('No echo status specified') 24 | echo_body = f'param status is required' 25 | else: 26 | echo_status = int(echo_status[0]) 27 | echo_body = f'Response echo status is {echo_status}' 28 | 29 | request_handler.send_response(echo_status) 30 | request_handler.send_header('Content-Type', 'text/plain') 31 | request_handler.send_header('X-Reqid', 'mocked-req-id') 32 | request_handler.end_headers() 33 | 34 | request_handler.wfile.write(echo_body.encode('utf-8')) 35 | -------------------------------------------------------------------------------- /tests/mock_server/routes/retry_me.py: -------------------------------------------------------------------------------- 1 | import http 2 | import random 3 | import string 4 | 5 | from urllib.parse import parse_qs 6 | 7 | __failure_record = {} 8 | 9 | 10 | def should_fail_by_times(success_times=None, failure_times=None): 11 | """ 12 | Parameters 13 | ---------- 14 | success_times: list[int], default=[1] 15 | failure_times: list[int], default=[0] 16 | 17 | Returns 18 | ------- 19 | Generator[bool, None, None] 20 | 21 | Examples 22 | -------- 23 | 24 | should_fail_by_times([2], [3]) 25 | will succeed 2 times and failed 3 times, and loop 26 | 27 | should_fail_by_times([2, 4], [3]) 28 | will succeed 2 times and failed 3 times, 29 | then succeeded 4 times and failed 3 time, and loop 30 | """ 31 | if not success_times: 32 | success_times = [1] 33 | if not failure_times: 34 | failure_times = [0] 35 | 36 | def success_times_gen(): 37 | while True: 38 | for i in success_times: 39 | yield i 40 | 41 | def failure_times_gen(): 42 | while True: 43 | for i in failure_times: 44 | yield i 45 | 46 | success_times_iter = success_times_gen() 47 | fail_times_iter = failure_times_gen() 48 | 49 | while True: 50 | success = next(success_times_iter) 51 | fail = next(fail_times_iter) 52 | for _ in range(success): 53 | yield False 54 | for _ in range(fail): 55 | yield True 56 | 57 | 58 | def handle_mgr_retry_me(method, parsed_uri, request_handler): 59 | """ 60 | Parameters 61 | ---------- 62 | method: str 63 | HTTP method 64 | parsed_uri: urllib.parse.ParseResult 65 | parsed URI 66 | request_handler: http.server.BaseHTTPRequestHandler 67 | request handler 68 | """ 69 | if method not in ['PUT', 'DELETE']: 70 | request_handler.send_response(http.HTTPStatus.METHOD_NOT_ALLOWED) 71 | return 72 | match method: 73 | case 'PUT': 74 | # s for success 75 | success_times = parse_qs(parsed_uri.query).get('s', []) 76 | # f for failure 77 | failure_times = parse_qs(parsed_uri.query).get('f', []) 78 | 79 | record_id = ''.join(random.choices(string.ascii_letters, k=16)) 80 | 81 | __failure_record[record_id] = should_fail_by_times( 82 | success_times=[int(n) for n in success_times], 83 | failure_times=[int(n) for n in failure_times] 84 | ) 85 | 86 | request_handler.send_response(http.HTTPStatus.OK) 87 | request_handler.send_header('Content-Type', 'text/plain') 88 | request_handler.send_header('X-Reqid', record_id) 89 | request_handler.end_headers() 90 | 91 | request_handler.wfile.write(record_id.encode('utf-8')) 92 | case 'DELETE': 93 | record_id = parse_qs(parsed_uri.query).get('id') 94 | if not record_id or not record_id[0]: 95 | request_handler.send_response(http.HTTPStatus.BAD_REQUEST) 96 | return 97 | record_id = record_id[0] 98 | 99 | if record_id in __failure_record: 100 | del __failure_record[record_id] 101 | 102 | request_handler.send_response(http.HTTPStatus.NO_CONTENT) 103 | request_handler.send_header('X-Reqid', record_id) 104 | request_handler.end_headers() 105 | 106 | 107 | def handle_retry_me(method, parsed_uri, request_handler): 108 | """ 109 | Parameters 110 | ---------- 111 | method: str 112 | HTTP method 113 | parsed_uri: urllib.parse.ParseResult 114 | parsed URI 115 | request_handler: http.server.BaseHTTPRequestHandler 116 | request handler 117 | """ 118 | if method not in []: 119 | # all method allowed 120 | pass 121 | record_id = parse_qs(parsed_uri.query).get('id') 122 | if not record_id or not record_id[0]: 123 | request_handler.send_response(http.HTTPStatus.BAD_REQUEST) 124 | return 125 | record_id = record_id[0] 126 | 127 | should_fail = next(__failure_record[record_id]) 128 | 129 | if should_fail: 130 | request_handler.send_response(-1) 131 | request_handler.send_header('Content-Type', 'text/plain') 132 | request_handler.send_header('X-Reqid', record_id) 133 | request_handler.end_headers() 134 | 135 | resp_body = 'service unavailable' 136 | request_handler.wfile.write(resp_body.encode('utf-8')) 137 | return 138 | 139 | request_handler.send_response(http.HTTPStatus.OK) 140 | request_handler.send_header('Content-Type', 'text/plain') 141 | request_handler.send_header('X-Reqid', record_id) 142 | request_handler.end_headers() 143 | 144 | resp_body = 'ok' 145 | request_handler.wfile.write(resp_body.encode('utf-8')) 146 | -------------------------------------------------------------------------------- /tests/mock_server/routes/timeout.py: -------------------------------------------------------------------------------- 1 | import http 2 | import logging 3 | import time 4 | 5 | from urllib.parse import parse_qs 6 | 7 | 8 | def handle_timeout(method, parsed_uri, request_handler): 9 | """ 10 | Parameters 11 | ---------- 12 | method: str 13 | HTTP method 14 | parsed_uri: urllib.parse.ParseResult 15 | parsed URI 16 | request_handler: http.server.BaseHTTPRequestHandler 17 | request handler 18 | """ 19 | if method not in []: 20 | # all method allowed 21 | pass 22 | delay = parse_qs(parsed_uri.query).get('delay') 23 | if not delay: 24 | delay = 3 25 | logging.info('No delay specified. Fallback to %s seconds.', delay) 26 | else: 27 | delay = float(delay[0]) 28 | 29 | time.sleep(delay) 30 | request_handler.send_response(http.HTTPStatus.OK) 31 | request_handler.send_header('Content-Type', 'text/plain') 32 | request_handler.send_header('X-Reqid', 'mocked-req-id') 33 | request_handler.end_headers() 34 | 35 | resp_body = f'Response after {delay} seconds' 36 | request_handler.wfile.write(resp_body.encode('utf-8')) 37 | --------------------------------------------------------------------------------