├── .flake8 ├── .github └── workflows │ └── pr.yml ├── .gitignore ├── .pylintrc ├── CHANGELOG.md ├── LICENSE ├── LICENSE.txt ├── Makefile ├── Pipfile ├── README.md ├── setup.py ├── siaskynet ├── __init__.py ├── _download.py ├── _encryption.py ├── _upload.py ├── test_utils.py └── utils.py ├── testdata ├── dir1 │ └── file2 ├── file1 └── file3 └── tests ├── __init__.py ├── test_integration_download.py └── test_integration_upload.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | count = True 3 | show-source = True 4 | statistics = True 5 | per-file-ignores = __init__.py:F401 6 | -------------------------------------------------------------------------------- /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | name: Pull Request 2 | 3 | # Run on PRs. 4 | on: [pull_request] 5 | 6 | jobs: 7 | build: 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest, macos-latest, windows-latest] 12 | python-version: [3.5, 3.6, 3.7, 3.8, 3.9] 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set up Python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v2 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | - name: Install dependencies 21 | run: make install 22 | - name: Build 23 | run: pipenv run build 24 | - name: Lint 25 | run: pipenv run lint 26 | - name: Test 27 | run: pipenv run test 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python modules. 2 | *.pyc 3 | 4 | # Setuptools distribution folder. 5 | /dist/ 6 | /build/ 7 | 8 | # Python egg metadata, regenerated from source files by setuptools. 9 | /*.egg-info 10 | 11 | .DS_Store 12 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | disable= 4 | fixme 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [2.2.0] 4 | 5 | _This release adds Windows support._ 6 | 7 | ### Added 8 | 9 | - Windows is now fully supported 10 | 11 | ### Changed 12 | 13 | - A bug with directory uploads on Windows has been fixed 14 | 15 | ## [2.1.0] 16 | 17 | ### Added 18 | 19 | - `upload` and `upload_request` API 20 | - Support for chunked uploading 21 | 22 | ## [2.0.0] 23 | 24 | ### Changed 25 | 26 | - This SDK has been updated to match Browser JS and require a client. You will 27 | first need to create a client and then make all API calls from this client. 28 | - Connection options can now be passed to the client, in addition to individual 29 | API calls, to be applied to all API calls. 30 | - The `defaultPortalUrl` string has been renamed to `defaultSkynetPortalUrl` and 31 | `defaultPortalUrl` is now a function. 32 | 33 | ## [1.1.0] 34 | 35 | ### Added 36 | 37 | - `metadata` function 38 | - Upload and download `_request` functions 39 | - Common Options object 40 | - API authentication 41 | 42 | ### Changed 43 | 44 | - Some upload bugs were fixed. 45 | 46 | ## [1.0.2] 47 | 48 | ## Added 49 | 50 | - Possibility to use chunks 51 | 52 | ## [1.0.1] 53 | 54 | ### Changed 55 | 56 | - Drop UUID 57 | 58 | ## [1.0.0] 59 | 60 | ### Added 61 | 62 | - Upload and download functionality. 63 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Nebulous 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Nebulous Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | dirs = siaskynet/ tests/ 2 | 3 | build: 4 | python -m compileall ./siaskynet 5 | 6 | install: 7 | python -m pip install pipenv 8 | pipenv install --dev 9 | 10 | lint: 11 | flake8 $(dirs) 12 | pylint $(dirs) 13 | 14 | test: 15 | pytest 16 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [scripts] 7 | build = "make build" 8 | install = "make install" 9 | lint = "make lint" 10 | test = "make test" 11 | 12 | [dev-packages] 13 | flake8 = "*" 14 | pylint = "*" 15 | pytest = "*" 16 | 17 | [packages] 18 | atomicwrites = "*" 19 | requests = "*" 20 | responses = "*" 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Skynet Python SDK 2 | 3 | > :warning: This repo has been archived and moved under the new [SkynetLabs](https://github.com/SkynetLabs) repo [here](https://github.com/SkynetLabs/python-skynet) 4 | 5 | [![Version](https://img.shields.io/pypi/v/siaskynet)](https://pypi.org/project/siaskynet) 6 | [![Python](https://img.shields.io/pypi/pyversions/siaskynet)](https://pypi.org/project/siaskynet) 7 | [![Build Status](https://img.shields.io/github/workflow/status/NebulousLabs/python-skynet/Pull%20Request)](https://github.com/NebulousLabs/python-skynet/actions) 8 | [![Contributors](https://img.shields.io/github/contributors/NebulousLabs/python-skynet)](https://github.com/NebulousLabs/python-skynet/graphs/contributors) 9 | [![License](https://img.shields.io/pypi/l/siaskynet)](https://pypi.org/project/siaskynet) 10 | 11 | An SDK for integrating Skynet into Python applications. 12 | 13 | ## Instructions 14 | 15 | We recommend running your Python application using [pipenv](https://pipenv-searchable.readthedocs.io/basics.html). 16 | 17 | You can use `siaskynet` by installing it with `pip`, adding it to your project's `Pipfile`, or by cloning this repository. 18 | 19 | ## Documentation 20 | 21 | For documentation complete with examples, please see [the Skynet SDK docs](https://siasky.net/docs/?python#introduction). 22 | 23 | ## Contributing 24 | 25 | ### Requirements 26 | 27 | In order to run lints and tests locally you will need to: 28 | 29 | 1. Check out the repository locally. 30 | 2. Make sure you have `make` installed (on Windows you can use [Chocolatey](https://chocolatey.org/) and run `choco install make`). 31 | 32 | ### Instructions 33 | 34 | To run lints and tests on `python-skynet`, first install dependencies: 35 | 36 | ``` 37 | make install 38 | ``` 39 | 40 | Now you can run 41 | 42 | ``` 43 | pipenv run lint 44 | ``` 45 | 46 | or 47 | 48 | ``` 49 | pipenv run test 50 | ``` 51 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="siaskynet", 8 | version="2.2.0", 9 | author="Peter-Jan Brone", 10 | author_email="peterjan.brone@gmail.com", 11 | description="Skynet SDK", 12 | url="https://github.com/NebulousLabs/python-skynet", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | packages=setuptools.find_packages(), 16 | install_requires=[ 17 | 'requests', 18 | 'responses', 19 | ], 20 | classifiers=[ 21 | "Programming Language :: Python :: 3", 22 | "License :: OSI Approved :: MIT License", 23 | "Operating System :: OS Independent", 24 | ], 25 | python_requires='>=3.6', 26 | ) 27 | -------------------------------------------------------------------------------- /siaskynet/__init__.py: -------------------------------------------------------------------------------- 1 | """An SDK for integrating Skynet into Python applications. 2 | """ 3 | 4 | 5 | import requests 6 | 7 | from . import utils 8 | from .utils import default_portal_url, uri_skynet_prefix 9 | 10 | 11 | # pylint: disable=too-few-public-methods 12 | class SkynetClient(): 13 | """The Skynet Client which can be used to access Skynet.""" 14 | 15 | # Imported methods 16 | 17 | # pylint: disable=import-outside-toplevel 18 | from ._download import ( 19 | download_file, download_file_request, get_metadata, 20 | get_metadata_request 21 | ) 22 | from ._encryption import ( 23 | add_skykey, create_skykey, get_skykey_by_id, get_skykey_by_name, 24 | get_skykeys 25 | ) 26 | from ._upload import ( 27 | upload, upload_request, 28 | upload_file, upload_file_request, 29 | upload_file_with_chunks, upload_file_request_with_chunks, 30 | upload_directory, upload_directory_request 31 | ) 32 | # pylint: enable=import-outside-toplevel 33 | 34 | def __init__(self, portal_url="", custom_opts=None): 35 | if portal_url == "": 36 | portal_url = utils.default_portal_url() 37 | self.portal_url = portal_url 38 | if custom_opts is None: 39 | custom_opts = {} 40 | self.custom_opts = custom_opts 41 | 42 | def execute_request(self, method, opts, **kwargs): 43 | """Makes and executes a request with the given options.""" 44 | 45 | url = utils.make_url( 46 | self.portal_url, 47 | opts["endpoint_path"], 48 | opts.get("extra_path", "") 49 | ) 50 | 51 | if opts["api_key"] is not None: 52 | kwargs["auth"] = ("", opts["api_key"]) 53 | 54 | if opts["custom_user_agent"] is not None: 55 | headers = kwargs.get("headers", {}) 56 | headers["User-Agent"] = opts["custom_user_agent"] 57 | kwargs["headers"] = headers 58 | 59 | if opts["timeout_seconds"] is not None: 60 | kwargs["timeout"] = opts["timeout_seconds"] 61 | 62 | try: 63 | return requests.request(method, url, **kwargs) 64 | except requests.exceptions.Timeout as err: 65 | raise TimeoutError("Request timed out") from err 66 | 67 | # pylint: enable=too-few-public-methods 68 | -------------------------------------------------------------------------------- /siaskynet/_download.py: -------------------------------------------------------------------------------- 1 | """Skynet download API. 2 | """ 3 | 4 | import json 5 | import os 6 | 7 | from . import utils 8 | 9 | 10 | def default_download_options(): 11 | """Returns the default download options.""" 12 | 13 | obj = utils.default_options("/") 14 | 15 | return obj 16 | 17 | 18 | def download_file(self, path, skylink, custom_opts=None): 19 | """Downloads file to path from given skylink with the given options.""" 20 | 21 | path = os.path.normpath(path) 22 | response = self.download_file_request(skylink, custom_opts) 23 | open(path, 'wb').write(response.content) 24 | response.close() 25 | 26 | 27 | def download_file_request(self, skylink, custom_opts=None, stream=False): 28 | """Posts request to download file.""" 29 | 30 | opts = default_download_options() 31 | opts.update(self.custom_opts) 32 | if custom_opts is not None: 33 | opts.update(custom_opts) 34 | 35 | skylink = utils.strip_prefix(skylink) 36 | opts["extra_path"] = skylink 37 | 38 | return self.execute_request( 39 | "GET", 40 | opts, 41 | allow_redirects=True, 42 | stream=stream, 43 | ) 44 | 45 | 46 | def get_metadata(self, skylink, custom_opts=None): 47 | """Downloads metadata from given skylink.""" 48 | 49 | response = self.get_metadata_request(skylink, custom_opts) 50 | return json.loads(response.headers["skynet-file-metadata"]) 51 | 52 | 53 | def get_metadata_request(self, skylink, custom_opts=None, stream=False): 54 | """Posts request to get metadata from given skylink.""" 55 | 56 | opts = default_download_options() 57 | opts.update(self.custom_opts) 58 | if custom_opts is not None: 59 | opts.update(custom_opts) 60 | 61 | skylink = utils.strip_prefix(skylink) 62 | 63 | return self.execute_request( 64 | "HEAD", 65 | opts, 66 | allow_redirects=True, 67 | stream=stream, 68 | ) 69 | -------------------------------------------------------------------------------- /siaskynet/_encryption.py: -------------------------------------------------------------------------------- 1 | """Skynet encryption API. 2 | """ 3 | 4 | 5 | from . import utils 6 | 7 | 8 | def default_add_skykey_options(): 9 | """Returns the default addskykey options.""" 10 | 11 | obj = utils.default_options("/skynet/addskykey") 12 | 13 | return obj 14 | 15 | 16 | def default_create_skykey_options(): 17 | """Returns the default createskykey options.""" 18 | 19 | obj = utils.default_options("/skynet/createskykey") 20 | 21 | return obj 22 | 23 | 24 | def default_get_skykey_options(): 25 | """Returns the default getskykey options.""" 26 | 27 | obj = utils.default_options("/skynet/skykey") 28 | 29 | return obj 30 | 31 | 32 | def default_get_skykeys_options(): 33 | """Returns the default getskykeys options.""" 34 | 35 | obj = utils.default_options("/skynet/skykeys") 36 | 37 | return obj 38 | 39 | 40 | def add_skykey(self, skykey, custom_opts=None): 41 | """Stores the given base-64 encoded skykey with the skykey manager.""" 42 | 43 | raise NotImplementedError 44 | 45 | 46 | def create_skykey(self, skykey_name, skykey_type, custom_opts=None): 47 | """Returns a new skykey created and stored under the given name with \ 48 | the given type. skykeyType can be either "public-id" or \ 49 | "private-id".""" 50 | 51 | raise NotImplementedError 52 | 53 | 54 | def get_skykey_by_name(self, skykey_name, custom_opts=None): 55 | """Returns the given skykey by name.""" 56 | 57 | raise NotImplementedError 58 | 59 | 60 | def get_skykey_by_id(self, skykey_id, custom_opts=None): 61 | """Returns the given skykey by id.""" 62 | 63 | raise NotImplementedError 64 | 65 | 66 | def get_skykeys(self, custom_opts=None): 67 | """Returns a get of all skykeys.""" 68 | 69 | raise NotImplementedError 70 | -------------------------------------------------------------------------------- /siaskynet/_upload.py: -------------------------------------------------------------------------------- 1 | """Skynet upload API. 2 | """ 3 | 4 | import os 5 | import platform 6 | 7 | from . import utils 8 | 9 | 10 | def default_upload_options(): 11 | """Returns the default upload options.""" 12 | 13 | obj = utils.default_options("/skynet/skyfile") 14 | obj['portal_file_fieldname'] = 'file' 15 | obj['portal_directory_file_fieldname'] = 'files[]' 16 | obj['custom_filename'] = '' 17 | obj['custom_dirname'] = '' 18 | 19 | return obj 20 | 21 | 22 | def upload(self, upload_data, custom_opts=None): 23 | """Uploads the given generic data and returns the skylink.""" 24 | 25 | response = self.upload_request(upload_data, custom_opts) 26 | sia_url = utils.uri_skynet_prefix() + response.json()["skylink"] 27 | response.close() 28 | return sia_url 29 | 30 | 31 | def upload_request(self, upload_data, custom_opts=None): 32 | """Uploads the given generic data and returns the response object.""" 33 | 34 | opts = default_upload_options() 35 | opts.update(self.custom_opts) 36 | if custom_opts is not None: 37 | opts.update(custom_opts) 38 | 39 | # Upload as a directory if the dirname is set, even if there is only 1 40 | # file. 41 | issinglefile = len(upload_data) == 1 and not opts['custom_dirname'] 42 | 43 | filename = '' 44 | if issinglefile: 45 | fieldname = opts['portal_file_fieldname'] 46 | else: 47 | if not opts['custom_dirname']: 48 | raise ValueError("custom_dirname must be set when " 49 | "uploading multiple files") 50 | fieldname = opts['portal_directory_file_fieldname'] 51 | filename = opts['custom_dirname'] 52 | 53 | params = { 54 | # 'skykeyname': opts['skykey_name'], 55 | # 'skykeyid': opts['skyket_id'], 56 | } 57 | if filename: 58 | params['filename'] = filename 59 | 60 | ftuples = [] 61 | for filename, data in upload_data.items(): 62 | ftuples.append((fieldname, 63 | (filename, data))) 64 | 65 | if issinglefile: 66 | filename, data = ftuples[0][1] 67 | params['filename'] = filename 68 | return self.execute_request( 69 | "POST", 70 | opts, 71 | data=data, 72 | headers={'Content-Type': 'application/octet-stream'}, 73 | params=params 74 | ) 75 | return self.execute_request( 76 | "POST", 77 | opts, 78 | files=ftuples, 79 | params=params 80 | ) 81 | 82 | 83 | def upload_file(self, path, custom_opts=None): 84 | """Uploads file at path with the given options.""" 85 | 86 | response = self.upload_file_request(path, custom_opts) 87 | sia_url = utils.uri_skynet_prefix() + response.json()["skylink"] 88 | response.close() 89 | return sia_url 90 | 91 | 92 | def upload_file_request(self, path, custom_opts=None): 93 | """ 94 | Posts request to upload file. 95 | 96 | :param str path: The local path of the file to upload. 97 | :param dict custom_opts: Custom options. See upload_file. 98 | :return: the full response 99 | :rtype: dict 100 | """ 101 | 102 | opts = default_upload_options() 103 | opts.update(self.custom_opts) 104 | if custom_opts is not None: 105 | opts.update(custom_opts) 106 | 107 | path = os.path.normpath(path) 108 | if not os.path.isfile(path): 109 | print("Given path is not a file") 110 | return None 111 | 112 | with open(path, 'rb') as file_h: 113 | filename = os.path.basename(file_h.name) 114 | if opts['custom_filename']: 115 | filename = opts['custom_filename'] 116 | 117 | upload_data = {filename: file_h} 118 | 119 | return self.upload_request(upload_data, opts) 120 | 121 | 122 | def upload_file_with_chunks(self, chunks, custom_opts=None): 123 | """ 124 | Uploads a chunked or streaming file with the given options. 125 | For more information on chunked uploading, see: 126 | https://requests.readthedocs.io/en/stable/user/advanced/#chunk-encoded-requests 127 | 128 | :param iter data: An iterator (for chunked encoding) or file-like object 129 | :param dict custom_opts: Custom options. See upload_file. 130 | """ 131 | 132 | response = self.upload_file_request_with_chunks(chunks, custom_opts) 133 | sia_url = utils.uri_skynet_prefix() + response.json()["skylink"] 134 | response.close() 135 | return sia_url 136 | 137 | 138 | def upload_file_request_with_chunks(self, chunks, custom_opts=None): 139 | """ 140 | Posts request for chunked or streaming upload of a single file. 141 | For more information on chunked uploading, see: 142 | https://requests.readthedocs.io/en/stable/user/advanced/#chunk-encoded-requests 143 | 144 | :param iter chunks: An iterator (for chunked encoding) or file-like object 145 | :param dict custom_opts: Custom options. See upload_file. 146 | :return: the full response 147 | :rtype: dict 148 | """ 149 | 150 | opts = default_upload_options() 151 | opts.update(self.custom_opts) 152 | if custom_opts is not None: 153 | opts.update(custom_opts) 154 | 155 | if opts['custom_filename']: 156 | filename = opts['custom_filename'] 157 | else: 158 | # this is the legacy behavior 159 | filename = str(chunks) 160 | 161 | upload_data = {filename: chunks} 162 | 163 | return self.upload_request(upload_data, opts) 164 | 165 | 166 | def upload_directory(self, path, custom_opts=None): 167 | """Uploads directory at path with the given options.""" 168 | 169 | response = self.upload_directory_request(path, custom_opts) 170 | sia_url = utils.uri_skynet_prefix() + response.json()["skylink"] 171 | response.close() 172 | return sia_url 173 | 174 | 175 | def upload_directory_request(self, path, custom_opts=None): 176 | """Posts request to upload directory.""" 177 | 178 | opts = default_upload_options() 179 | opts.update(self.custom_opts) 180 | if custom_opts is not None: 181 | opts.update(custom_opts) 182 | 183 | path = os.path.normpath(path) 184 | if not os.path.isdir(path): 185 | print("Given path is not a directory") 186 | return None 187 | 188 | upload_data = {} 189 | if platform.system() == "Windows": 190 | path = path.replace("\\", '/') 191 | basepath = path if path == '/' else path + '/' 192 | for filepath in utils.walk_directory(path): 193 | assert filepath.startswith(basepath) 194 | upload_data[filepath[len(basepath):]] = open(filepath, 'rb') 195 | 196 | if not opts['custom_dirname']: 197 | opts['custom_dirname'] = path 198 | 199 | return self.upload_request(upload_data, opts) 200 | -------------------------------------------------------------------------------- /siaskynet/test_utils.py: -------------------------------------------------------------------------------- 1 | """Skynet SDK tests.""" 2 | 3 | import os 4 | 5 | from . import utils 6 | 7 | 8 | PORTAL_URL = utils.default_portal_url() 9 | SKYLINK = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg" 10 | 11 | 12 | def test_make_url(): 13 | """Test make_url.""" 14 | 15 | assert utils.make_url(PORTAL_URL, "/") == PORTAL_URL+"/" 16 | assert utils.make_url(PORTAL_URL, "/skynet") == PORTAL_URL+"/skynet" 17 | assert utils.make_url(PORTAL_URL, "/skynet/") == PORTAL_URL+"/skynet/" 18 | 19 | assert utils.make_url(PORTAL_URL, "/", SKYLINK) == PORTAL_URL+"/"+SKYLINK 20 | assert utils.make_url(PORTAL_URL, "/skynet", SKYLINK) == \ 21 | PORTAL_URL+"/skynet/"+SKYLINK 22 | assert utils.make_url(PORTAL_URL, "//skynet/", SKYLINK) == \ 23 | PORTAL_URL+"/skynet/"+SKYLINK 24 | 25 | 26 | def test_walk_directory(): 27 | """Test walk_directory.""" 28 | 29 | path = "./testdata/" 30 | 31 | # Quick test that normalizing removes the final slash. 32 | assert os.path.normpath(path) == "testdata" 33 | 34 | files = utils.walk_directory(path) 35 | expected_files = [ 36 | "testdata/file1", 37 | "testdata/dir1/file2", 38 | "testdata/file3" 39 | ] 40 | assert len(expected_files) == len(files) 41 | for expected_file in expected_files: 42 | assert expected_file in files 43 | -------------------------------------------------------------------------------- /siaskynet/utils.py: -------------------------------------------------------------------------------- 1 | """Skynet utility functions. 2 | """ 3 | 4 | import os 5 | import platform 6 | 7 | 8 | def default_options(endpoint_path): 9 | """Returns the default options with the given endpoint path.""" 10 | 11 | return { 12 | 'endpoint_path': endpoint_path, 13 | 14 | 'api_key': None, 15 | 'custom_user_agent': None, 16 | "timeout_seconds": None, 17 | } 18 | 19 | 20 | def default_portal_url(): 21 | """DefaultPortalURL intelligently selects a default portal.""" 22 | 23 | # TODO: This will be smarter. See 24 | # https://github.com/NebulousLabs/skynet-docs/issues/21. 25 | 26 | return default_skynet_portal_url() 27 | 28 | 29 | def default_skynet_portal_url(): 30 | """Returns the default Skynet portal URL.""" 31 | 32 | return 'https://siasky.net' 33 | 34 | 35 | def make_url(portal_url, *arg): 36 | """Makes a URL from the given portal url and path elements.""" 37 | 38 | url = portal_url 39 | for path_element in arg: 40 | if path_element == "": 41 | continue 42 | while url.endswith("/"): 43 | url = url[:-1] 44 | while path_element.startswith("/"): 45 | path_element = path_element[1:] 46 | url = url+"/"+path_element 47 | 48 | return url 49 | 50 | 51 | def strip_prefix(string): 52 | """Strips Skynet prefix from input.""" 53 | 54 | if string.startswith(uri_skynet_prefix()): 55 | return string[len(uri_skynet_prefix()):] 56 | return string 57 | 58 | 59 | def uri_skynet_prefix(): 60 | """Returns the Skynet URI prefix.""" 61 | 62 | return "sia://" 63 | 64 | 65 | def walk_directory(path): 66 | """Walks given directory returning all files recursively.""" 67 | 68 | path = os.path.normpath(path) 69 | 70 | files = {} 71 | for root, subdirs, subfiles in os.walk(path): 72 | for subdir in subdirs: 73 | subdir = os.path.join(root, subdir) 74 | files.update(walk_directory(subdir)) 75 | for subfile in subfiles: 76 | fullpath = os.path.join(root, subfile) 77 | if platform.system() == "Windows": 78 | fullpath = fullpath.replace("\\", "/") 79 | files[fullpath] = True 80 | return files 81 | -------------------------------------------------------------------------------- /testdata/dir1/file2: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /testdata/file1: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /testdata/file3: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Integration tests for the Skynet API. 2 | """ 3 | -------------------------------------------------------------------------------- /tests/test_integration_download.py: -------------------------------------------------------------------------------- 1 | """Download API integration tests.""" 2 | 3 | import filecmp 4 | import os 5 | import platform 6 | import sys 7 | import tempfile 8 | 9 | import responses 10 | 11 | import siaskynet as skynet 12 | 13 | 14 | SKYLINK = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg" 15 | 16 | client = skynet.SkynetClient() 17 | 18 | 19 | @responses.activate 20 | def test_download_file(): 21 | """Test downloading a file to a temporary location.""" 22 | 23 | # This test fails on CI for Windows so skip it. 24 | if platform.system() == "Windows" and 'CI' in os.environ: 25 | return 26 | 27 | src_file = "./testdata/file1" 28 | 29 | # download a file 30 | 31 | responses.add( 32 | responses.GET, 33 | 'https://siasky.net/'+SKYLINK, 34 | "test\n", 35 | status=200 36 | ) 37 | 38 | dst_file = tempfile.NamedTemporaryFile().name 39 | print("Downloading to "+dst_file) 40 | client.download_file(dst_file, SKYLINK) 41 | if not filecmp.cmp(src_file, dst_file): 42 | sys.exit("ERROR: Downloaded file at "+dst_file + 43 | " did not equal uploaded file "+src_file) 44 | 45 | print("File download successful") 46 | 47 | assert len(responses.calls) == 1 48 | -------------------------------------------------------------------------------- /tests/test_integration_upload.py: -------------------------------------------------------------------------------- 1 | """Upload API integration tests.""" 2 | 3 | import sys 4 | 5 | import responses 6 | 7 | import siaskynet as skynet 8 | 9 | 10 | SKYLINK = "XABvi7JtJbQSMAcDwnUnmp2FKDPjg8_tTTFP4BwMSxVdEg" 11 | SIALINK = skynet.uri_skynet_prefix() + SKYLINK 12 | 13 | client = skynet.SkynetClient() 14 | 15 | 16 | def response_callback(request): 17 | """Called by responses for HTTP requests. 18 | This function can perform any processing of 19 | requests needed for tests. 20 | """ 21 | 22 | # process body content 23 | if hasattr(request.body, 'read'): 24 | # upload is a file object 25 | # read the file and store its content for test to compare 26 | request.body = request.body.read() 27 | elif not isinstance(request.body, (str, bytes)): 28 | # upload is a chunked iterator 29 | # convert it into the iterated content 30 | chunks = [*request.body] 31 | if len(chunks) > 0 and isinstance(chunks[0], str): 32 | request.body = ''.join(chunks) 33 | else: 34 | request.body = b''.join(chunks) 35 | 36 | # return the status code and headers for 37 | # the responses module to provide 38 | return ( 39 | 200, 40 | {'Content-Type': 'application/json'}, 41 | '{"skylink": "' + SKYLINK + '"}' 42 | ) 43 | 44 | 45 | @responses.activate 46 | def test_upload_file(): 47 | """Test uploading a file.""" 48 | 49 | src_file = "./testdata/file1" 50 | 51 | # upload a file 52 | 53 | responses.add_callback( 54 | responses.POST, 55 | 'https://siasky.net/skynet/skyfile', 56 | callback=response_callback 57 | ) 58 | 59 | print("Uploading file "+src_file) 60 | sialink2 = client.upload_file(src_file) 61 | if SIALINK != sialink2: 62 | sys.exit("ERROR: expected returned sialink "+SIALINK + 63 | ", received "+sialink2) 64 | print("File upload successful, sialink: " + sialink2) 65 | 66 | headers = responses.calls[0].request.headers 67 | assert headers["Content-Type"] 68 | assert headers["User-Agent"].startswith("python-requests") 69 | assert "Authorization" not in headers 70 | 71 | params = responses.calls[0].request.params 72 | assert params["filename"] == "file1" 73 | 74 | body = responses.calls[0].request.body 75 | with open(src_file, 'rb') as file_h: 76 | contents = file_h.read() 77 | assert contents == body 78 | 79 | assert len(responses.calls) == 1 80 | 81 | 82 | @responses.activate 83 | def test_upload_file_custom_filename(): 84 | """Test uploading a file with a custom filename.""" 85 | 86 | src_file = './testdata/file1' 87 | custom_name = 'testname' 88 | 89 | # upload a file with custom filename 90 | 91 | responses.add_callback( 92 | responses.POST, 93 | 'https://siasky.net/skynet/skyfile', 94 | callback=response_callback 95 | ) 96 | 97 | print("Uploading file "+src_file) 98 | sialink2 = client.upload_file(src_file, {'custom_filename': custom_name}) 99 | if SIALINK != sialink2: 100 | sys.exit("ERROR: expected returned sialink "+SIALINK + 101 | ", received "+sialink2) 102 | print("File upload successful, sialink: " + sialink2) 103 | 104 | body = responses.calls[0].request.body 105 | with open(src_file, 'rb') as file_h: 106 | contents = file_h.read() 107 | assert body == contents 108 | 109 | params = responses.calls[0].request.params 110 | assert params["filename"] == custom_name 111 | 112 | assert len(responses.calls) == 1 113 | 114 | 115 | @responses.activate 116 | def test_upload_file_api_key(): 117 | """Test uploading a file with authorization.""" 118 | 119 | src_file = "./testdata/file1" 120 | 121 | # Upload a file with an API password set. 122 | 123 | responses.add_callback( 124 | responses.POST, 125 | "https://siasky.net/skynet/skyfile", 126 | callback=response_callback 127 | ) 128 | 129 | print("Uploading file "+src_file) 130 | sialink2 = client.upload_file(src_file, {"api_key": "foobar"}) 131 | if SIALINK != sialink2: 132 | sys.exit("ERROR: expected returned sialink "+SIALINK + 133 | ", received "+sialink2) 134 | print("File upload successful, sialink: " + sialink2) 135 | 136 | headers = responses.calls[0].request.headers 137 | assert headers["Authorization"] == "Basic OmZvb2Jhcg==" 138 | 139 | assert len(responses.calls) == 1 140 | 141 | 142 | @responses.activate 143 | def test_upload_file_custom_user_agent(): 144 | """Test uploading a file with authorization.""" 145 | 146 | src_file = "./testdata/file1" 147 | client2 = skynet.SkynetClient( 148 | "https://testportal.net", 149 | {"custom_user_agent": "Sia-Agent"} 150 | ) 151 | 152 | # Upload a file using the client's user agent. 153 | 154 | responses.add_callback( 155 | responses.POST, 156 | "https://testportal.net/skynet/skyfile", 157 | callback=response_callback 158 | ) 159 | 160 | print("Uploading file "+src_file) 161 | sialink2 = client2.upload_file(src_file) 162 | if SIALINK != sialink2: 163 | sys.exit("ERROR: expected returned sialink "+SIALINK + 164 | ", received "+sialink2) 165 | print("File upload successful, sialink: " + sialink2) 166 | 167 | headers = responses.calls[0].request.headers 168 | assert headers["User-Agent"] == "Sia-Agent" 169 | 170 | # Upload a file with a new user agent set. 171 | 172 | responses.add_callback( 173 | responses.POST, 174 | "https://testportal.net/skynet/skyfile", 175 | callback=response_callback 176 | ) 177 | 178 | print("Uploading file "+src_file) 179 | sialink2 = client2.upload_file( 180 | src_file, 181 | {"custom_user_agent": "Sia-Agent-2"} 182 | ) 183 | if SIALINK != sialink2: 184 | sys.exit("ERROR: expected returned sialink "+SIALINK + 185 | ", received "+sialink2) 186 | print("File upload successful, sialink: " + sialink2) 187 | 188 | headers = responses.calls[1].request.headers 189 | assert headers["User-Agent"] == "Sia-Agent-2" 190 | 191 | assert len(responses.calls) == 2 192 | 193 | 194 | @responses.activate 195 | def test_upload_directory(): 196 | """Test uploading a directory.""" 197 | 198 | src_dir = './testdata' 199 | 200 | # upload a directory 201 | 202 | responses.add_callback( 203 | responses.POST, 204 | 'https://siasky.net/skynet/skyfile', 205 | callback=response_callback 206 | ) 207 | 208 | print('Uploading dir '+src_dir) 209 | sialink2 = client.upload_directory(src_dir) 210 | if SIALINK != sialink2: 211 | sys.exit('ERROR: expected returned sialink '+SIALINK + 212 | ', received '+sialink2) 213 | print('Dir upload successful, sialink: ' + sialink2) 214 | 215 | headers = responses.calls[0].request.headers 216 | assert headers["Content-Type"].startswith("multipart/form-data;") 217 | 218 | params = responses.calls[0].request.params 219 | assert params["filename"] == "testdata" 220 | 221 | body = str(responses.calls[0].request.body) 222 | print(body) 223 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 224 | filename="file1"') != -1 225 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 226 | filename="file3"') != -1 227 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 228 | filename="dir1/file2"') != -1 229 | # Check a file that shouldn't be there. 230 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 231 | filename="file0"') == -1 232 | 233 | assert len(responses.calls) == 1 234 | 235 | 236 | @responses.activate 237 | def test_upload_directory_custom_dirname(): 238 | """Test uploading a directory with a custom dirname.""" 239 | 240 | src_dir = './testdata' 241 | custom_dirname = "testdir" 242 | 243 | # upload a directory 244 | 245 | responses.add_callback( 246 | responses.POST, 247 | 'https://siasky.net/skynet/skyfile?filename=testdir', 248 | match_querystring=True, 249 | callback=response_callback 250 | ) 251 | 252 | print('Uploading dir '+src_dir) 253 | sialink2 = client.upload_directory( 254 | src_dir, 255 | {"custom_dirname": custom_dirname} 256 | ) 257 | if SIALINK != sialink2: 258 | sys.exit('ERROR: expected returned sialink '+SIALINK + 259 | ', received '+sialink2) 260 | print('Dir upload successful, sialink: ' + sialink2) 261 | 262 | headers = responses.calls[0].request.headers 263 | assert headers["Content-Type"].startswith("multipart/form-data;") 264 | 265 | params = responses.calls[0].request.params 266 | assert params["filename"] == custom_dirname 267 | 268 | body = str(responses.calls[0].request.body) 269 | print(body) 270 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 271 | filename="file1"') != -1 272 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 273 | filename="file3"') != -1 274 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 275 | filename="dir1/file2"') != -1 276 | # Check a file that shouldn't be there. 277 | assert body.find('Content-Disposition: form-data; name="files[]"; \ 278 | filename="file0"') == -1 279 | 280 | assert len(responses.calls) == 1 281 | 282 | 283 | @responses.activate 284 | def test_upload_file_chunks(): 285 | """Test uploading a file with chunks.""" 286 | 287 | src_file = "./testdata/file1" 288 | 289 | # upload a file 290 | 291 | responses.add_callback( 292 | responses.POST, 293 | 'https://siasky.net/skynet/skyfile', 294 | callback=response_callback 295 | ) 296 | 297 | print("Uploading file "+src_file) 298 | 299 | def chunker(filename): 300 | with open(filename, 'rb') as file: 301 | while True: 302 | data = file.read(3) 303 | if not data: 304 | break 305 | yield data 306 | chunks = chunker(src_file) 307 | sialink2 = client.upload_file_with_chunks(chunks, 308 | {'custom_filename': src_file}) 309 | if SIALINK != sialink2: 310 | sys.exit("ERROR: expected returned sialink "+SIALINK + 311 | ", received "+sialink2) 312 | print("File upload successful, sialink: " + sialink2) 313 | 314 | headers = responses.calls[0].request.headers 315 | assert headers["Content-Type"] 316 | assert headers["Transfer-Encoding"] == "chunked" 317 | assert headers["User-Agent"].startswith("python-requests") 318 | assert "Authorization" not in headers 319 | 320 | params = responses.calls[0].request.params 321 | assert params["filename"] == src_file 322 | 323 | body = responses.calls[0].request.body 324 | with open(src_file, 'rb') as file_h: 325 | contents = file_h.read() 326 | assert contents == body 327 | 328 | assert len(responses.calls) == 1 329 | --------------------------------------------------------------------------------