├── .gitignore ├── CHANGELOG ├── COPYING ├── MANIFEST.in ├── README.md ├── etc ├── haproxy │ └── haproxy.cfg ├── init.d │ ├── swftp-ftp │ └── swftp-sftp ├── init │ ├── swftp-ftp.conf │ └── swftp-sftp.conf ├── supervisor │ └── conf.d │ │ └── swftp.conf └── swftp │ ├── swftp.conf.sample │ └── test.conf.sample ├── requirements.txt ├── setup.py ├── swftp ├── __init__.py ├── auth.py ├── ftp │ ├── __init__.py │ ├── server.py │ └── service.py ├── logging.py ├── realm.py ├── report.py ├── sftp │ ├── __init__.py │ ├── server.py │ ├── service.py │ ├── swiftdirectory.py │ └── swiftfile.py ├── statsd.py ├── swift.py ├── swiftfilesystem.py ├── test │ ├── __init__.py │ ├── functional │ │ ├── __init__.py │ │ ├── test_ftp.py │ │ └── test_sftp.py │ ├── test-ftp.conf │ ├── test-sftp.conf │ ├── test_id_rsa │ ├── test_id_rsa.pub │ └── unit │ │ ├── __init__.py │ │ ├── test_auth.py │ │ ├── test_ftp.py │ │ ├── test_sftp.py │ │ ├── test_swift.py │ │ └── test_utils.py └── utils.py └── twisted └── plugins ├── swftp_ftp.py └── swftp_sftp.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | *.tmp 4 | .DS_Store 5 | *.egg-info 6 | dist 7 | build 8 | *.cache 9 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | swftp (1.0.6) 2 | 3 | * Adds Optional Stats Web Interface (JSON Output) 4 | 5 | * Less memory usage for SFTP uploads 6 | 7 | * Several Bug Fixes 8 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Unless otherwise noted, all files are released under the MIT license, 2 | exceptions contain licensing information in them. 3 | 4 | Copyright (C) 2014 SoftLayer Technologies, Inc. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | 24 | Except as contained in this notice, the name of SoftLayer Technologies, Inc. shall not 25 | be used in advertising or otherwise to promote the sale, use or other dealings 26 | in this Software without prior written authorization from SoftLayer Technologies, Inc. 27 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include COPYING 2 | include README.md 3 | graft etc 4 | graft twisted -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | SwFTP 2 | ===== 3 | SwFTP is an FTP and SFTP interface for Openstack Object Storage (swift). It will act as a proxy between the FTP/SFTP protocols and OpenStack Object Storage. 4 | 5 | Features 6 | -------- 7 | * Configurable auth endpoint to use any OpenStack Swift installation 8 | * Server-wide Configurable HTTP Connection Pool for Swift Communications (size and timeout) 9 | * Support for HTTPS communication to the backend OpenStack Object Storage cluster 10 | * Simple Installation `pip install swftp` 11 | * StatsD Support 12 | * Stats Web Interface 13 | * Chef Cookbook: https://github.com/softlayer/chef-swftp 14 | 15 | Requirements 16 | ------------ 17 | * Python 2.6-2.7 18 | * OpenSSL/pycrypto 19 | * Twisted/Twisted-Conch 20 | * pyasn1 21 | 22 | Getting Started 23 | --------------- 24 | ### Installing 25 | Install via pip: 26 | ```bash 27 | $ pip install swftp 28 | ``` 29 | Note: If you don't have pip [here's how to install it](http://www.pip-installer.org/en/latest/installing.html). 30 | 31 | Install using git/pip: 32 | ```bash 33 | $ pip install -U git+git://github.com/softlayer/swftp.git 34 | ``` 35 | 36 | Install from source: 37 | ```bash 38 | $ python setup.py install 39 | ``` 40 | 41 | ### Start FTP Server 42 | To run the FTP server, simply run this command. 43 | ```bash 44 | $ swftp-ftp -a http://127.0.0.1:8080/auth/v1.0 45 | 2013-02-18 16:28:50-0600 [-] Log opened. 46 | 2013-02-18 16:28:50-0600 [-] FTPFactory starting on 5021 47 | 2013-02-18 16:28:50-0600 [-] Starting factory 48 | ``` 49 | 50 | ### Start SFTP Server 51 | The SFTP requires a bit of setup the first time. 52 | 53 | 54 | You'll need to create a public/private key pair for SSH and move them to the /etc/swftp directory (this location is configurable). 55 | ```bash 56 | $ mkdir /etc/swftp 57 | $ ssh-keygen -h -b 2048 -N "" -t rsa -f /etc/swftp/id_rsa 58 | ``` 59 | 60 | After placing the required files, the command to start the server is similar to the FTP one. 61 | ```bash 62 | $ swftp-sftp -a http://127.0.0.1:8080/auth/v1.0 63 | 2013-02-18 16:29:14-0600 [-] Log opened. 64 | 2013-02-18 22:29:14+0000 [-] SSHFactory starting on 5022 65 | ``` 66 | 67 | Configuration 68 | ------------- 69 | ### Command Line 70 | The command line configuration allows you to speficfy a custom OpenStack Swift Auth URL, as well as the location of the config file (detailed later). 71 | 72 | FTP Command-line options: 73 | ```bash 74 | $ swftp-ftp --help 75 | Usage: swftp-ftp [options] 76 | Options: 77 | -c, --config_file= Location of the swftp config file. [default: 78 | /etc/swftp/swftp.conf] 79 | -a, --auth_url= Auth Url to use. Defaults to the config file value if it 80 | exists. [default: http://127.0.0.1:8080/auth/v1.0] 81 | -p, --port= Port to bind to. 82 | -h, --host= IP to bind to. 83 | --version Display Twisted version and exit. 84 | --help Display this help and exit. 85 | ``` 86 | 87 | SFTP Command-line options: 88 | ```bash 89 | $ swftp-sftp --help 90 | Usage: swftp-sftp [options] 91 | Options: 92 | -c, --config_file= Location of the swftp config file. [default: 93 | /etc/swftp/swftp.conf] 94 | -a, --auth_url= Auth Url to use. Defaults to the config file value if it 95 | exists.[default: http://127.0.0.1:8080/auth/v1.0] 96 | -p, --port= Port to bind to. 97 | -h, --host= IP to bind to. 98 | --priv_key= Private Key Location. 99 | --pub_key= Public Key Location. 100 | --version Display Twisted version and exit. 101 | --help Display this help and exit. 102 | ``` 103 | 104 | ### Config File 105 | The default location for the config file is /etc/swftp/swftp.conf. 106 | 107 | Here is an example swftp.conf with all defaults: 108 | ``` 109 | [sftp] 110 | host = 0.0.0.0 111 | port = 5022 112 | priv_key = /etc/swftp/id_rsa 113 | pub_key = /etc/swftp/id_rsa.pub 114 | connection_timeout = 240 115 | 116 | auth_url = http://127.0.0.1:8080/auth/v1.0 117 | num_persistent_connections = 20 118 | num_connections_per_session = 10 119 | rewrite_storage_scheme = 120 | rewrite_storage_netloc = 121 | extra_headers = X-Swftp: true, X-Forwarded-Proto: SFTP 122 | 123 | log_statsd_host = 124 | log_statsd_port = 8125 125 | log_statsd_sample_rate = 10 126 | log_statsd_metric_prefix = sftp 127 | 128 | stats_host = 129 | stats_port = 38022 130 | 131 | [ftp] 132 | host = 0.0.0.0 133 | port = 5021 134 | sessions_per_user = 10 135 | connection_timeout = 240 136 | welcome_message = Welcome to SwFTP - An FTP/SFTP interface for Openstack Swift 137 | 138 | auth_url = http://127.0.0.1:8080/auth/v1.0 139 | num_persistent_connections = 20 140 | num_connections_per_session = 10 141 | rewrite_storage_scheme = 142 | rewrite_storage_netloc = 143 | extra_headers = X-Swftp: true, X-Forwarded-Proto: SFTP 144 | 145 | log_statsd_host = 146 | log_statsd_port = 8125 147 | log_statsd_sample_rate = 10 148 | log_statsd_metric_prefix = ftp 149 | 150 | stats_host = 151 | stats_port = 38021 152 | ``` 153 | 154 | **Server Options** 155 | 156 | * **host** - Address that the FTP/SFTP server will listen on. 157 | * **port** - Port that the FTP/SFTP server will listen on. 158 | * **sessions_per_user** - Number of FTP/SFTP sessions per unique swift username to allow. 159 | * **priv_key** - (SFTP Only) - File path to the private SSH key that the SFTP server will use. 160 | * **pub_key** - (SFTP Only) - File path to the public SSH key generated from the private key. 161 | * **session_timeout** - (FTP Only) - Session timeout in seconds. Idle sessions will be closed after this much time. 162 | * **welcome_message** - (FTP Only) - Custom FTP welcome message. 163 | 164 | **Swift Options** 165 | 166 | * **auth_url** - Auth URL to use to authenticate with the backend swift cluster. 167 | * **num_persistent_connections** - Number of persistent connections to the backend swift cluster for an entire swftp instance. 168 | * **num_connections_per_session** - Number of persistent connections to the backend swift cluster per FTP/SFTP session. 169 | * **connection_timeout** - Connection timeout in seconds to the backend swift cluster. 170 | * **extra_headers** - Extra HTTP headers that are sent to swift cluster. 171 | * e.g.: extra_headers = X-Swftp: true, X-Forwarded-Proto: SFTP 172 | * **rewrite_storage_scheme** - Rewrite the URL scheme of each storage URL returned from Swift auth to this value. 173 | * e.g.: rewrite_storage_scheme = https 174 | * **rewrite_storage_netloc** - Rewrite the URL netloc (hostname:port) of each storage URL returned from Swift auth to this value. 175 | * e.g.: rewrite_storage_netloc = 127.0.0.1:12345 176 | 177 | **Stats Options** 178 | 179 | * **stats_host** - Address that the HTTP stats interface will listen on. 180 | * **stats_port** - Port that the HTTP stats interface will listen on. 181 | * **log_statsd_host** - statsd hostname. 182 | * **log_statsd_port** - statsd port. 183 | * **log_statsd_sample_rate** - How often in seconds to send metrics to the statsd server. 184 | * **log_statsd_metric_prefix** - Prefix appended to each stat sent to statsd. 185 | 186 | 187 | Caveats 188 | ------- 189 | * You cannot create top-level files, just directories (because the top level are containers). 190 | * You cannot rename any non-empty directory. 191 | * No recursive delete. Most clients will explicitly delete each file/directory recursively anyway. 192 | * Fake-directories and real objects of the same name will simply display the directory. A lot of FTP/SFTP clients [actually explode](http://gifsoup.com/webroot/animatedgifs2/1095919_o.gif) if a directory listing has duplicates. 193 | 194 | Project Organization 195 | -------------------- 196 | * etc/: Sample config files 197 | * swftp/: Core/shared code 198 | * ftp/: FTP server 199 | * sftp/: SFTP server 200 | * test/: Unit and functional tests 201 | * twisted/: For the Twisted Plugin System 202 | 203 | Packaging/Creating Init Scripts 204 | ------------------------------- 205 | Packaged with SwFTP are a set of example init scripts, upstart scripts. They are all located within the /etc/ directory in the source. 206 | 207 | * Upstart 208 | * /etc/init/swftp-ftp.conf 209 | * /etc/init/swftp-sftp.conf 210 | * init.d 211 | * /etc/init.d/swftp-ftp 212 | * /etc/init.d/swftp-sftp 213 | * Supervisor 214 | * /etc/supervisor/conf.d/swftp.conf 215 | * Example swftp.conf file 216 | * /etc/swftp/swftp.conf.sample 217 | 218 | Stats Web Interface 219 | ------------------- 220 | The web interface is an HTTP interface that provides a way to get more app-specific metrics. The only format supported currently is JSON. If the 'stats_host' config value is set, the server will listen to that interface. 221 | 222 | **http://{stats_host}:{stats_port}/stats.json** 223 | 224 | ```bash 225 | $ curl http://127.0.0.1:38022/stats.json | python -mjson.tool 226 | { 227 | "rates": { 228 | "auth.fail": 0, 229 | "auth.succeed": 0, 230 | "command.getAttrs": 0, 231 | "command.login": 0, 232 | "command.logout": 9, 233 | "command.makeDirectory": 0, 234 | "command.openDirectory": 0, 235 | "command.openFile": 0, 236 | "command.removeDirectory": 0, 237 | "command.removeFile": 0, 238 | "command.renameFile": 0, 239 | "num_clients": -9, 240 | "transfer.egress_bytes": 0, 241 | "transfer.ingress_bytes": 47662 242 | }, 243 | "totals": { 244 | "auth.fail": 0, 245 | "auth.succeed": 91, 246 | "command.getAttrs": 15, 247 | "command.login": 91, 248 | "command.logout": 91, 249 | "command.makeDirectory": 0, 250 | "command.openDirectory": 7, 251 | "command.openFile": 8, 252 | "command.removeDirectory": 3, 253 | "command.removeFile": 0, 254 | "command.renameFile": 7, 255 | "num_clients": 0, 256 | "transfer.egress_bytes": 11567105, 257 | "transfer.ingress_bytes": 11567105 258 | } 259 | } 260 | ``` 261 | 262 | Statsd Support 263 | -------------- 264 | Statsd support relies on [txStatsD](https://pypi.python.org/pypi/txStatsD). If the 'log_statsd_host' config value is set, the following paths will be emited into statsd. 265 | 266 | ### General 267 | 268 | * stats.[prefix].egress_bytes 269 | * stats.[prefix].ingress_bytes 270 | * stats.gauges.[prefix].clients 271 | * stats.gauges.[prefix].proc.threads 272 | * stats.gauges.[prefix].proc.cpu.percent 273 | * stats.gauges.[prefix].proc.cpu.system 274 | * stats.gauges.[prefix].proc.cpu.user 275 | * stats.gauges.[prefix].proc.memory.percent 276 | * stats.gauges.[prefix].proc.memory.rss 277 | * stats.gauges.[prefix].proc.memory.vsize 278 | * stats.gauges.[prefix].proc.net.status.[tcp_state] 279 | 280 | ### SFTP-related 281 | 282 | * stats.[prefix].command.getAttrs 283 | * stats.[prefix].command.login 284 | * stats.[prefix].command.logout 285 | * stats.[prefix].command.makeDirectory 286 | * stats.[prefix].command.openDirectory 287 | * stats.[prefix].command.openFile 288 | * stats.[prefix].command.removeDirectory 289 | * stats.[prefix].command.removeFile 290 | * stats.[prefix].command.renameFile 291 | 292 | ### FTP-related 293 | 294 | * stats.[prefix].command.access 295 | * stats.[prefix].command.list 296 | * stats.[prefix].command.login 297 | * stats.[prefix].command.logout 298 | * stats.[prefix].command.makeDirectory 299 | * stats.[prefix].command.openForReading 300 | * stats.[prefix].command.openForWriting 301 | * stats.[prefix].command.removeDirectory 302 | * stats.[prefix].command.removeFile 303 | * stats.[prefix].command.rename 304 | * stats.[prefix].command.stat 305 | 306 | Development 307 | ----------- 308 | Development works with a fork and pull request process. Feel free submit pull requests. 309 | 310 | To run the tests, run 311 | ```bash 312 | $ trial swftp 313 | ``` 314 | 315 | To run tests against live swftp servers (ftp and sftp) it requires a test config. The default location is `/etc/swftp/test.conf` but can be set with the SWFTP_TEST_CONFIG_FILE environmental variable. Here is a sample test config 316 | 317 | ``` 318 | [func_test] 319 | auth_host = 127.0.0.1 320 | auth_port = 8080 321 | auth_ssl = no 322 | auth_prefix = /auth/ 323 | 324 | account = test 325 | username = tester 326 | password = testing 327 | 328 | sftp_host = 127.0.0.1 329 | sftp_port = 5022 330 | 331 | ftp_host = 127.0.0.1 332 | ftp_port = 5021 333 | 334 | ``` 335 | 336 | License 337 | ------- 338 | Copyright (c) 2014 SoftLayer Technologies, Inc. 339 | 340 | Permission is hereby granted, free of charge, to any person obtaining a copy of 341 | this software and associated documentation files (the "Software"), to deal in 342 | the Software without restriction, including without limitation the rights to 343 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 344 | of the Software, and to permit persons to whom the Software is furnished to do 345 | so, subject to the following conditions: 346 | 347 | The above copyright notice and this permission notice shall be included in all 348 | copies or substantial portions of the Software. 349 | 350 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 351 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 352 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 353 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 354 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 355 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 356 | SOFTWARE. 357 | -------------------------------------------------------------------------------- /etc/haproxy/haproxy.cfg: -------------------------------------------------------------------------------- 1 | global 2 | user haproxy 3 | group haproxy 4 | daemon 5 | 6 | defaults 7 | option tcplog 8 | log 127.0.0.1 local0 9 | timeout connect 12000 10 | timeout client 12000 11 | timeout server 12000 12 | 13 | frontend sftp-in 14 | bind *:22 15 | mode tcp 16 | maxconn 3000 17 | timeout client 240s 18 | 19 | # If there are less than 1 local servers online, we're 'down' locally 20 | acl down nbsrv(local-swftp) lt 1 21 | 22 | default_backend swftp-local 23 | 24 | # Use remote hosts if we're down. 25 | use_backend swftp-remote-failover if down 26 | 27 | backend swftp-local 28 | balance leastconn 29 | retries 3 30 | option redispatch 31 | timeout connect 60s 32 | timeout server 240s 33 | 34 | server sftp-22000 127.0.0.1:22000 inter 2s rise 2 fall 2 35 | server sftp-22001 127.0.0.1:22001 inter 2s rise 2 fall 2 36 | 37 | backend swftp-remote-failover 38 | balance roundrobin 39 | retries 1 40 | option redispatch 41 | timeout connect 2s 42 | timeout server 2s 43 | 44 | server failover-1 failoverhost1:22 inter 5s rise 1 fall 2 45 | server failover-2 failoverhost2:22 inter 5s rise 1 fall 2 46 | 47 | listen stats :1936 48 | mode http 49 | stats enable 50 | stats hide-version 51 | stats realm appserver 52 | stats uri / 53 | stats auth user:password -------------------------------------------------------------------------------- /etc/init.d/swftp-ftp: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: swftp-ftp 4 | # Required-Start: $local_fs $remote_fs $network $syslog $named 5 | # Required-Stop: $local_fs $remote_fs $network $syslog $named 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: 0 1 6 8 | # Short-Description: Start/stop/restart swftp-ftp server 9 | ### END INIT INFO 10 | 11 | set -e 12 | 13 | pidfile="/var/run/swftp-ftp.pid" 14 | uid=65534 15 | gid=65534 16 | PREFIX="swftp-ftp" 17 | 18 | case "$1" in 19 | start) 20 | /usr/bin/env twistd --reactor=epoll \ 21 | --syslog --prefix=$PREFIX \ 22 | --pidfile=$pidfile \ 23 | --uid=$uid \ 24 | --gid=$gid \ 25 | swftp-ftp 26 | ;; 27 | stop) 28 | kill $(cat $pidfile) 29 | ;; 30 | restart) 31 | $0 stop 32 | $0 start 33 | ;; 34 | *) 35 | echo "Usage: /etc/init.d/swftp-ftp {start|stop|restart}" 36 | exit 1 37 | ;; 38 | esac 39 | 40 | exit 0 -------------------------------------------------------------------------------- /etc/init.d/swftp-sftp: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: swftp-sftp 4 | # Required-Start: $local_fs $remote_fs $network $syslog $named 5 | # Required-Stop: $local_fs $remote_fs $network $syslog $named 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: 0 1 6 8 | # Short-Description: Start/stop/restart swftp-sftp server 9 | ### END INIT INFO 10 | 11 | set -e 12 | 13 | pidfile="/var/run/swftp-sftp.pid" 14 | uid=65534 15 | gid=65534 16 | PREFIX="swftp-sftp" 17 | 18 | case "$1" in 19 | start) 20 | /usr/bin/env twistd --reactor=epoll \ 21 | --syslog --prefix=$PREFIX \ 22 | --pidfile=$pidfile \ 23 | --uid=$uid \ 24 | --gid=$gid \ 25 | swftp-sftp 26 | ;; 27 | stop) 28 | kill $(cat $pidfile) 29 | ;; 30 | restart) 31 | $0 stop 32 | $0 start 33 | ;; 34 | *) 35 | echo "Usage: /etc/init.d/swftp-sftp {start|stop|restart}" 36 | exit 1 37 | ;; 38 | esac 39 | 40 | exit 0 -------------------------------------------------------------------------------- /etc/init/swftp-ftp.conf: -------------------------------------------------------------------------------- 1 | description "SwFTP - OpenStack Swift FTP Service" 2 | author "Kevin McDonald " 3 | 4 | start on startup 5 | stop on runlevel [!2345] 6 | respawn 7 | 8 | exec /usr/bin/env twistd --reactor=epoll \ 9 | --syslog --prefix=swftp-sftp \ 10 | swftp-ftp 11 | -------------------------------------------------------------------------------- /etc/init/swftp-sftp.conf: -------------------------------------------------------------------------------- 1 | description "SwFTP - OpenStack Swift SFTP Service" 2 | author "Kevin McDonald " 3 | 4 | start on startup 5 | stop on runlevel [!2345] 6 | respawn 7 | 8 | exec /usr/bin/env twistd --reactor=epoll \ 9 | --syslog --prefix=swftp-sftp \ 10 | swftp-sftp 11 | -------------------------------------------------------------------------------- /etc/supervisor/conf.d/swftp.conf: -------------------------------------------------------------------------------- 1 | [program:sftp] 2 | command=/usr/bin/env twistd -n --reactor=epoll --prefix=sftp-%(process_num)02d --uid=65534 --gid=65534 --pidfile=/var/run/sftp.%(process_num)02d.pid swftp-sftp -p 220%(process_num)02d 3 | process_name=swftp-sftp:220%(process_num)02d 4 | numprocs=2 5 | autostart=true 6 | autorestart=true 7 | startsecs=5 8 | startretries=3 9 | exitcodes=0,2 10 | stopsignal=TERM 11 | stopwaitsecs=10 12 | redirect_stderr=false 13 | priority=5 -------------------------------------------------------------------------------- /etc/swftp/swftp.conf.sample: -------------------------------------------------------------------------------- 1 | [sftp] 2 | #host = 0.0.0.0 3 | #port = 5022 4 | #priv_key = /etc/swftp/id_rsa 5 | #pub_key = /etc/swftp/id_rsa.pub 6 | #connection_timeout = 240 7 | 8 | #auth_url = http://127.0.0.1:8080/auth/v1.0 9 | #num_persistent_connections = 20 10 | #num_connections_per_session = 10 11 | #rewrite_storage_scheme = 12 | #rewrite_storage_netloc = 13 | #extra_headers = 14 | 15 | #log_statsd_host = 16 | #log_statsd_port = 8125 17 | #log_statsd_sample_rate = 10 18 | #log_statsd_metric_prefix = sftp 19 | 20 | #stats_host = 21 | #stats_port = 38022 22 | 23 | [ftp] 24 | #host = 0.0.0.0 25 | #port = 5021 26 | #sessions_per_user = 10 27 | #connection_timeout = 240 28 | #welcome_message = Welcome to SwFTP - An FTP/SFTP interface for Openstack Swift 29 | 30 | #auth_url = http://127.0.0.1:8080/auth/v1.0 31 | #num_persistent_connections = 20 32 | #num_connections_per_session = 10 33 | #rewrite_storage_scheme = 34 | #rewrite_storage_netloc = 35 | #extra_headers = 36 | 37 | #log_statsd_host = 38 | #log_statsd_port = 8125 39 | #log_statsd_sample_rate = 10 40 | #log_statsd_metric_prefix = ftp 41 | 42 | #stats_host = 43 | #stats_port = 38021 44 | -------------------------------------------------------------------------------- /etc/swftp/test.conf.sample: -------------------------------------------------------------------------------- 1 | [func_test] 2 | auth_host = 127.0.0.1 3 | auth_port = 8080 4 | auth_ssl = no 5 | auth_prefix = /auth/ 6 | 7 | account = test 8 | username = tester 9 | password = testing 10 | 11 | sftp_host = 127.0.0.1 12 | sftp_port = 5022 13 | 14 | ftp_host = 127.0.0.1 15 | ftp_port = 5021 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | paramiko 2 | mock -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from setuptools import setup, find_packages 3 | import sys 4 | 5 | from swftp import VERSION 6 | 7 | short_description = ('An FTP and SFTP interface for Openstack Object Storage' 8 | '(swift)') 9 | long_description = short_description 10 | try: 11 | long_description = open('README.md').read() 12 | except: 13 | pass 14 | 15 | requires = ['twisted >= 12.1', 'pyopenssl', 'pycrypto', 'pyasn1', 'psutil'] 16 | 17 | if sys.version_info < (2, 7): 18 | requires.append('ordereddict') 19 | 20 | setup( 21 | name='swftp', 22 | version=VERSION, 23 | author='Kevin McDonald', 24 | author_email='kmcdonald@softlayer.com', 25 | license='MIT', 26 | url='https://github.com/softlayer/swftp', 27 | description=short_description, 28 | long_description=long_description, 29 | packages=find_packages() + ['twisted.plugins'], 30 | install_requires=requires, 31 | entry_points={ 32 | 'console_scripts': ['swftp-ftp = swftp.ftp.service:run', 33 | 'swftp-sftp = swftp.sftp.service:run'], 34 | }, 35 | package_data={ 36 | 'twisted.plugins': ['twisted/plugins/swftp_ftp.py', 37 | 'twisted/plugins/swftp_sftp.py'], 38 | 'swftp.test': [ 39 | 'test-sftp.conf', 40 | 'test-ftp.conf', 41 | 'test_id_rsa', 42 | 'test_id_rsa.pub', 43 | ], 44 | }, 45 | data_files=[ 46 | ('', ['README.md']), 47 | ('', ['etc/swftp/swftp.conf.sample']), 48 | ], 49 | zip_safe=False, 50 | classifiers=[ 51 | 'Environment :: Console', 52 | 'Operating System :: OS Independent', 53 | 'Environment :: No Input/Output (Daemon)', 54 | 'Framework :: Twisted', 55 | 'License :: OSI Approved :: MIT License', 56 | 'Topic :: Internet :: File Transfer Protocol (FTP)', 57 | 'Programming Language :: Python', 58 | 'Programming Language :: Python :: 2.6', 59 | 'Programming Language :: Python :: 2.7', 60 | ], 61 | ) 62 | 63 | # Make Twisted regenerate the dropin.cache, if possible. This is necessary 64 | # because in a site-wide install, dropin.cache cannot be rewritten by 65 | # normal users. 66 | try: 67 | from twisted.plugin import IPlugin, getPlugins 68 | except ImportError: 69 | pass 70 | else: 71 | list(getPlugins(IPlugin)) 72 | -------------------------------------------------------------------------------- /swftp/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | SwFTP is an FTP and SFTP interface for Openstack Swift 3 | 4 | See COPYING for license information. 5 | """ 6 | VERSION = '1.0.7' 7 | USER_AGENT = 'SwFTP v%s' % VERSION 8 | 9 | __title__ = 'swftp' 10 | __version__ = VERSION 11 | __author__ = 'SoftLayer Technologies, Inc.' 12 | __license__ = 'MIT' 13 | __copyright__ = 'Copyright 2014 SoftLayer Technologies, Inc.' 14 | -------------------------------------------------------------------------------- /swftp/auth.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | import urlparse 5 | 6 | from zope.interface import implements 7 | from twisted.internet import defer, reactor 8 | from twisted.web.client import HTTPConnectionPool 9 | from twisted.python import log 10 | from twisted.cred import checkers, error, credentials 11 | 12 | from swftp.swift import ThrottledSwiftConnection, UnAuthenticated, UnAuthorized 13 | from swftp import USER_AGENT 14 | 15 | 16 | class SwiftBasedAuthDB(object): 17 | """ 18 | Swift-based authentication 19 | 20 | Implements twisted.cred.ICredentialsChecker 21 | 22 | :param auth_url: auth endpoint for swift 23 | :param int global_max_concurrency: The max concurrency for the entire 24 | server 25 | :param int max_concurrency: The max concurrency for each 26 | ThrottledSwiftConnection object 27 | :param bool verbose: verbose setting 28 | """ 29 | implements(checkers.ICredentialsChecker) 30 | credentialInterfaces = ( 31 | credentials.IUsernamePassword, 32 | ) 33 | 34 | def __init__(self, 35 | auth_url, 36 | global_max_concurrency=100, 37 | max_concurrency=10, 38 | timeout=260, 39 | extra_headers=None, 40 | verbose=False, 41 | rewrite_scheme=None, 42 | rewrite_netloc=None): 43 | self.auth_url = auth_url 44 | self.global_max_concurrency = global_max_concurrency 45 | self.max_concurrency = max_concurrency 46 | self.timeout = timeout 47 | self.extra_headers = extra_headers 48 | self.verbose = verbose 49 | self.rewrite_scheme = rewrite_scheme 50 | self.rewrite_netloc = rewrite_netloc 51 | 52 | def _rewrite_storage_url(self, connection): 53 | if not any((self.rewrite_scheme, self.rewrite_netloc)): 54 | return 55 | 56 | storage_url_parsed = urlparse.urlparse(connection.storage_url) 57 | 58 | new_parts = { 59 | 'scheme': storage_url_parsed.scheme, 60 | 'netloc': storage_url_parsed.netloc, 61 | 'path': storage_url_parsed.path, 62 | 'query': storage_url_parsed.query, 63 | 'fragment': storage_url_parsed.fragment, 64 | } 65 | 66 | part_mapping = { 67 | 'scheme': self.rewrite_scheme, 68 | 'netloc': self.rewrite_netloc, 69 | } 70 | 71 | for k, v in part_mapping.items(): 72 | if v: 73 | new_parts[k] = v 74 | 75 | # Rebuild the URL and set it to the connection's storage_url 76 | connection.storage_url = urlparse.urlunsplit(( 77 | new_parts['scheme'], new_parts['netloc'], new_parts['path'], 78 | new_parts['query'], new_parts['fragment'])) 79 | 80 | def _after_auth(self, result, connection): 81 | log.msg(metric='auth.succeed') 82 | self._rewrite_storage_url(connection) 83 | return connection 84 | 85 | def requestAvatarId(self, c): 86 | creds = credentials.IUsernamePassword(c, None) 87 | 88 | if creds is not None: 89 | locks = [] 90 | pool = HTTPConnectionPool(reactor, persistent=False) 91 | pool.cachedConnectionTimeout = self.timeout 92 | if self.max_concurrency: 93 | pool.persistent = True 94 | pool.maxPersistentPerHost = self.max_concurrency 95 | locks.append( 96 | defer.DeferredSemaphore(self.max_concurrency)) 97 | 98 | if self.global_max_concurrency: 99 | locks.append( 100 | defer.DeferredSemaphore(self.global_max_concurrency)) 101 | 102 | conn = ThrottledSwiftConnection( 103 | locks, self.auth_url, creds.username, creds.password, 104 | pool=pool, 105 | extra_headers=self.extra_headers, 106 | verbose=self.verbose) 107 | conn.user_agent = USER_AGENT 108 | 109 | d = conn.authenticate() 110 | d.addCallback(self._after_auth, conn) 111 | d.addErrback(eb_failed_auth) 112 | return d 113 | return defer.fail(error.UnauthorizedLogin()) 114 | 115 | 116 | def eb_failed_auth(failure): 117 | failure.trap(UnAuthenticated, UnAuthorized) 118 | log.msg(metric='auth.fail') 119 | return defer.fail(error.UnauthorizedLogin()) 120 | -------------------------------------------------------------------------------- /swftp/ftp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softlayer/swftp/6363985ed51c0d34b9d3aab9ead4f4246e93cd7b/swftp/ftp/__init__.py -------------------------------------------------------------------------------- /swftp/ftp/server.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains the primary server code for the FTP server. 3 | 4 | See COPYING for license information. 5 | """ 6 | import stat 7 | from collections import defaultdict 8 | 9 | from zope.interface import implements 10 | from twisted.protocols.ftp import ( 11 | FTP, IFTPShell, IReadFile, IWriteFile, FileNotFoundError, 12 | CmdNotImplementedForArgError, IsNotADirectoryError, IsADirectoryError, 13 | RESPONSE, TOO_MANY_CONNECTIONS) 14 | from twisted.internet import defer, reactor 15 | from twisted.internet.protocol import Protocol 16 | from twisted.python import log 17 | from twisted.protocols.ftp import ( 18 | CmdArgSyntaxError, BadCmdSequenceError, 19 | REQ_FILE_ACTN_PENDING_FURTHER_INFO, PortConnectionError 20 | ) 21 | 22 | from swftp.logging import msg 23 | from swftp.swiftfilesystem import SwiftFileSystem, swift_stat, obj_to_path 24 | from swftp.swift import NotFound, Conflict 25 | 26 | 27 | def stat_format(keys, props): 28 | st = swift_stat(**props) 29 | l = [] 30 | for key in keys: 31 | if key == 'size': 32 | val = st.st_size 33 | elif key == 'directory': 34 | val = st.st_mode & stat.S_IFDIR == stat.S_IFDIR 35 | elif key == 'permissions': 36 | val = st.st_mode 37 | elif key == 'hardlinks': 38 | val = st.st_nlink 39 | elif key == 'modified': 40 | val = int(st.st_mtime) 41 | elif key in 'owner': 42 | val = 'nobody' 43 | elif key in 'group': 44 | val = 'nobody' 45 | else: # Unknown Value 46 | val = '' 47 | l.append(val) 48 | return l 49 | 50 | 51 | class SwftpFTPProtocol(FTP, object): 52 | _connCountMap = defaultdict(int) 53 | maxConnectionsPerUser = 10 54 | 55 | def connectionMade(self, *args, **kwargs): 56 | log.msg(metric='num_clients') 57 | return super(SwftpFTPProtocol, self).connectionMade(*args, **kwargs) 58 | 59 | def connectionLost(self, *args, **kwargs): 60 | log.msg(metric='num_clients', count=-1) 61 | 62 | if self.shell: 63 | username = self.shell.username() 64 | msg("User Disconnected (%s) [%s/%s]" % ( 65 | username, 66 | self._connCountMap[username], 67 | self.maxConnectionsPerUser, 68 | )) 69 | self._connCountMap[username] -= 1 70 | # To avoid a slow memory leak 71 | if self._connCountMap[username] == 0: 72 | del self._connCountMap[username] 73 | return super(SwftpFTPProtocol, self).connectionLost(*args, **kwargs) 74 | 75 | def ftp_PASS(self, *args, **kwargs): 76 | # Check to see if the user has too many connections 77 | d = super(SwftpFTPProtocol, self).ftp_PASS(*args, **kwargs) 78 | 79 | def pass_cb(res): 80 | username = self.shell.username() 81 | self._connCountMap[username] += 1 82 | msg("User Connected (%s) [%s/%s]" % ( 83 | username, 84 | self._connCountMap[username], 85 | self.maxConnectionsPerUser, 86 | )) 87 | if self.maxConnectionsPerUser != 0 and \ 88 | self._connCountMap[username] > self.maxConnectionsPerUser: 89 | msg("Too Many Connections For User (%s) [%s/%s]" % ( 90 | username, 91 | self._connCountMap[username], 92 | self.maxConnectionsPerUser, 93 | )) 94 | self.sendLine(RESPONSE[TOO_MANY_CONNECTIONS]) 95 | self.transport.loseConnection() 96 | return res 97 | 98 | d.addCallback(pass_cb) 99 | return d 100 | 101 | def ftp_LIST(self, path=''): 102 | # ignore special flags for command LIST 103 | keys = ['-a', '-l', '-la', '-al'] 104 | segm = path.split() 105 | path = " ".join(s for s in segm if s.lower() not in keys) 106 | 107 | return super(SwftpFTPProtocol, self).ftp_LIST(path) 108 | 109 | def ftp_NLST(self, path=''): 110 | """ 111 | Overwrite for fix http://twistedmatrix.com/trac/ticket/4258 112 | """ 113 | return super(SwftpFTPProtocol, self).ftp_NLST(path) 114 | 115 | def ftp_PASV(self): 116 | d = super(SwftpFTPProtocol, self).ftp_PASV() 117 | 118 | def dtp_connect_timeout_eb(failure): 119 | failure.trap(PortConnectionError) 120 | 121 | return d.addErrback(dtp_connect_timeout_eb) 122 | 123 | def ftp_REST(self, value): 124 | if self.dtpInstance is None: 125 | raise BadCmdSequenceError('PORT or PASV required before RETR') 126 | 127 | try: 128 | value = int(value) 129 | if value < 0: 130 | raise ValueError 131 | except ValueError: 132 | raise CmdArgSyntaxError('Value must be nonnegative integer') 133 | else: 134 | self.dtpInstance.rest_offset = value 135 | 136 | return (REQ_FILE_ACTN_PENDING_FURTHER_INFO, ) 137 | 138 | def cleanupDTP(self): 139 | """ 140 | Overwrite cleanupDTP() for fix socket leak 141 | (see http://twistedmatrix.com/trac/ticket/5367) 142 | """ 143 | transport = None 144 | if self.dtpInstance is not None: 145 | if self.dtpInstance.transport is not None: 146 | transport = self.dtpInstance.transport 147 | 148 | super(SwftpFTPProtocol, self).cleanupDTP() 149 | 150 | if transport: 151 | transport.abortConnection() 152 | 153 | 154 | class SwiftFTPShell(object): 155 | """ Implements all the methods needed to treat Swift as an FTP Shell """ 156 | implements(IFTPShell) 157 | 158 | def __init__(self, swiftconn): 159 | self.swiftconn = swiftconn 160 | self.swiftfilesystem = SwiftFileSystem(self.swiftconn) 161 | self.log_command('login') 162 | 163 | def log_command(self, command, *args): 164 | arg_list = ', '.join(str(arg) for arg in args) 165 | msg("cmd: %s(%s)" % (command, arg_list), 166 | system="SwFTP-FTP, (%s)" % self.swiftconn.username, 167 | metric='command.%s' % command) 168 | 169 | def username(self): 170 | return self.swiftconn.username 171 | 172 | def logout(self): 173 | self.log_command('logout') 174 | if self.swiftconn.pool: 175 | self.swiftconn.pool.closeCachedConnections() 176 | del self.swiftconn 177 | 178 | def _fullpath(self, path_parts): 179 | return '/'.join(path_parts) 180 | 181 | def makeDirectory(self, path): 182 | self.log_command('makeDirectory', path) 183 | fullpath = self._fullpath(path) 184 | return self.swiftfilesystem.makeDirectory(fullpath) 185 | 186 | def removeDirectory(self, path): 187 | self.log_command('removeDirectory', path) 188 | fullpath = self._fullpath(path) 189 | 190 | def not_found_eb(failure): 191 | failure.trap(NotFound) 192 | 193 | def conflict_eb(failure): 194 | failure.trap(Conflict) 195 | raise CmdNotImplementedForArgError( 196 | 'Cannot delete non-empty directories.') 197 | 198 | d = self.swiftfilesystem.removeDirectory(fullpath) 199 | d.addErrback(not_found_eb) 200 | d.addErrback(conflict_eb) 201 | return d 202 | 203 | def removeFile(self, path): 204 | self.log_command('removeFile', path) 205 | fullpath = self._fullpath(path) 206 | 207 | def errback(failure): 208 | failure.trap(NotFound, NotImplementedError) 209 | if failure.check(NotImplementedError): 210 | return defer.fail(IsADirectoryError(fullpath)) 211 | d = defer.maybeDeferred(self.swiftfilesystem.removeFile, fullpath) 212 | d.addErrback(errback) 213 | return d 214 | 215 | def rename(self, fromPath, toPath): 216 | self.log_command('rename', fromPath, toPath) 217 | oldpath = self._fullpath(fromPath) 218 | newpath = self._fullpath(toPath) 219 | 220 | d = self.swiftfilesystem.renameFile(oldpath, newpath) 221 | 222 | def errback(failure): 223 | failure.trap(NotFound, Conflict, NotImplementedError) 224 | if failure.check(NotFound): 225 | return defer.fail(FileNotFoundError(oldpath)) 226 | else: 227 | return defer.fail(CmdNotImplementedForArgError(oldpath)) 228 | d.addErrback(errback) 229 | return d 230 | 231 | def access(self, path): 232 | self.log_command('access', path) 233 | fullpath = self._fullpath(path) 234 | 235 | d = self.swiftfilesystem.getAttrs(fullpath) 236 | 237 | def cb(result): 238 | if result['content_type'] == 'application/directory': 239 | return defer.succeed(lambda: None) 240 | return defer.fail(IsNotADirectoryError(fullpath)) 241 | d.addCallback(cb) 242 | 243 | def err(failure): 244 | failure.trap(NotFound) 245 | # Containers need to actually exist before uploading anything 246 | # inside of them. Therefore require containers to actually exist. 247 | # All other paths don't have to. 248 | if len(path) != 1: 249 | return defer.succeed(lambda: None) 250 | else: 251 | return defer.fail(IsNotADirectoryError(fullpath)) 252 | 253 | d.addErrback(err) 254 | return d 255 | 256 | def stat(self, path, keys=()): 257 | self.log_command('stat', path, keys) 258 | fullpath = self._fullpath(path) 259 | 260 | def cb(result): 261 | return stat_format(keys, result) 262 | 263 | def err(failure): 264 | failure.trap(NotFound) 265 | return defer.fail(FileNotFoundError(fullpath)) 266 | 267 | d = self.swiftfilesystem.getAttrs(fullpath) 268 | d.addCallback(cb) 269 | d.addErrback(err) 270 | return d 271 | 272 | def list(self, path=None, keys=()): 273 | self.log_command('list', path) 274 | fullpath = self._fullpath(path) 275 | 276 | def cb(results): 277 | l = [] 278 | for key, value in results.iteritems(): 279 | l.append([key, stat_format(keys, value)]) 280 | return l 281 | 282 | def err(failure): 283 | failure.trap(NotFound) 284 | return defer.fail(FileNotFoundError(fullpath)) 285 | 286 | d = self.swiftfilesystem.get_full_listing(fullpath) 287 | d.addCallback(cb) 288 | d.addErrback(err) 289 | return d 290 | 291 | def openForReading(self, path): 292 | self.log_command('openForReading', path) 293 | fullpath = self._fullpath(path) 294 | 295 | def cb(results): 296 | return SwiftReadFile(self.swiftfilesystem, fullpath) 297 | 298 | def err(failure): 299 | failure.trap(NotFound) 300 | return defer.fail(FileNotFoundError(fullpath)) 301 | 302 | try: 303 | d = self.swiftfilesystem.checkFileExistance(fullpath) 304 | d.addCallback(cb) 305 | d.addErrback(err) 306 | return d 307 | except NotImplementedError: 308 | return defer.fail(IsADirectoryError(fullpath)) 309 | 310 | def openForWriting(self, path): 311 | self.log_command('openForWriting', path) 312 | fullpath = self._fullpath(path) 313 | container, obj = obj_to_path(fullpath) 314 | if not container or not obj: 315 | raise CmdNotImplementedForArgError( 316 | 'Cannot upload files to root directory.') 317 | f = SwiftWriteFile(self.swiftfilesystem, fullpath) 318 | return defer.succeed(f) 319 | 320 | 321 | class SwiftWriteFile(object): 322 | implements(IWriteFile) 323 | 324 | def __init__(self, swiftfilesystem, fullpath): 325 | self.swiftfilesystem = swiftfilesystem 326 | self.fullpath = fullpath 327 | self.finished = None 328 | 329 | def receive(self): 330 | d, writer = self.swiftfilesystem.startFileUpload(self.fullpath) 331 | self.finished = d 332 | return writer.started 333 | 334 | def close(self): 335 | return self.finished 336 | 337 | 338 | class SwiftReadFile(Protocol): 339 | implements(IReadFile) 340 | 341 | def __init__(self, swiftfilesystem, fullpath): 342 | self.swiftfilesystem = swiftfilesystem 343 | self.fullpath = fullpath 344 | self.finished = defer.Deferred() 345 | self.backend_transport = None 346 | self.timeout = None 347 | self._timedout = False 348 | 349 | def setTimeout(self, seconds): 350 | if self.timeout: 351 | self.cancelTimeout() 352 | self.timeout = reactor.callLater(seconds, self.timedOut) 353 | 354 | def cancelTimeout(self): 355 | if self.timeout: 356 | self.timeout.cancel() 357 | 358 | def timedOut(self): 359 | self._timedout = True 360 | self.stopProducing() 361 | 362 | # IReadFile Interface 363 | def send(self, consumer): 364 | at = getattr(consumer, "rest_offset", 0) 365 | if at: 366 | del consumer.rest_offset # reset for next command 367 | self.consumer = consumer 368 | d = self.swiftfilesystem.startFileDownload( 369 | self.fullpath, self, offset=at) 370 | d.addCallback(lambda _: self.finished) 371 | self.consumer.registerProducer(self, True) 372 | return d 373 | 374 | # Producer Interface 375 | def resumeProducing(self): 376 | self.setTimeout(20) 377 | if self.backend_transport: 378 | self.backend_transport.resumeProducing() 379 | 380 | def pauseProducing(self): 381 | if self.backend_transport: 382 | self.backend_transport.pauseProducing() 383 | 384 | def stopProducing(self): 385 | if self.backend_transport: 386 | self.backend_transport.stopProducing() 387 | 388 | # Protocol 389 | def dataReceived(self, data): 390 | log.msg(metric='transfer.egress_bytes', count=len(data)) 391 | self.consumer.write(data) 392 | self.setTimeout(20) 393 | 394 | def connectionLost(self, reason): 395 | from twisted.web._newclient import ResponseDone 396 | from twisted.web.http import PotentialDataLoss 397 | 398 | if reason.check(ResponseDone) or reason.check(PotentialDataLoss): 399 | self.finished.callback(None) 400 | else: 401 | if self._timedout: 402 | defer.timeout(self.finished) 403 | else: 404 | self.finished.errback(reason) 405 | self.backend_transport = None 406 | self.consumer.unregisterProducer() 407 | 408 | def makeConnection(self, transport): 409 | self.backend_transport = transport 410 | 411 | def connectionMade(self): 412 | pass 413 | -------------------------------------------------------------------------------- /swftp/ftp/service.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file defines what is required for swftp-ftp to work with twistd. 3 | 4 | See COPYING for license information. 5 | """ 6 | from swftp import VERSION 7 | from swftp.logging import StdOutObserver 8 | 9 | from twisted.application import internet, service 10 | from twisted.python import usage, log 11 | from twisted.internet import reactor 12 | 13 | import ConfigParser 14 | import signal 15 | import os 16 | import sys 17 | 18 | CONFIG_DEFAULTS = { 19 | 'auth_url': 'http://127.0.0.1:8080/auth/v1.0', 20 | 'host': '0.0.0.0', 21 | 'port': '5021', 22 | 23 | 'rewrite_storage_scheme': '', 24 | 'rewrite_storage_netloc': '', 25 | 26 | 'num_persistent_connections': '100', 27 | 'num_connections_per_session': '10', 28 | 'connection_timeout': '240', 29 | 'session_timeout': '60', 30 | 'sessions_per_user': '10', 31 | 'extra_headers': '', 32 | 'verbose': 'false', 33 | 'welcome_message': 'Welcome to SwFTP' 34 | ' - an FTP interface for Openstack Swift', 35 | 'log_statsd_host': '', 36 | 'log_statsd_port': '8125', 37 | 'log_statsd_sample_rate': '10.0', 38 | 'log_statsd_metric_prefix': 'swftp.ftp', 39 | 40 | 'stats_host': '', 41 | 'stats_port': '38021', 42 | } 43 | 44 | 45 | def run(): 46 | options = Options() 47 | try: 48 | options.parseOptions(sys.argv[1:]) 49 | except usage.UsageError as errortext: 50 | print '%s: %s' % (sys.argv[0], errortext) 51 | print '%s: Try --help for usage details.' % (sys.argv[0]) 52 | sys.exit(1) 53 | 54 | # Start Logging 55 | obs = StdOutObserver() 56 | obs.start() 57 | 58 | s = makeService(options) 59 | s.startService() 60 | reactor.run() 61 | 62 | 63 | def get_config(config_path, overrides): 64 | c = ConfigParser.ConfigParser(CONFIG_DEFAULTS) 65 | c.add_section('ftp') 66 | if config_path: 67 | log.msg('Reading configuration from path: %s' % config_path) 68 | c.read(config_path) 69 | else: 70 | config_paths = [ 71 | '/etc/swftp/swftp.conf', 72 | os.path.expanduser('~/.swftp.cfg') 73 | ] 74 | log.msg('Reading configuration from paths: %s' % config_paths) 75 | c.read(config_paths) 76 | for k, v in overrides.iteritems(): 77 | if v: 78 | c.set('ftp', k, str(v)) 79 | return c 80 | 81 | 82 | class Options(usage.Options): 83 | "Defines Command-line options for the swftp-ftp service" 84 | optFlags = [ 85 | ["verbose", "v", "Make the server more talkative"] 86 | ] 87 | optParameters = [ 88 | ["config_file", "c", None, "Location of the swftp config file."], 89 | ["auth_url", "a", None, 90 | "Auth Url to use. Defaults to the config file value if it exists. " 91 | "[default: http://127.0.0.1:8080/auth/v1.0]"], 92 | ["port", "p", None, "Port to bind to."], 93 | ["host", "h", None, "IP to bind to."], 94 | ] 95 | 96 | 97 | def makeService(options): 98 | """ 99 | Makes a new swftp-ftp service. The only option is the config file 100 | location. See CONFIG_DEFAULTS for list of configuration options. 101 | """ 102 | from twisted.protocols.ftp import FTPFactory 103 | from twisted.cred.portal import Portal 104 | 105 | from swftp.ftp.server import SwftpFTPProtocol 106 | from swftp.realm import SwftpRealm 107 | from swftp.auth import SwiftBasedAuthDB 108 | from swftp.utils import ( 109 | log_runtime_info, GLOBAL_METRICS, parse_key_value_config) 110 | 111 | print('Starting SwFTP-ftp %s' % VERSION) 112 | 113 | c = get_config(options['config_file'], options) 114 | ftp_service = service.MultiService() 115 | 116 | # Add statsd service 117 | if c.get('ftp', 'log_statsd_host'): 118 | try: 119 | from swftp.statsd import makeService as makeStatsdService 120 | makeStatsdService( 121 | c.get('ftp', 'log_statsd_host'), 122 | c.getint('ftp', 'log_statsd_port'), 123 | sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'), 124 | prefix=c.get('ftp', 'log_statsd_metric_prefix') 125 | ).setServiceParent(ftp_service) 126 | except ImportError: 127 | sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n') 128 | 129 | if c.get('ftp', 'stats_host'): 130 | from swftp.report import makeService as makeReportService 131 | known_fields = [ 132 | 'command.login', 133 | 'command.logout', 134 | 'command.makeDirectory', 135 | 'command.removeDirectory', 136 | 'command.removeFile', 137 | 'command.rename', 138 | 'command.access', 139 | 'command.stat', 140 | 'command.list', 141 | 'command.openForReading', 142 | 'command.openForWriting', 143 | ] + GLOBAL_METRICS 144 | makeReportService( 145 | c.get('ftp', 'stats_host'), 146 | c.getint('ftp', 'stats_port'), 147 | known_fields=known_fields 148 | ).setServiceParent(ftp_service) 149 | 150 | authdb = SwiftBasedAuthDB( 151 | c.get('ftp', 'auth_url'), 152 | global_max_concurrency=c.getint('ftp', 'num_persistent_connections'), 153 | max_concurrency=c.getint('ftp', 'num_connections_per_session'), 154 | timeout=c.getint('ftp', 'connection_timeout'), 155 | extra_headers=parse_key_value_config(c.get('ftp', 'extra_headers')), 156 | verbose=c.getboolean('ftp', 'verbose'), 157 | rewrite_scheme=c.get('ftp', 'rewrite_storage_scheme'), 158 | rewrite_netloc=c.get('ftp', 'rewrite_storage_netloc'), 159 | ) 160 | 161 | ftpportal = Portal(SwftpRealm()) 162 | ftpportal.registerChecker(authdb) 163 | ftpfactory = FTPFactory(ftpportal) 164 | protocol = SwftpFTPProtocol 165 | protocol.maxConnectionsPerUser = c.getint('ftp', 'sessions_per_user') 166 | ftpfactory.protocol = protocol 167 | ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message') 168 | ftpfactory.allowAnonymous = False 169 | ftpfactory.timeOut = c.getint('ftp', 'session_timeout') 170 | 171 | signal.signal(signal.SIGUSR1, log_runtime_info) 172 | signal.signal(signal.SIGUSR2, log_runtime_info) 173 | 174 | internet.TCPServer( 175 | c.getint('ftp', 'port'), 176 | ftpfactory, 177 | interface=c.get('ftp', 'host')).setServiceParent(ftp_service) 178 | return ftp_service 179 | -------------------------------------------------------------------------------- /swftp/logging.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | import sys 5 | import syslog as pysyslog 6 | 7 | from twisted.python import syslog 8 | from twisted.python import log 9 | 10 | WHITELISTED_LOG_SYSTEMS = ['SwFTP', '-'] 11 | 12 | 13 | def msg(message, *args, **kwargs): 14 | if not kwargs.get('system'): 15 | kwargs['system'] = 'SwFTP' 16 | return log.msg(message, *args, **kwargs) 17 | 18 | 19 | class LogObserver(object): 20 | def start(self): 21 | log.addObserver(self) 22 | 23 | def stop(self): 24 | log.removeObserver(self) 25 | 26 | def __call__(self, event_dict): 27 | if any((True for system in WHITELISTED_LOG_SYSTEMS 28 | if event_dict.get('system', '').startswith(system))) \ 29 | or event_dict.get('isError', False): 30 | self.obs.emit(event_dict) 31 | 32 | 33 | class StdOutObserver(LogObserver): 34 | def __init__(self): 35 | self.obs = log.FileLogObserver(sys.stdout) 36 | 37 | 38 | class SysLogObserver(LogObserver): 39 | facility = pysyslog.LOG_USER 40 | 41 | def __init__(self): 42 | self.obs = syslog.SyslogObserver('swftp', facility=self.facility) 43 | 44 | 45 | class LOG_USER(SysLogObserver): 46 | facility = pysyslog.LOG_USER 47 | 48 | 49 | class LOG_DAEMON(SysLogObserver): 50 | facility = pysyslog.LOG_DAEMON 51 | 52 | 53 | class LOG_SYSLOG(SysLogObserver): 54 | facility = pysyslog.LOG_SYSLOG 55 | 56 | 57 | class LOG_LOCAL0(SysLogObserver): 58 | facility = pysyslog.LOG_LOCAL0 59 | 60 | 61 | class LOG_LOCAL1(SysLogObserver): 62 | facility = pysyslog.LOG_LOCAL1 63 | 64 | 65 | class LOG_LOCAL2(SysLogObserver): 66 | facility = pysyslog.LOG_LOCAL2 67 | 68 | 69 | class LOG_LOCAL3(SysLogObserver): 70 | facility = pysyslog.LOG_LOCAL3 71 | 72 | 73 | class LOG_LOCAL4(SysLogObserver): 74 | facility = pysyslog.LOG_LOCAL4 75 | 76 | 77 | class LOG_LOCAL5(SysLogObserver): 78 | facility = pysyslog.LOG_LOCAL5 79 | 80 | 81 | class LOG_LOCAL6(SysLogObserver): 82 | facility = pysyslog.LOG_LOCAL6 83 | 84 | 85 | class LOG_LOCAL7(SysLogObserver): 86 | facility = pysyslog.LOG_LOCAL7 87 | -------------------------------------------------------------------------------- /swftp/realm.py: -------------------------------------------------------------------------------- 1 | from twisted.cred import portal 2 | from zope import interface 3 | 4 | try: 5 | from twisted.conch.avatar import IConchUser 6 | from swftp.sftp.server import SwiftSFTPUser 7 | HAS_SFTP = True 8 | except ImportError: 9 | HAS_SFTP = False 10 | 11 | try: 12 | from twisted.protocols.ftp import IFTPShell 13 | from swftp.ftp.server import SwiftFTPShell 14 | HAS_FTP = True 15 | except ImportError: 16 | HAS_FTP = False 17 | 18 | 19 | class SwftpRealm(object): 20 | interface.implements(portal.IRealm) 21 | 22 | def getHomeDirectory(self): 23 | return '/' 24 | 25 | def requestAvatar(self, avatarId, mind, *interfaces): 26 | if avatarId: 27 | interface = interfaces[0] 28 | if HAS_SFTP and interface == IConchUser: 29 | avatar = SwiftSFTPUser(avatarId) 30 | return interface, avatar, avatar.logout 31 | elif HAS_FTP and interface == IFTPShell: 32 | shell = SwiftFTPShell(avatarId) 33 | return interface, shell, shell.logout 34 | 35 | raise NotImplementedError( 36 | 'Only the IFTPShell and IConchUser interfaces are supported ' 37 | 'by this realm') 38 | -------------------------------------------------------------------------------- /swftp/report.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | import json 5 | from copy import copy 6 | 7 | from twisted.web.server import Site 8 | from twisted.web.resource import Resource 9 | from twisted.web.http_headers import Headers 10 | from twisted.application import internet, service 11 | 12 | from swftp.utils import MetricCollector, runtime_info 13 | 14 | 15 | class Stats(Resource): 16 | """ Stats resource 17 | 18 | Routes: 19 | GET /stats.json 20 | 21 | """ 22 | isLeaf = True 23 | 24 | def __init__(self, metric_collector, known_fields=None): 25 | self.metric_collector = metric_collector 26 | self.known_fields = known_fields or [] 27 | 28 | def _populate_known_fields(self, d, default=0): 29 | for field in self.known_fields: 30 | d[field] = d.get(field, default) 31 | 32 | def get_stats(self): 33 | totals = copy(self.metric_collector.totals) 34 | samples = copy(self.metric_collector.samples) 35 | self._populate_known_fields(totals, 0) 36 | self._populate_known_fields(samples, [0]) 37 | return { 38 | 'totals': totals, 39 | 'rates': dict( 40 | (key, sum(value) / len(value)) for (key, value) in 41 | samples.items()), 42 | } 43 | 44 | def render_GET(self, request): 45 | if request.path == '/stats.json': 46 | request.responseHeaders = Headers({ 47 | 'Content-Type': ['application/json']}) 48 | return json.dumps(self.get_stats(), indent=4) 49 | elif request.path == '/debug.json': 50 | request.responseHeaders = Headers({ 51 | 'Content-Type': ['application/json']}) 52 | return json.dumps(runtime_info(), cls=CustomEncoder, indent=4) 53 | else: 54 | request.setResponseCode(404) 55 | return 'not found' 56 | 57 | 58 | class CustomEncoder(json.JSONEncoder): 59 | def default(self, obj): 60 | # Convert anything that isn't a normal Python type to a string 61 | if type(obj) in [str, dict, list, tuple, int, float]: 62 | return json.JSONEncoder.default(self, obj) 63 | return str(obj) 64 | 65 | 66 | def makeService(host='127.0.0.1', port=8125, known_fields=None): 67 | metric_collector = MetricCollector() 68 | metric_collector.start() 69 | 70 | root = Stats(metric_collector, known_fields=known_fields) 71 | site = Site(root) 72 | 73 | def sample_metrics(): 74 | metric_collector.sample() 75 | 76 | s = service.MultiService() 77 | internet.TCPServer( 78 | port, 79 | site, 80 | interface=host).setServiceParent(s) 81 | internet.TimerService(1, sample_metrics).setServiceParent(s) 82 | return s 83 | -------------------------------------------------------------------------------- /swftp/sftp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softlayer/swftp/6363985ed51c0d34b9d3aab9ead4f4246e93cd7b/swftp/sftp/__init__.py -------------------------------------------------------------------------------- /swftp/sftp/server.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains the primary server code for the SFTP server. 3 | 4 | See COPYING for license information. 5 | """ 6 | from zope import interface 7 | from collections import defaultdict 8 | 9 | from twisted.conch.interfaces import ISFTPServer, ISession 10 | from twisted.python import components, log 11 | from twisted.internet import defer 12 | from twisted.conch import avatar 13 | from twisted.conch.ssh import session 14 | from twisted.conch.ssh.filetransfer import ( 15 | FileTransferServer, SFTPError, FX_FAILURE, FX_NO_SUCH_FILE) 16 | from twisted.conch.ssh.common import getNS 17 | from twisted.conch.ssh.transport import ( 18 | SSHServerTransport, DISCONNECT_TOO_MANY_CONNECTIONS) 19 | from twisted.conch.ssh.userauth import SSHUserAuthServer 20 | 21 | from swftp.swift import NotFound, Conflict 22 | from swftp.logging import msg 23 | from swftp.sftp.swiftfile import SwiftFile 24 | from swftp.sftp.swiftdirectory import SwiftDirectory 25 | from swftp.swiftfilesystem import SwiftFileSystem, swift_stat, obj_to_path 26 | 27 | 28 | class SwiftSession(object): 29 | """ Barebones Session that closes when a client tries to open a shell. 30 | Provides t.c.i.ISession 31 | 32 | :param avatar: SwiftSFTPUser instance 33 | 34 | """ 35 | interface.implements(ISession) 36 | 37 | def __init__(self, avatar): 38 | self.avatar = avatar 39 | 40 | def openShell(self, proto): 41 | # Immediately Close the connection 42 | self.avatar.conn.transport.transport.loseConnection() 43 | 44 | def getPty(self, term, windowSize, modes): 45 | pass 46 | 47 | def closed(self): 48 | pass 49 | 50 | def execCommand(self, proto, command): 51 | # Immediately Close the connection 52 | self.avatar.conn.transport.transport.loseConnection() 53 | 54 | 55 | class SwiftFileTransferServer(FileTransferServer): 56 | client = None 57 | transport = None 58 | 59 | # Overridden to expose the session to the file object to do intellegent 60 | # throttling. Without this memory bloat occurs. 61 | def _cbOpenFile(self, fileObj, requestId): 62 | fileObj.session = self.transport.session 63 | FileTransferServer._cbOpenFile(self, fileObj, requestId) 64 | 65 | # This is overridden because Flow was sending data that looks to be invalid 66 | def packet_REALPATH(self, data): 67 | requestId = data[:4] 68 | data = data[4:] 69 | path, data = getNS(data) 70 | # assert data == '', 'still have data in REALPATH: %s' % repr(data) 71 | d = defer.maybeDeferred(self.client.realPath, path) 72 | d.addCallback(self._cbReadLink, requestId) # same return format 73 | d.addErrback(self._ebStatus, requestId, 'realpath failed') 74 | 75 | 76 | class SwiftSSHServerTransport(SSHServerTransport, object): 77 | # Overridden to set the version string. 78 | version = 'SwFTP' 79 | ourVersionString = 'SSH-2.0-SwFTP' 80 | maxConnectionsPerUser = 10 81 | 82 | _connCountMap = defaultdict(int) 83 | 84 | def sendDisconnect(self, *args, **kwargs): 85 | return super(SwiftSSHServerTransport, self).sendDisconnect( 86 | *args, **kwargs) 87 | 88 | def loseConnection(self): 89 | return super(SwiftSSHServerTransport, self).loseConnection() 90 | 91 | def connectionMade(self): 92 | log.msg(metric='num_clients') 93 | return super(SwiftSSHServerTransport, self).connectionMade() 94 | 95 | def connectionLost(self, reason): 96 | log.msg(metric='num_clients', count=-1) 97 | if getattr(self, 'avatar', None): 98 | username = self.avatar.username() 99 | msg("User Disconnected (%s) [%s/%s]" % ( 100 | username, 101 | self._connCountMap[username], 102 | self.maxConnectionsPerUser, 103 | )) 104 | self._connCountMap[username] -= 1 105 | # To avoid a slow memory leak 106 | if self._connCountMap[username] == 0: 107 | del self._connCountMap[username] 108 | 109 | if self.service: 110 | self.service.serviceStopped() 111 | if hasattr(self, 'avatar'): 112 | self.logoutFunction() 113 | 114 | def on_auth(self, res): 115 | if not getattr(self, 'avatar', None): 116 | return res 117 | username = self.avatar.username() 118 | self._connCountMap[username] += 1 119 | msg("User Connected (%s) [%s/%s]" % ( 120 | username, 121 | self._connCountMap[username], 122 | self.maxConnectionsPerUser, 123 | )) 124 | if self.maxConnectionsPerUser != 0 and \ 125 | self._connCountMap[username] > self.maxConnectionsPerUser: 126 | msg("Too Many Connections For User (%s) [%s/%s]" % ( 127 | username, 128 | self._connCountMap[username], 129 | self.maxConnectionsPerUser, 130 | )) 131 | self.sendDisconnect( 132 | DISCONNECT_TOO_MANY_CONNECTIONS, 133 | 'too many connections') 134 | self.loseConnection() 135 | return res 136 | 137 | 138 | class SwiftSSHUserAuthServer(SSHUserAuthServer, object): 139 | 140 | def ssh_USERAUTH_REQUEST(self, *args, **kwargs): 141 | d = super(SwiftSSHUserAuthServer, self).ssh_USERAUTH_REQUEST( 142 | *args, **kwargs) 143 | 144 | d.addCallback(self.transport.on_auth) 145 | return d 146 | 147 | 148 | class SwiftSFTPUser(avatar.ConchUser): 149 | """ Swift SFTP User. Provides t.c.i.IConchUser 150 | 151 | :param swiftconn: an swftp.swift.SwiftConnection instance 152 | 153 | """ 154 | def __init__(self, swiftconn): 155 | avatar.ConchUser.__init__(self) 156 | self.swiftconn = swiftconn 157 | 158 | self.channelLookup.update({"session": session.SSHSession}) 159 | self.subsystemLookup.update({"sftp": SwiftFileTransferServer}) 160 | 161 | self.cwd = '' 162 | 163 | def username(self): 164 | return self.swiftconn.username 165 | 166 | def logout(self): 167 | """ Log-out/clean up avatar-related things """ 168 | self.log_command('logout') 169 | if self.swiftconn.pool: 170 | self.swiftconn.pool.closeCachedConnections() 171 | del self.swiftconn 172 | 173 | def log_command(self, command, *args): 174 | """ Log command 175 | 176 | :param str command: Name of the command 177 | :param \*args args: Arguments passed into the command to be logged 178 | 179 | """ 180 | arg_list = ', '.join(str(arg) for arg in args) 181 | msg("cmd.%s(%s)" % (command, arg_list), 182 | system="SwFTP-SFTP, (%s)" % self.swiftconn.username, 183 | metric='command.%s' % command) 184 | 185 | 186 | class SFTPServerForSwiftConchUser(object): 187 | """ SFTP Server For a Swift User. Provides t.c.i.ISFTPServer 188 | 189 | :param avatar: SwiftSFTPUser instance 190 | 191 | """ 192 | interface.implements(ISFTPServer) 193 | 194 | def __init__(self, avatar): 195 | self.swiftconn = avatar.swiftconn 196 | self.swiftfilesystem = SwiftFileSystem(self.swiftconn) 197 | self.avatar = avatar 198 | self.conn = avatar.conn 199 | self.log_command('login') 200 | 201 | def log_command(self, *args, **kwargs): 202 | """ Logs the given command. 203 | 204 | :param \*args: args to log 205 | :param \*\*kwargs: kwargs to log 206 | 207 | """ 208 | return self.avatar.log_command(*args, **kwargs) 209 | 210 | def gotVersion(self, otherVersion, extData): 211 | """ Client sent their version info """ 212 | self.log_command('gotVersion', otherVersion, extData) 213 | return {} 214 | 215 | def openFile(self, fullpath, flags, attrs): 216 | """ Open File/Object. Checks for Object Existence 217 | 218 | :param str fullpath: path to an object 219 | :param flags: flags to open the object with 220 | :param dict attrs: extra attributes to open the object with 221 | 222 | """ 223 | self.log_command('openFile', fullpath, flags, attrs) 224 | f = SwiftFile(self, fullpath, flags=flags, attrs=attrs) 225 | d = f.checkExistance() 226 | 227 | def errback(failure): 228 | failure.trap(NotFound) 229 | raise SFTPError(FX_FAILURE, "Container Doesn't Exist") 230 | 231 | d.addCallback(lambda r: f) 232 | d.addErrback(errback) 233 | return d 234 | 235 | def removeFile(self, fullpath): 236 | """ Remove Object 237 | 238 | :param str fullpath: path to an object 239 | 240 | """ 241 | self.log_command('removeFile', fullpath) 242 | 243 | def errback(failure): 244 | failure.trap(NotFound) 245 | if failure.check(NotFound): 246 | return 247 | d = self.swiftfilesystem.removeFile(fullpath) 248 | d.addErrback(errback) 249 | return d 250 | 251 | def renameFile(self, oldpath, newpath): 252 | """ Rename an Object 253 | 254 | :param str oldpath: old path to an object 255 | :param str newpath: new path to an object 256 | 257 | """ 258 | self.log_command('renameFile', oldpath, newpath) 259 | d = self.swiftfilesystem.renameFile(oldpath, newpath) 260 | 261 | def errback(failure): 262 | failure.trap(NotFound, Conflict) 263 | if failure.check(NotFound): 264 | raise SFTPError(FX_NO_SUCH_FILE, 'No Such File') 265 | if failure.check(Conflict): 266 | raise NotImplementedError 267 | 268 | d.addErrback(errback) 269 | return d 270 | 271 | def makeDirectory(self, fullpath, attrs): 272 | """ Make a 'directory' (either container or object). The container must 273 | exist to create a directory object inside of it. 274 | 275 | :param str fullpath: path to the directory 276 | :param dict attrs: attributes to create the directory with 277 | 278 | """ 279 | self.log_command('makeDirectory', fullpath, attrs) 280 | 281 | def errback(failure): 282 | failure.trap(NotFound) 283 | raise SFTPError(FX_NO_SUCH_FILE, 'Directory Not Found') 284 | 285 | d = self.swiftfilesystem.makeDirectory(fullpath, attrs) 286 | d.addErrback(errback) 287 | return d 288 | 289 | def removeDirectory(self, fullpath): 290 | """ Remove a 'directory' (either container or object). Not recursive. 291 | Will not delete a non-empty container 292 | 293 | :param str fullpath: path to the directory 294 | 295 | """ 296 | self.log_command('removeDirectory', fullpath) 297 | d = self.swiftfilesystem.removeDirectory(fullpath) 298 | 299 | def errback(failure): 300 | failure.trap(NotFound, Conflict) 301 | if failure.check(NotFound): 302 | return 303 | if failure.check(Conflict): 304 | raise SFTPError(FX_FAILURE, 'Directory Not Empty') 305 | 306 | d.addErrback(errback) 307 | return d 308 | 309 | def openDirectory(self, fullpath): 310 | """ Open a 'directory' 311 | 312 | :param str fullpath: path to the directory 313 | 314 | """ 315 | self.log_command('openDirectory', fullpath) 316 | directory = SwiftDirectory(self.swiftfilesystem, fullpath) 317 | 318 | def cb(*result): 319 | return directory 320 | 321 | def errback(failure): 322 | failure.trap(NotFound) 323 | raise SFTPError(FX_FAILURE, 'Not Found') 324 | 325 | d = directory.get_full_listing() 326 | d.addCallback(cb) 327 | d.addErrback(errback) 328 | return d 329 | 330 | def getAttrs(self, fullpath, followLinks=False): 331 | """ Get attributes for an Object/Container 332 | 333 | :param str fullpath: path to the directory 334 | :param bool followLinks: whether or not to follow links (not used) 335 | 336 | """ 337 | self.log_command('getAttrs', fullpath) 338 | d = self.swiftfilesystem.getAttrs(fullpath) 339 | 340 | def cb(result): 341 | return self.format_attrs(result) 342 | 343 | def errback(failure): 344 | failure.trap(NotFound) 345 | raise SFTPError(FX_NO_SUCH_FILE, 'Not Found') 346 | 347 | d.addCallback(cb) 348 | d.addErrback(errback) 349 | 350 | return d 351 | 352 | def format_attrs(self, result): 353 | """ Helper for formatting getAttr results """ 354 | s = swift_stat(**result) 355 | return { 356 | "size": s.st_size, 357 | "uid": s.st_uid, 358 | "gid": s.st_gid, 359 | "permissions": s.st_mode, 360 | "atime": int(s.st_atime), 361 | "mtime": int(s.st_mtime) 362 | } 363 | 364 | def setAttrs(self, path, attrs): 365 | """ Set attributes on a container/object. No-Op """ 366 | return 367 | 368 | def readLink(self, path): 369 | """ No-Op """ 370 | raise NotImplementedError 371 | 372 | def makeLink(self, linkPath, targetPath): 373 | """ No-Op """ 374 | raise NotImplementedError 375 | 376 | def realPath(self, path): 377 | """ Normalizes a filepath """ 378 | container, obj = obj_to_path(path) 379 | real_path = '/' 380 | if container: 381 | real_path += container 382 | if obj: 383 | real_path += '/' + obj 384 | return real_path 385 | 386 | def extendedRequest(self, extName, extData): 387 | """ No-op """ 388 | raise NotImplementedError 389 | 390 | components.registerAdapter( 391 | SFTPServerForSwiftConchUser, SwiftSFTPUser, ISFTPServer) 392 | components.registerAdapter(SwiftSession, SwiftSFTPUser, ISession) 393 | -------------------------------------------------------------------------------- /swftp/sftp/service.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file defines what is required for swftp-sftp to work with twistd. 3 | 4 | See COPYING for license information. 5 | """ 6 | from swftp import VERSION 7 | from swftp.logging import StdOutObserver 8 | from swftp.sftp.server import SwiftSSHServerTransport 9 | 10 | from twisted.application import internet, service 11 | from twisted.python import usage, log 12 | from twisted.internet import reactor 13 | 14 | import ConfigParser 15 | import signal 16 | import os 17 | import sys 18 | import time 19 | 20 | 21 | CONFIG_DEFAULTS = { 22 | 'auth_url': 'http://127.0.0.1:8080/auth/v1.0', 23 | 'host': '0.0.0.0', 24 | 'port': '5022', 25 | 26 | 'rewrite_storage_scheme': '', 27 | 'rewrite_storage_netloc': '', 28 | 29 | 'priv_key': '/etc/swftp/id_rsa', 30 | 'pub_key': '/etc/swftp/id_rsa.pub', 31 | 'num_persistent_connections': '100', 32 | 'num_connections_per_session': '10', 33 | 'connection_timeout': '240', 34 | 'sessions_per_user': '10', 35 | 'extra_headers': '', 36 | 'verbose': 'false', 37 | 38 | 'log_statsd_host': '', 39 | 'log_statsd_port': '8125', 40 | 'log_statsd_sample_rate': '10.0', 41 | 'log_statsd_metric_prefix': 'swftp.sftp', 42 | 43 | 'stats_host': '', 44 | 'stats_port': '38022', 45 | 46 | # ordered by performance 47 | 'chiphers': 'blowfish-cbc,aes128-cbc,aes192-cbc,cast128-cbc,aes128-ctr,' 48 | 'aes256-cbc,aes192-ctr,aes256-ctr,3des-cbc', 49 | 'macs': 'hmac-md5, hmac-sha1', 50 | 'compressions': 'none, zlib', 51 | } 52 | 53 | 54 | def run(): 55 | options = Options() 56 | try: 57 | options.parseOptions(sys.argv[1:]) 58 | except usage.UsageError as errortext: 59 | print '%s: %s' % (sys.argv[0], errortext) 60 | print '%s: Try --help for usage details.' % (sys.argv[0]) 61 | sys.exit(1) 62 | 63 | # Start Logging 64 | obs = StdOutObserver() 65 | obs.start() 66 | 67 | s = makeService(options) 68 | s.startService() 69 | reactor.run() 70 | 71 | 72 | def parse_config_list(conf_name, conf_value, valid_options_list): 73 | lst, lst_not = [], [] 74 | for ch in conf_value.split(","): 75 | ch = ch.strip() 76 | if ch in valid_options_list: 77 | lst.append(ch) 78 | else: 79 | lst_not.append(ch) 80 | 81 | if lst_not: 82 | log.msg( 83 | "Unsupported {}: {}".format(conf_name, ", ".join(lst_not))) 84 | return lst 85 | 86 | 87 | def get_config(config_path, overrides): 88 | c = ConfigParser.ConfigParser(CONFIG_DEFAULTS) 89 | c.add_section('sftp') 90 | if config_path: 91 | log.msg('Reading configuration from path: %s' % config_path) 92 | c.read(config_path) 93 | else: 94 | config_paths = [ 95 | '/etc/swftp/swftp.conf', 96 | os.path.expanduser('~/.swftp.cfg') 97 | ] 98 | log.msg('Reading configuration from paths: %s' % config_paths) 99 | c.read(config_paths) 100 | for k, v in overrides.iteritems(): 101 | if v: 102 | c.set('sftp', k, str(v)) 103 | 104 | # Parse Chipher List 105 | chiphers = parse_config_list('ciphers', 106 | c.get('sftp', 'chiphers'), 107 | SwiftSSHServerTransport.supportedCiphers) 108 | c.set('sftp', 'chiphers', chiphers) 109 | 110 | # Parse Mac List 111 | macs = parse_config_list('macs', 112 | c.get('sftp', 'macs'), 113 | SwiftSSHServerTransport.supportedMACs) 114 | c.set('sftp', 'macs', macs) 115 | 116 | # Parse Compression List 117 | compressions = parse_config_list( 118 | 'compressions', c.get('sftp', 'compressions'), 119 | SwiftSSHServerTransport.supportedCompressions) 120 | c.set('sftp', 'compressions', compressions) 121 | 122 | return c 123 | 124 | 125 | class Options(usage.Options): 126 | "Defines Command-line options for the swftp-sftp service" 127 | optFlags = [ 128 | ["verbose", "v", "Make the server more talkative"] 129 | ] 130 | optParameters = [ 131 | ["config_file", "c", None, "Location of the swftp config file."], 132 | ["auth_url", "a", None, 133 | "Auth Url to use. Defaults to the config file value if it exists." 134 | "[default: http://127.0.0.1:8080/auth/v1.0]"], 135 | ["port", "p", None, "Port to bind to."], 136 | ["host", "h", None, "IP to bind to."], 137 | ["priv_key", "priv-key", None, "Private Key Location."], 138 | ["pub_key", "pub-key", None, "Public Key Location."], 139 | ] 140 | 141 | 142 | def makeService(options): 143 | """ 144 | Makes a new swftp-sftp service. The only option is the config file 145 | location. See CONFIG_DEFAULTS for list of configuration options. 146 | """ 147 | from twisted.conch.ssh.connection import SSHConnection 148 | from twisted.conch.ssh.factory import SSHFactory 149 | from twisted.conch.ssh.keys import Key 150 | from twisted.cred.portal import Portal 151 | 152 | from swftp.realm import SwftpRealm 153 | from swftp.sftp.server import SwiftSSHUserAuthServer 154 | from swftp.auth import SwiftBasedAuthDB 155 | from swftp.utils import ( 156 | log_runtime_info, GLOBAL_METRICS, parse_key_value_config) 157 | 158 | c = get_config(options['config_file'], options) 159 | 160 | sftp_service = service.MultiService() 161 | 162 | # ensure timezone is GMT 163 | os.environ['TZ'] = 'GMT' 164 | time.tzset() 165 | 166 | print('Starting SwFTP-sftp %s' % VERSION) 167 | 168 | # Add statsd service 169 | if c.get('sftp', 'log_statsd_host'): 170 | try: 171 | from swftp.statsd import makeService as makeStatsdService 172 | makeStatsdService( 173 | c.get('sftp', 'log_statsd_host'), 174 | c.getint('sftp', 'log_statsd_port'), 175 | sample_rate=c.getfloat('sftp', 'log_statsd_sample_rate'), 176 | prefix=c.get('sftp', 'log_statsd_metric_prefix') 177 | ).setServiceParent(sftp_service) 178 | except ImportError: 179 | sys.stderr.write('Missing Statsd Module. Requires "txstatsd" \n') 180 | 181 | if c.get('sftp', 'stats_host'): 182 | from swftp.report import makeService as makeReportService 183 | known_fields = [ 184 | 'command.login', 185 | 'command.logout', 186 | 'command.gotVersion', 187 | 'command.openFile', 188 | 'command.removeFile', 189 | 'command.renameFile', 190 | 'command.makeDirectory', 191 | 'command.removeDirectory', 192 | 'command.openDirectory', 193 | 'command.getAttrs', 194 | ] + GLOBAL_METRICS 195 | makeReportService( 196 | c.get('sftp', 'stats_host'), 197 | c.getint('sftp', 'stats_port'), 198 | known_fields=known_fields 199 | ).setServiceParent(sftp_service) 200 | 201 | authdb = SwiftBasedAuthDB( 202 | c.get('sftp', 'auth_url'), 203 | global_max_concurrency=c.getint('sftp', 'num_persistent_connections'), 204 | max_concurrency=c.getint('sftp', 'num_connections_per_session'), 205 | timeout=c.getint('sftp', 'connection_timeout'), 206 | extra_headers=parse_key_value_config(c.get('sftp', 'extra_headers')), 207 | verbose=c.getboolean('sftp', 'verbose'), 208 | rewrite_scheme=c.get('sftp', 'rewrite_storage_scheme'), 209 | rewrite_netloc=c.get('sftp', 'rewrite_storage_netloc'), 210 | ) 211 | 212 | realm = SwftpRealm() 213 | sftpportal = Portal(realm) 214 | sftpportal.registerChecker(authdb) 215 | 216 | sshfactory = SSHFactory() 217 | protocol = SwiftSSHServerTransport 218 | protocol.maxConnectionsPerUser = c.getint('sftp', 'sessions_per_user') 219 | protocol.supportedCiphers = c.get('sftp', 'chiphers') 220 | protocol.supportedMACs = c.get('sftp', 'macs') 221 | protocol.supportedCompressions = c.get('sftp', 'compressions') 222 | sshfactory.protocol = protocol 223 | sshfactory.noisy = False 224 | sshfactory.portal = sftpportal 225 | sshfactory.services['ssh-userauth'] = SwiftSSHUserAuthServer 226 | sshfactory.services['ssh-connection'] = SSHConnection 227 | 228 | pub_key_string = file(c.get('sftp', 'pub_key')).read() 229 | priv_key_string = file(c.get('sftp', 'priv_key')).read() 230 | sshfactory.publicKeys = { 231 | 'ssh-rsa': Key.fromString(data=pub_key_string)} 232 | sshfactory.privateKeys = { 233 | 'ssh-rsa': Key.fromString(data=priv_key_string)} 234 | 235 | signal.signal(signal.SIGUSR1, log_runtime_info) 236 | signal.signal(signal.SIGUSR2, log_runtime_info) 237 | 238 | internet.TCPServer( 239 | c.getint('sftp', 'port'), 240 | sshfactory, 241 | interface=c.get('sftp', 'host')).setServiceParent(sftp_service) 242 | 243 | return sftp_service 244 | -------------------------------------------------------------------------------- /swftp/sftp/swiftdirectory.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | from twisted.conch import ls 5 | 6 | from swftp.utils import OrderedDict 7 | from swftp.swiftfilesystem import swift_stat 8 | 9 | 10 | class SwiftDirectory(object): 11 | "Swift Directory is an iterator that returns a listing of the directory." 12 | def __init__(self, swiftfilesystem, fullpath): 13 | self.swiftfilesystem = swiftfilesystem 14 | self.fullpath = fullpath 15 | # A lot of clients require . and .. to be within the directory listing 16 | self.files = OrderedDict( 17 | [ 18 | ('.', {}), 19 | ('..', {}), 20 | ]) 21 | 22 | def get_full_listing(self): 23 | "Populate the directory listing." 24 | def cb(results): 25 | for k, v in results.iteritems(): 26 | self.files[k] = v 27 | 28 | d = self.swiftfilesystem.get_full_listing(self.fullpath) 29 | d.addCallback(cb) 30 | return d 31 | 32 | def __iter__(self): 33 | return self 34 | 35 | def next(self): 36 | try: 37 | name, f = self.files.popitem(last=False) 38 | lstat = swift_stat(**f) 39 | longname = ls.lsLine(name, lstat) 40 | return (name, longname, { 41 | "size": lstat.st_size, 42 | "uid": lstat.st_uid, 43 | "gid": lstat.st_gid, 44 | "permissions": lstat.st_mode, 45 | "atime": int(lstat.st_atime), 46 | "mtime": int(lstat.st_mtime) 47 | }) 48 | except KeyError: 49 | raise StopIteration 50 | 51 | def close(self): 52 | self.files = [] 53 | self.offset = 0 54 | -------------------------------------------------------------------------------- /swftp/sftp/swiftfile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Deals with file upload and download. This streams to and from the OpenStack 3 | Swift server. 4 | 5 | See COPYING for license information. 6 | """ 7 | from zope import interface 8 | 9 | from twisted.internet import defer, task, reactor 10 | from twisted.conch.ssh.filetransfer import ( 11 | FXF_CREAT, FXF_TRUNC, SFTPError, FX_NO_SUCH_FILE, FX_FAILURE, 12 | FX_CONNECTION_LOST) 13 | from twisted.conch.interfaces import ISFTPFile 14 | from twisted.internet.protocol import Protocol 15 | from twisted.internet.interfaces import IPushProducer 16 | from twisted.internet.error import ConnectionLost 17 | from twisted.python import log 18 | 19 | from swftp.swift import NotFound 20 | 21 | 22 | def cb_log_egress_bytes(result): 23 | if result: 24 | log.msg(metric='transfer.egress_bytes', count=len(result)) 25 | return result 26 | 27 | 28 | class SwiftFileReceiver(Protocol): 29 | "Streams data from Swift user to SFTP session" 30 | download_buffer_limit = 1024 * 1024 31 | upload_buffer_limit = 1024 * 1024 32 | 33 | def __init__(self, size, session): 34 | self.size = size 35 | self.session = session 36 | self.finished = defer.Deferred() 37 | self.done = False 38 | self.consume_paused = False 39 | 40 | self._offset = 0 41 | self._recv_buffer = "" 42 | self._recv_listeners = [] 43 | self.transport = None 44 | 45 | def dataReceived(self, _bytes): 46 | """ 47 | Data has been received from Swift. Pauses Swift if the 48 | download_buffer_limit has been reached. 49 | """ 50 | self._recv_buffer += _bytes 51 | self._readloop() 52 | if len(self._recv_buffer) > self.download_buffer_limit: 53 | self.consume_paused = True 54 | self.transport.pauseProducing() 55 | 56 | def _checksessionbuffertimer(self): 57 | """ 58 | Checks session buffer to see if we need to resume. 59 | Reschedules itself if the buffer is still not small enough 60 | """ 61 | if not self.consume_paused: 62 | return 63 | if len(self.session.buf) <= self.upload_buffer_limit: 64 | self.consume_paused = False 65 | self.transport.resumeProducing() 66 | else: 67 | reactor.callLater(5, self._checksessionbuffertimer) 68 | 69 | def _checksessionbuffer(self): 70 | "Checks buffer size to see if we need to pause" 71 | if not self.transport: 72 | return 73 | if self.consume_paused: 74 | return 75 | if len(self.session.buf) > self.upload_buffer_limit: 76 | self.consume_paused = True 77 | self.transport.pauseProducing() 78 | reactor.callLater(0, self._checksessionbuffertimer) 79 | 80 | def _readloop(self): 81 | """ 82 | The loop that checks to see if there is enough data to give back to 83 | the SFTP client. 84 | """ 85 | self._checksessionbuffer() 86 | for callback in self._recv_listeners: 87 | d, _, length = callback 88 | if len(self._recv_buffer) >= length: 89 | data = self._recv_buffer[:length] 90 | d.callback(data) 91 | self._recv_listeners.remove(callback) 92 | self._offset += len(data) 93 | self._recv_buffer = self._recv_buffer[length:] 94 | 95 | if self.consume_paused and \ 96 | len(self._recv_buffer) <= self.download_buffer_limit: 97 | self.consume_paused = False 98 | self.transport.resumeProducing() 99 | else: 100 | break 101 | 102 | def read(self, offset, length): 103 | """ 104 | Register the fact that this session wants a slice of data 105 | described by the given offset/length. Returns a deferred that fires 106 | with the data once it is available. 107 | """ 108 | if offset + length > self.size: 109 | length = self.size - offset 110 | 111 | def cb(result): 112 | if result is None: 113 | raise EOFError("EOF") 114 | return result 115 | 116 | # It looks like the SFTP client is asking for too much. 117 | if self.done and len(self._recv_buffer) == 0: 118 | raise EOFError("EOF") 119 | 120 | d = defer.Deferred() 121 | d.addCallback(cb) 122 | self._recv_listeners.append((d, offset, length)) 123 | self._readloop() 124 | return d 125 | 126 | def connectionLost(self, reason): 127 | """ 128 | For some reason, the HTTP connection has been lost. We can either 129 | be done reading from Swift or something back could have happened. 130 | """ 131 | from twisted.web._newclient import ResponseDone 132 | from twisted.web.http import PotentialDataLoss 133 | 134 | self.done = True 135 | 136 | if reason.check(ResponseDone) or reason.check(PotentialDataLoss): 137 | self._readloop() 138 | for callback in self._recv_listeners: 139 | d, _, _ = callback 140 | d.errback(reason) 141 | self._recv_listeners = [] 142 | self.finished.callback(None) 143 | else: 144 | for callback in self._recv_listeners: 145 | d, _, _ = callback 146 | d.errback(SFTPError(FX_CONNECTION_LOST, 'Connection Lost')) 147 | self._recv_listeners = [] 148 | self.finished.errback(reason) 149 | 150 | 151 | class SwiftFileSender(object): 152 | "Streams data from SFTP user to Swift" 153 | interface.implements(IPushProducer) 154 | max_buffer_writes = 20 155 | buffer_writes_resume = 5 156 | 157 | def __init__(self, swiftfilesystem, fullpath, session): 158 | self.swiftfilesystem = swiftfilesystem 159 | self.fullpath = fullpath 160 | self.session = session 161 | 162 | self.write_finished = None # Deferred that fires when finished writing 163 | self._task = None # Task loop 164 | self._done_sending = False # Set to True when the user closes the file 165 | self._writeBuffer = [] 166 | 167 | self.paused = False 168 | self.started = False 169 | 170 | def pauseProducing(self): 171 | self._task.pause() 172 | 173 | def resumeProducing(self): 174 | self._task.resume() 175 | 176 | def stopProducing(self): 177 | if self._task: 178 | try: 179 | self._task.stop() 180 | except task.TaskStopped: 181 | pass 182 | 183 | for buf in self._writeBuffer: 184 | d, _ = buf 185 | d.errback(SFTPError(FX_CONNECTION_LOST, 'Connection Lost')) 186 | self._writeBuffer.remove(buf) 187 | self._writeBuffer = [] 188 | 189 | def _writeFlusher(self, writer): 190 | while True: 191 | if self._done_sending and len(self._writeBuffer) == 0: 192 | writer.unregisterProducer() 193 | break 194 | 195 | if len(self._writeBuffer) == 0: 196 | yield 197 | continue 198 | 199 | try: 200 | d, data = self._writeBuffer.pop(0) 201 | writer.write(data) 202 | d.callback(len(data)) 203 | self._checkBuffer() 204 | yield 205 | except IndexError: 206 | pass 207 | finally: 208 | yield 209 | 210 | def _checkBuffer(self): 211 | if self.paused and len(self._writeBuffer) < self.buffer_writes_resume: 212 | self.session.conn.transport.transport.resumeProducing() 213 | self.paused = False 214 | elif not self.paused \ 215 | and len(self._writeBuffer) > self.max_buffer_writes: 216 | self.session.conn.transport.transport.pauseProducing() 217 | self.paused = True 218 | 219 | def cb_start_task(self, writer): 220 | self._task = task.cooperate(self._writeFlusher(writer)) 221 | 222 | def close(self): 223 | self._done_sending = True 224 | return self.write_finished 225 | 226 | def write(self, data): 227 | if not self.started: 228 | # If we haven't started uploading to Swift, start up that process 229 | self.write_finished, writer = \ 230 | self.swiftfilesystem.startFileUpload(self.fullpath) 231 | writer.registerProducer(self, streaming=True) 232 | writer.started.addCallback(self.cb_start_task) 233 | self.started = True 234 | d = defer.Deferred() 235 | self._writeBuffer.append((d, data)) 236 | self._checkBuffer() 237 | return d 238 | 239 | 240 | class SwiftFile(object): 241 | "Acts as an open file for the SFTP Server instance" 242 | interface.implements(ISFTPFile) 243 | 244 | def __init__(self, server, fullpath, flags=None, attrs=None): 245 | self.server = server 246 | self.swiftfilesystem = server.swiftfilesystem 247 | self.fullpath = fullpath 248 | self.flags = flags 249 | self.attrs = attrs 250 | self.r = None 251 | self.w = None 252 | self.props = None 253 | self.session = None # Set later 254 | 255 | def checkExistance(self): 256 | """ 257 | Checks whether or not the file exists. If the file flags specify, 258 | it will create the file and return a deffered with that has been 259 | completed. 260 | """ 261 | d = self.swiftfilesystem.checkFileExistance(self.fullpath) 262 | 263 | def cb(props): 264 | self.props = props 265 | 266 | def errback(failure): 267 | failure.trap(NotFound) 268 | if self.flags & FXF_CREAT == FXF_CREAT: 269 | return self.swiftfilesystem.touchFile(self.fullpath) 270 | if self.flags & FXF_TRUNC == FXF_TRUNC: 271 | return self.swiftfilesystem.touchFile(self.fullpath) 272 | else: 273 | raise SFTPError(FX_NO_SUCH_FILE, 'File Not Found') 274 | 275 | d.addCallback(cb) 276 | d.addErrback(errback) 277 | return d 278 | 279 | # New Writer Methods 280 | def close(self): 281 | " Returns a deferred that fires when the connection is closed " 282 | if self.w: 283 | d = defer.maybeDeferred(self.w.close) 284 | d.addErrback(self._errClose) 285 | return d 286 | del self.session 287 | 288 | def _errClose(self, failure): 289 | failure.trap(ConnectionLost, NotFound) 290 | if failure.check(ConnectionLost): 291 | raise SFTPError(FX_CONNECTION_LOST, "Connection Lost") 292 | elif failure.check(NotFound): 293 | raise SFTPError(FX_FAILURE, "Container Doesn't Exist") 294 | 295 | def writeChunk(self, offset, data): 296 | if not self.w: 297 | self.w = SwiftFileSender( 298 | self.swiftfilesystem, self.fullpath, self.session) 299 | 300 | d = self.w.write(data) 301 | 302 | def errback(failure): 303 | raise SFTPError(FX_FAILURE, 'Upload Failure') 304 | d.addErrback(errback) 305 | 306 | def cb(result): 307 | return result 308 | d.addCallback(cb) 309 | return d 310 | 311 | # Reading Methods 312 | def readChunk(self, offset, length): 313 | if not self.r: 314 | self.r = SwiftFileReceiver(int(self.props['size']), self.session) 315 | self.swiftfilesystem.startFileDownload( 316 | self.fullpath, self.r, offset=offset) 317 | d = self.r.read(offset, length) 318 | d.addCallback(cb_log_egress_bytes) 319 | return d 320 | 321 | def getAttrs(self): 322 | return self.server.getAttrs(self.fullpath) 323 | 324 | def setAttrs(self, attrs): 325 | raise NotImplementedError 326 | -------------------------------------------------------------------------------- /swftp/statsd.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | from twisted.internet import reactor 5 | from txstatsd.client import TwistedStatsDClient, StatsDClientProtocol 6 | from txstatsd.metrics.metrics import Metrics 7 | from txstatsd.process import PROCESS_STATS, COUNTER_STATS 8 | from txstatsd.report import ReportingService 9 | 10 | from swftp.utils import MetricCollector 11 | 12 | 13 | def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''): 14 | client = TwistedStatsDClient(host, port) 15 | metrics = Metrics(connection=client, namespace=prefix) 16 | reporting = ReportingService() 17 | 18 | for report in PROCESS_STATS: 19 | reporting.schedule(report, sample_rate, metrics.gauge) 20 | 21 | for report in COUNTER_STATS: 22 | reporting.schedule(report, sample_rate, metrics.gauge) 23 | 24 | # Attach log observer to collect metrics for us 25 | metric_collector = MetricCollector() 26 | metric_collector.start() 27 | 28 | metric_reporter = MetricReporter(metrics, metric_collector) 29 | reporting.schedule(metric_reporter.report_metrics, sample_rate, None) 30 | 31 | protocol = StatsDClientProtocol(client) 32 | reactor.listenUDP(0, protocol) 33 | return reporting 34 | 35 | 36 | class MetricReporter(object): 37 | def __init__(self, metric, collector): 38 | self.metric = metric 39 | self.collector = collector 40 | 41 | def report_metrics(self): 42 | # Report collected metrics 43 | results = self.collector.current 44 | for name, value in results.items(): 45 | self.metric.increment(name, value) 46 | self.collector.sample() 47 | -------------------------------------------------------------------------------- /swftp/swift.py: -------------------------------------------------------------------------------- 1 | """ 2 | swift includes a basic swift client for twisted. 3 | 4 | client = SwiftConnection(auth_url, username, api_key, pool=pool) 5 | d = client.put_object('container', 'path/to/obj') 6 | 7 | See COPYING for license information. 8 | """ 9 | from twisted.internet import reactor 10 | from twisted.internet.defer import Deferred, succeed 11 | from twisted.web.client import Agent, WebClientContextFactory 12 | from twisted.internet.protocol import Protocol 13 | from twisted.web.http_headers import Headers 14 | from twisted.web import error 15 | from twisted.web._newclient import ResponseDone 16 | from twisted.web.http import PotentialDataLoss 17 | from twisted.python import log 18 | 19 | import json 20 | from urllib import quote as _quote 21 | 22 | 23 | class RequestError(error.Error): 24 | pass 25 | 26 | 27 | class NotFound(RequestError): 28 | pass 29 | 30 | 31 | class UnAuthenticated(RequestError): 32 | pass 33 | 34 | 35 | class UnAuthorized(RequestError): 36 | pass 37 | 38 | 39 | class Conflict(RequestError): 40 | pass 41 | 42 | 43 | class ResponseReceiver(Protocol): 44 | """ 45 | Assembles HTTP response from return stream. 46 | """ 47 | 48 | def __init__(self, finished): 49 | self.recv_chunks = [] 50 | self.finished = finished 51 | 52 | def dataReceived(self, _bytes): 53 | self.recv_chunks.append(_bytes) 54 | 55 | def connectionLost(self, reason): 56 | if reason.check(ResponseDone) or reason.check(PotentialDataLoss): 57 | self.finished.callback(''.join(self.recv_chunks)) 58 | else: 59 | self.finished.errback(reason) 60 | 61 | 62 | class ResponseIgnorer(Protocol): 63 | def __init__(self, finished): 64 | self.finished = finished 65 | 66 | def makeConnection(self, transport): 67 | transport.stopProducing() 68 | self.finished.callback(None) 69 | 70 | def dataReceived(self, _bytes): 71 | pass 72 | 73 | def connectionLost(self, reason): 74 | pass 75 | 76 | 77 | def cb_recv_resp(response, load_body=False, receiver=None): 78 | d_resp_recvd = Deferred() 79 | if response.code == 204: 80 | response.deliverBody(ResponseIgnorer(d_resp_recvd)) 81 | elif load_body: 82 | response.deliverBody(ResponseReceiver(d_resp_recvd)) 83 | else: 84 | if receiver: 85 | response.deliverBody(receiver) 86 | return response 87 | else: 88 | response.deliverBody(ResponseIgnorer(d_resp_recvd)) 89 | d_resp_recvd.addCallback(cb_process_resp, response) 90 | return d_resp_recvd 91 | 92 | 93 | def cb_process_resp(body, response): 94 | if response.code == 404: 95 | raise NotFound(response.code, body) 96 | if response.code == 401: 97 | raise UnAuthenticated(response.code, body) 98 | if response.code == 403: 99 | raise UnAuthorized(response.code, body) 100 | if response.code == 409: 101 | raise Conflict(response.code, body) 102 | elif response.code > 299 and response.code < 400: 103 | raise error.PageRedirect(response.code, body) 104 | elif response.code > 399: 105 | raise RequestError(response.code, body) 106 | headers = {} 107 | for k, v in response.headers.getAllRawHeaders(): 108 | headers[k.lower()] = v.pop() 109 | response.headers = headers 110 | return response, body 111 | 112 | 113 | def format_head_response(result): 114 | resp, _ = result 115 | return resp.headers 116 | 117 | 118 | def cb_json_decode(result): 119 | resp, body = result 120 | return resp, json.loads(body) 121 | 122 | 123 | class SwiftConnection(object): 124 | """ A basic connection class to interface with OpenStack Swift. 125 | 126 | :param auth_url: auth endpoint for swift 127 | :param username: username for swift 128 | :param api_key: password/api_key for swift 129 | :param pool: A twisted.web.client.HTTPConnectionPool object 130 | :param dict extra_headers: extra HTTP headers to send with each request 131 | :param bool verbose: verbose setting 132 | """ 133 | user_agent = 'Twisted Swift' 134 | 135 | def __init__(self, auth_url, username, api_key, pool=None, 136 | extra_headers=None, verbose=False): 137 | self.auth_url = auth_url 138 | self.username = username 139 | self.api_key = api_key 140 | self.storage_url = None 141 | self.auth_token = None 142 | contextFactory = WebClientContextFactory() 143 | contextFactory.noisy = False 144 | self.pool = pool 145 | self.agent = Agent(reactor, contextFactory, pool=self.pool) 146 | self.extra_headers = extra_headers 147 | self.verbose = verbose 148 | 149 | def _form_url(self, path, params): 150 | url = "/".join((self.storage_url, path)) 151 | if params: 152 | param_lst = [] 153 | for k, v in params.iteritems(): 154 | param_lst.append("%s=%s" % (k, v)) 155 | url = "%s?%s" % (url, "&".join(param_lst)) 156 | return url 157 | 158 | def make_request(self, method, path, params=None, headers=None, body=None): 159 | """ Make an HTTP request against Swift. This method will try once to 160 | re-authenticate to swift after receiving a 401 or 403 and then 161 | (if successful) will re-attempt the request. 162 | 163 | :param method: HTTP Method. E.G. GET, POST, PUT 164 | :param path: Path to be appended to the storage url 165 | :param dict params: Parameters to be used at the query parameter 166 | :param dict headers: Additional headers for the request 167 | :param body: Object which implements twisted.web.iweb.IBodyProducer 168 | 169 | :returns t.w.c.Response: 170 | 171 | """ 172 | h = { 173 | 'User-Agent': [self.user_agent], 174 | } 175 | if headers: 176 | for k, v in headers.iteritems(): 177 | h[k] = [v] 178 | 179 | if self.extra_headers: 180 | for k, v in self.extra_headers.iteritems(): 181 | h[k] = [v] 182 | 183 | def doRequest(ignored): 184 | h['X-Auth-Token'] = [self.auth_token] 185 | url = self._form_url(path, params) 186 | if self.verbose: 187 | log.msg('Request: %s %s, headers: %s' % (method, url, h)) 188 | return self.agent.request(method, url, Headers(h), body) 189 | 190 | d = doRequest(None) 191 | 192 | def retryAuth(response): 193 | if response.code in [401, 403]: 194 | d_resp_recvd = Deferred() 195 | response.deliverBody(ResponseIgnorer(d_resp_recvd)) 196 | d_resp_recvd.addCallback(self.cb_retry_auth) 197 | d_resp_recvd.addCallback(doRequest) 198 | return d_resp_recvd 199 | return response 200 | d.addCallback(retryAuth) 201 | 202 | return d 203 | 204 | def cb_retry_auth(self, ignored): 205 | return self.authenticate() 206 | 207 | def after_authenticate(self, result): 208 | response, _ = result 209 | self.storage_url = response.headers['x-storage-url'] 210 | self.auth_token = response.headers['x-auth-token'] 211 | return result 212 | 213 | def authenticate(self): 214 | """ Authenticate against Swift (using v1 auth) 215 | 216 | :returns t.w.c.Response: 217 | 218 | """ 219 | h = { 220 | 'User-Agent': [self.user_agent], 221 | 'X-Auth-User': [self.username], 222 | 'X-Auth-Key': [self.api_key], 223 | } 224 | 225 | if self.extra_headers: 226 | for k, v in self.extra_headers.iteritems(): 227 | h[k] = [v] 228 | 229 | d = self.agent.request('GET', self.auth_url, Headers(h)) 230 | d.addCallback(cb_recv_resp, load_body=True) 231 | d.addCallback(self.after_authenticate) 232 | return d 233 | 234 | def head_account(self): 235 | " Get details of the account " 236 | d = self.make_request('HEAD', '') 237 | d.addCallback(cb_recv_resp) 238 | d.addCallback(format_head_response) 239 | return d 240 | 241 | def get_account(self, limit=None, marker=None, end_marker=None): 242 | """ Get listing of containers in the account 243 | 244 | :param int limit: The max number of results to return 245 | :param marker: container names greater than this value 246 | :param end_marker: container names less than this value 247 | 248 | :returns t.w.c.Response, list: 249 | 250 | """ 251 | params = {'format': 'json'} 252 | if limit: 253 | params['limit'] = str(limit) 254 | if marker: 255 | params['marker'] = quote(marker) 256 | if marker: 257 | params['end_marker'] = quote(end_marker) 258 | 259 | d = self.make_request('GET', '', params=params) 260 | d.addCallback(cb_recv_resp, load_body=True) 261 | d.addCallback(cb_json_decode) 262 | return d 263 | 264 | def head_container(self, container): 265 | """ Get details on a container 266 | 267 | :param container: The container name 268 | 269 | :returns dict: 270 | 271 | """ 272 | d = self.make_request('HEAD', quote(container)) 273 | d.addCallback(cb_recv_resp) 274 | d.addCallback(format_head_response) 275 | return d 276 | 277 | def get_container(self, container, limit=None, marker=None, 278 | end_marker=None, prefix=None, path=None, delimiter=None): 279 | """ Create a container 280 | 281 | :param container: The container name 282 | :param int limit: The max number of results to return 283 | :param marker: object names greater than this value 284 | :param end_marker: object names less than this value 285 | :param prefix: return objects with names that start with this value 286 | :param delimiter: Delimiter to use for hierarchy 287 | 288 | :returns t.w.c.Response, list: 289 | 290 | """ 291 | params = {'format': 'json'} 292 | if limit: 293 | params['limit'] = str(limit) 294 | if marker: 295 | params['marker'] = quote(marker) 296 | if end_marker: 297 | params['end_marker'] = quote(end_marker) 298 | if prefix: 299 | params['prefix'] = quote(prefix) 300 | if path: 301 | params['path'] = quote(path) 302 | if delimiter: 303 | params['delimiter'] = quote(delimiter) 304 | 305 | d = self.make_request('GET', quote(container), params=params) 306 | d.addCallback(cb_recv_resp, load_body=True) 307 | d.addCallback(cb_json_decode) 308 | return d 309 | 310 | def put_container(self, container, headers=None): 311 | """ Create a container 312 | 313 | :param container: The container name 314 | :param header: Optional headers to add to the request 315 | 316 | :returns t.w.c.Response: 317 | 318 | """ 319 | d = self.make_request('PUT', quote(container), headers=headers) 320 | d.addCallback(cb_recv_resp) 321 | return d 322 | 323 | def delete_container(self, container): 324 | """ Delete a container 325 | 326 | :param container: The container name 327 | 328 | :returns t.w.c.Response: 329 | 330 | """ 331 | d = self.make_request('DELETE', quote(container)) 332 | d.addCallback(cb_recv_resp) 333 | return d 334 | 335 | def head_object(self, container, path): 336 | """ Get details about an object 337 | 338 | :param container: The container name 339 | :param path: The object name/path 340 | 341 | :returns dict: 342 | 343 | """ 344 | _path = "/".join((quote(container), quote(path))) 345 | d = self.make_request('HEAD', _path) 346 | d.addCallback(cb_recv_resp) 347 | d.addCallback(format_head_response) 348 | return d 349 | 350 | def get_object(self, container, path, headers=None, receiver=None): 351 | """ Download an object 352 | 353 | :param container: The container name 354 | :param path: The object name/path 355 | :param dict headers: Extra headers to use with the HTTP request 356 | :param receiver: A twisted.internet.protocol.Protocol that will receive 357 | the contents of the object 358 | 359 | :returns t.w.c.Response: 360 | 361 | """ 362 | _path = "/".join((quote(container), quote(path))) 363 | d = self.make_request('GET', _path, headers=headers) 364 | d.addCallback(cb_recv_resp, receiver=receiver) 365 | return d 366 | 367 | def put_object(self, container, path, headers=None, body=None): 368 | """ Create a new object 369 | 370 | :param container: The container name 371 | :param path: The object name/path 372 | :param dict headers: Extra headers to use with the HTTP request 373 | :param body: Object which implements twisted.web.iweb.IBodyProducer 374 | 375 | :returns t.w.c.Response: 376 | 377 | """ 378 | if not headers: 379 | headers = {} 380 | if not body: 381 | headers['Content-Length'] = '0' 382 | _path = "/".join((quote(container), quote(path))) 383 | d = self.make_request('PUT', _path, headers=headers, body=body) 384 | d.addCallback(cb_recv_resp, load_body=True) 385 | return d 386 | 387 | def delete_object(self, container, path): 388 | """ Delete an object 389 | 390 | :param container: The container name 391 | :param path: The object name/path 392 | 393 | :returns t.w.c.Response: 394 | 395 | """ 396 | _path = "/".join((quote(container), quote(path))) 397 | d = self.make_request('DELETE', _path) 398 | d.addCallback(cb_recv_resp) 399 | return d 400 | 401 | 402 | class ThrottledSwiftConnection(SwiftConnection): 403 | """ A SwiftConnection that has a list of locks that it needs to acquire 404 | before making requests. Locks can either be a DeferredSemaphore, a 405 | DeferredLock, or anything else that implements 406 | twisted.internet.defer._ConcurrencyPrimitive. Locks are acquired in the 407 | order in the list. 408 | 409 | :param locks: list of locks that implement 410 | twisted.internet.defer._ConcurrencyPrimitive 411 | :param \*args: same arguments as `SwiftConnection` 412 | :param \*\*args: same keyword arguments as `SwiftConnection` 413 | """ 414 | def __init__(self, locks, *args, **kwargs): 415 | SwiftConnection.__init__(self, *args, **kwargs) 416 | self.locks = locks or [] 417 | 418 | def _release_all(self, result): 419 | for lock in self.locks: 420 | lock.release() 421 | return result 422 | 423 | def _aquire_all(self): 424 | d = succeed(None) 425 | for lock in self.locks: 426 | d.addCallback(lambda r: lock.acquire()) 427 | return d 428 | 429 | def make_request(self, *args, **kwargs): 430 | def execute(ignored): 431 | d = SwiftConnection.make_request(self, *args, **kwargs) 432 | d.addBoth(self._release_all) 433 | return d 434 | 435 | d = self._aquire_all() 436 | d.addCallback(execute) 437 | return d 438 | 439 | 440 | def quote(value, safe='/'): 441 | """ 442 | Patched version of urllib.quote that encodes utf8 strings before quoting 443 | """ 444 | value = encode_utf8(value) 445 | if isinstance(value, str): 446 | return _quote(value, safe) 447 | else: 448 | return value 449 | 450 | 451 | def encode_utf8(value): 452 | if isinstance(value, unicode): 453 | value = value.encode('utf8') 454 | return value 455 | -------------------------------------------------------------------------------- /swftp/swiftfilesystem.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file includes a set of helpers that make treating Swift as a filesystem a 3 | bit easier. 4 | 5 | See COPYING for license information. 6 | """ 7 | import datetime 8 | import stat 9 | import os 10 | import urlparse 11 | import time 12 | 13 | from twisted.internet import defer, reactor, task 14 | from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH 15 | from twisted.internet.interfaces import IConsumer 16 | from twisted.python import log 17 | 18 | from zope import interface 19 | 20 | from swftp.utils import OrderedDict 21 | from swftp.utils import try_datetime_parse 22 | from swftp.swift import NotFound, Conflict 23 | 24 | 25 | def obj_to_path(path): 26 | " Convert an entire path to a (container, item) tuple " 27 | path = path.strip('/') 28 | path = urlparse.urljoin('/', path) 29 | path = path.strip('/') 30 | path_parts = path.split('/', 1) 31 | 32 | container = None 33 | if len(path_parts) > 0 and path_parts[0] != '': 34 | container = path_parts[0] 35 | item = None 36 | if len(path_parts) > 1: 37 | item = path_parts[1] 38 | 39 | return container, item 40 | 41 | 42 | def cb_parse_account_headers(headers): 43 | return { 44 | 'count': headers.get('x-account-container-count', 0), 45 | 'size': headers.get('x-account-bytes-used', 0), 46 | 'content_type': 'application/directory', 47 | } 48 | 49 | 50 | def cb_parse_container_headers(headers): 51 | return { 52 | 'count': headers.get('x-container-object-count', 0), 53 | 'size': headers.get('x-container-bytes-used', 0), 54 | 'content_type': 'application/directory', 55 | } 56 | 57 | 58 | def cb_parse_object_headers(headers): 59 | return { 60 | 'size': headers.get('content-length', 0), 61 | 'last_modified': headers.get('last-modified', 0), 62 | 'content_type': headers.get('content-type'), 63 | } 64 | 65 | 66 | def swift_stat(last_modified=None, content_type="application/directory", 67 | count=1, bytes=0, size=0, **kwargs): 68 | size = int(size) or int(bytes) 69 | mtime = try_datetime_parse(last_modified) 70 | if not mtime: 71 | mtime = time.mktime(datetime.datetime.utcnow().timetuple()) 72 | 73 | if content_type == "application/directory": 74 | mode = 0o700 | stat.S_IFDIR 75 | else: 76 | mode = 0o600 | stat.S_IFREG 77 | return os.stat_result((mode, 0, 0, count, 65535, 65535, size, mtime, 78 | mtime, mtime)) 79 | 80 | 81 | class SwiftWriteFile(object): 82 | """ Adapts IBodyProducer and IConsumer """ 83 | interface.implements(IBodyProducer, IConsumer) 84 | 85 | def __init__(self, length=None): 86 | self.length = length or UNKNOWN_LENGTH 87 | self.started = defer.Deferred() 88 | self.finished = defer.Deferred() 89 | self.consumer = None # is set later 90 | self.producer = None # is set later 91 | 92 | # IConsumer 93 | def registerProducer(self, producer, streaming): 94 | self.producer = producer 95 | assert streaming 96 | 97 | def unregisterProducer(self): 98 | self.finished.callback(None) 99 | 100 | def write(self, data): 101 | self.consumer.write(data) 102 | log.msg(metric='transfer.ingress_bytes', count=len(data)) 103 | 104 | # IBodyProducer 105 | def startProducing(self, consumer): 106 | self.consumer = consumer 107 | self.started.callback(self) 108 | return self.finished 109 | 110 | def pauseProducing(self): 111 | self.producer.pauseProducing() 112 | 113 | def resumeProducing(self): 114 | self.producer.resumeProducing() 115 | 116 | def stopProducing(self): 117 | self.producer.stopProducing() 118 | 119 | 120 | class SwiftFileSystem(object): 121 | "Defines a common interface used to create Swift similar to a filesystem" 122 | def __init__(self, swiftconn): 123 | self.swiftconn = swiftconn 124 | 125 | def startFileUpload(self, fullpath): 126 | "returns IConsumer to write to object data to" 127 | container, path = obj_to_path(fullpath) 128 | consumer = SwiftWriteFile() 129 | d = self.swiftconn.put_object(container, path, body=consumer) 130 | return d, consumer 131 | 132 | def startFileDownload(self, fullpath, consumer, offset=0): 133 | "consumer: Protocol" 134 | container, path = obj_to_path(fullpath) 135 | headers = {} 136 | if offset > 0: 137 | headers['Range'] = 'bytes=%s-' % offset 138 | d = self.swiftconn.get_object(container, path, receiver=consumer, 139 | headers=headers) 140 | return d 141 | 142 | def touchFile(self, fullpath): 143 | container, path = obj_to_path(fullpath) 144 | return self.swiftconn.put_object(container, path, body=None) 145 | 146 | def checkFileExistance(self, fullpath): 147 | container, path = obj_to_path(fullpath) 148 | if container is None or path is None: 149 | raise NotImplementedError 150 | 151 | d = self.swiftconn.head_object(container, path) 152 | d.addCallback(cb_parse_object_headers) 153 | return d 154 | 155 | def removeFile(self, fullpath): 156 | container, path = obj_to_path(fullpath) 157 | if container is None or path is None: 158 | raise NotImplementedError 159 | d = self.swiftconn.delete_object(container, path) 160 | return d 161 | 162 | @defer.inlineCallbacks 163 | def renameFile(self, oldpath, newpath): 164 | container, path = obj_to_path(oldpath) 165 | newcontainer, newpath = obj_to_path(newpath) 166 | if not container or not newcontainer: 167 | raise NotImplementedError 168 | 169 | if not path and not newpath: 170 | # Attempt to 'rename' a container (metadata is lost) 171 | yield self.swiftconn.delete_container(container) 172 | yield self.swiftconn.put_container(newcontainer) 173 | defer.returnValue(None) 174 | else: 175 | # If the object doesn't actually exist, ABORT 176 | path = path or '' 177 | newpath = newpath or '' 178 | try: 179 | yield self.swiftconn.head_object(container, path) 180 | except NotFound: 181 | raise NotImplementedError 182 | 183 | # List out children of this path. If there are any, ABORT 184 | prefix = None 185 | if path: 186 | prefix = "%s/" % path 187 | _, children = yield self.swiftconn.get_container( 188 | container, prefix=prefix, limit=1) 189 | if len(children) > 0: 190 | raise NotImplementedError 191 | 192 | # This is an actual object with no children. Free to rename. 193 | yield self.swiftconn.put_object( 194 | newcontainer, newpath, 195 | headers={'X-Copy-From': '%s/%s' % (container, path)}) 196 | yield self.swiftconn.delete_object(container, path) 197 | 198 | @defer.inlineCallbacks 199 | def getAttrs(self, fullpath): 200 | container, path = obj_to_path(fullpath) 201 | if path: 202 | try: 203 | headers = yield self.swiftconn.head_object(container, path) 204 | defer.returnValue( 205 | cb_parse_object_headers(headers)) 206 | except NotFound: 207 | prefix = None 208 | if path: 209 | prefix = "%s/" % path 210 | _, children = yield self.swiftconn.get_container( 211 | container, prefix=prefix, limit=1) 212 | if len(children) == 0: 213 | raise NotFound(404, 'Not Found') 214 | defer.returnValue( 215 | {'content_type': 'application/directory'}) 216 | 217 | elif container: 218 | headers = yield self.swiftconn.head_container(container) 219 | defer.returnValue(cb_parse_container_headers(headers)) 220 | else: 221 | headers = yield self.swiftconn.head_account() 222 | defer.returnValue(cb_parse_account_headers(headers)) 223 | 224 | def makeDirectory(self, fullpath, attrs=None): 225 | container, path = obj_to_path(fullpath) 226 | if path: 227 | headers = {'Content-Type': 'application/directory'} 228 | d = self.swiftconn.put_object(container, path, headers=headers) 229 | else: 230 | d = self.swiftconn.put_container(container) 231 | return d 232 | 233 | @defer.inlineCallbacks 234 | def removeDirectory(self, fullpath): 235 | container, path = obj_to_path(fullpath) 236 | if path: 237 | yield self.swiftconn.delete_object(container, path) 238 | else: 239 | try: 240 | yield self.swiftconn.delete_container(container) 241 | except Conflict: 242 | # Wait 2 seconds and try to delete the container once more 243 | yield task.deferLater( 244 | reactor, 2, self.swiftconn.delete_container, container) 245 | 246 | def get_full_listing(self, fullpath): 247 | """ 248 | Return a full listing of objects, collapsed into a directory 249 | structure. Works for account, container and object prefix listings. 250 | 251 | 252 | @returns dict of {name: property} values 253 | """ 254 | container, path = obj_to_path(fullpath) 255 | if container: 256 | return self.get_container_listing(container, path) 257 | else: 258 | return self.get_account_listing() 259 | 260 | def get_container_listing(self, container, path, marker=None, 261 | all_files=None): 262 | if all_files is None: 263 | all_files = OrderedDict() 264 | prefix = None 265 | if path: 266 | prefix = "%s/" % path 267 | d = self.swiftconn.get_container( 268 | container, prefix=prefix, delimiter='/', marker=marker) 269 | 270 | def cb(results): 271 | _, files = results 272 | next_marker = None 273 | for f in files: 274 | if 'subdir' in f: 275 | f['name'] = f['subdir'] 276 | f['content-type'] = 'application/directory' 277 | f['formatted_name'] = os.path.basename( 278 | f['name'].encode("utf-8").rstrip('/')) 279 | all_files[f['formatted_name']] = f 280 | next_marker = f['name'] 281 | if len(files) > 0: 282 | return self.get_container_listing( 283 | container, path, marker=next_marker, all_files=all_files) 284 | return all_files 285 | d.addCallback(cb) 286 | return d 287 | 288 | def get_account_listing(self, marker=None, all_files=None): 289 | if all_files is None: 290 | all_files = OrderedDict() 291 | d = self.swiftconn.get_account(marker=marker) 292 | 293 | def cb(results): 294 | _, files = results 295 | next_marker = None 296 | for f in files: 297 | f['content-type'] = 'application/directory' 298 | f['formatted_name'] = f['name'].encode("utf-8") 299 | all_files[f['formatted_name']] = f 300 | next_marker = f['name'] 301 | if len(files) > 0: 302 | return self.get_account_listing( 303 | marker=next_marker, all_files=all_files) 304 | return all_files 305 | d.addCallback(cb) 306 | return d 307 | -------------------------------------------------------------------------------- /swftp/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softlayer/swftp/6363985ed51c0d34b9d3aab9ead4f4246e93cd7b/swftp/test/__init__.py -------------------------------------------------------------------------------- /swftp/test/functional/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | import hashlib 5 | import unittest 6 | import os 7 | import uuid 8 | import ConfigParser 9 | import time 10 | 11 | from twisted.internet import defer 12 | from twisted.internet.defer import DeferredList 13 | from twisted.web.client import FileBodyProducer 14 | from swftp.swift import SwiftConnection, Conflict 15 | from swftp.swiftfilesystem import SwiftFileSystem 16 | 17 | utf8_chars = u'\uF10F\uD20D\uB30B\u9409\u8508\u5605\u3703\u1801'\ 18 | u'\u0900\uF110\uD20E\uB30C\u940A\u8509\u5606\u3704'\ 19 | u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\ 20 | u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ 21 | u'\u5608\u3706\u1804\u0903\u03A9\u2603' 22 | 23 | 24 | def get_config(): 25 | config_file = os.environ.get('SWFTP_TEST_CONFIG_FILE', 26 | '/etc/swftp/test.conf') 27 | section = 'func_test' 28 | config = ConfigParser.ConfigParser() 29 | config.read(config_file) 30 | 31 | config_dict = {} 32 | for option in config.options(section): 33 | config_dict[option] = config.get(section, option) 34 | 35 | return config_dict 36 | 37 | conf = get_config() 38 | 39 | 40 | def has_item(name, listing): 41 | return len(filter(lambda h: h['name'] == name, listing)) > 0 42 | 43 | 44 | def compute_md5(filepath): 45 | hsh = hashlib.md5() 46 | with open(filepath, 'rb') as f: 47 | for chunk in iter(lambda: f.read(128 * hsh.block_size), b''): 48 | hsh.update(chunk) 49 | return hsh.hexdigest() 50 | 51 | 52 | def create_test_file(tmpdir, size): 53 | hsh = hashlib.md5() 54 | filepath = os.path.join(tmpdir, uuid.uuid4().hex) 55 | with open(filepath, 'w+') as f: 56 | current_size = 0 57 | chunk_size = 100000 58 | while current_size < size: 59 | s = chunk_size 60 | if current_size + chunk_size > size: 61 | s = size - current_size 62 | data = os.urandom(s) 63 | hsh.update(data) 64 | f.write(data) 65 | current_size += s 66 | return filepath, hsh.hexdigest() 67 | 68 | 69 | def get_swift_client(config, pool=None): 70 | for key in 'account username password'.split(): 71 | if key not in config: 72 | raise unittest.SkipTest("%s not set in the test config file" % key) 73 | protocol = 'http' 74 | if config.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): 75 | protocol = 'https' 76 | host = config.get('auth_host', '127.0.0.1') 77 | port = config.get('auth_port', '8080') 78 | auth_prefix = config.get('auth_prefix', '/auth/') 79 | auth_url = '%s://%s:%s%sv1.0' % (protocol, host, port, auth_prefix) 80 | username = "%s:%s" % (config['account'], config['username']) 81 | api_key = config['password'] 82 | return SwiftConnection(auth_url, username, api_key, pool=pool) 83 | 84 | 85 | def upload_file(swift, container, path, src_path, md5): 86 | def cb(result): 87 | resp, body = result 88 | assert md5 == resp.headers['etag'] 89 | 90 | d = swift.put_object( 91 | container, path, body=FileBodyProducer(open(src_path))) 92 | d.addCallback(cb) 93 | return d 94 | 95 | 96 | @defer.inlineCallbacks 97 | def clean_swift(swift): 98 | yield swift.authenticate() 99 | yield remove_test_data(swift, 'sftp_tests') 100 | yield remove_test_data(swift, 'ftp_tests') 101 | 102 | 103 | @defer.inlineCallbacks 104 | def remove_test_data(swift, prefix): 105 | swift_fs = SwiftFileSystem(swift) 106 | time.sleep(2) 107 | containers = yield swift_fs.get_account_listing() 108 | sem = defer.DeferredSemaphore(200) 109 | for container in containers: 110 | if container.startswith(prefix): 111 | while True: 112 | objs = yield list_all_objects(swift, container) 113 | dl = [] 114 | for obj in objs: 115 | dl.append(sem.run( 116 | swift.delete_object, container, obj['name'])) 117 | # Wait till all objects are done deleting 118 | yield DeferredList(dl, fireOnOneErrback=True) 119 | try: 120 | # Delete the container 121 | yield swift.delete_container(container) 122 | break 123 | except Conflict: 124 | # Retry listing if there are still objects 125 | # (this can happen a lot since the container server isn't 126 | # guarenteed to be consistent) 127 | pass 128 | 129 | 130 | @defer.inlineCallbacks 131 | def list_all_objects(swift, container): 132 | all_objects = [] 133 | next_marker = None 134 | 135 | while True: 136 | _, files = yield swift.get_container(container, marker=next_marker) 137 | for f in files: 138 | all_objects.append(f) 139 | next_marker = f['name'] 140 | if len(files) == 0: 141 | break 142 | defer.returnValue(all_objects) 143 | 144 | 145 | class RandFile(object): 146 | def __init__(self, size): 147 | self.size = size 148 | self.offset = 0 149 | self.hash = hashlib.md5() 150 | 151 | def computed_hash(self): 152 | return self.hash.hexdigest() 153 | 154 | def read(self, length): 155 | if self.offset + length > self.size: 156 | length = self.size - self.offset 157 | data = os.urandom(length) 158 | self.hash.update(data) 159 | self.offset += len(length) 160 | return data 161 | -------------------------------------------------------------------------------- /swftp/test/functional/test_ftp.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | from twisted.trial import unittest 5 | from twisted.internet import defer, reactor 6 | from twisted.web.client import HTTPConnectionPool 7 | 8 | import ftplib 9 | import tempfile 10 | import shutil 11 | import time 12 | import os 13 | 14 | from . import get_config, has_item, create_test_file, clean_swift, \ 15 | compute_md5, upload_file, utf8_chars, get_swift_client 16 | 17 | conf = get_config() 18 | 19 | 20 | class FTPFuncTest(unittest.TestCase): 21 | @defer.inlineCallbacks 22 | def setUp(self): 23 | self.pool = HTTPConnectionPool(reactor, persistent=True) 24 | self.swift = get_swift_client(conf, pool=self.pool) 25 | self.tmpdir = tempfile.mkdtemp() 26 | self.ftp = self.get_ftp_client() 27 | yield clean_swift(self.swift) 28 | 29 | @defer.inlineCallbacks 30 | def tearDown(self): 31 | shutil.rmtree(self.tmpdir) 32 | self.ftp.close() 33 | yield clean_swift(self.swift) 34 | yield self.pool.closeCachedConnections() 35 | 36 | def get_ftp_client(self): 37 | return get_ftp_client(conf) 38 | 39 | 40 | def validate_config(config): 41 | for key in 'ftp_host ftp_port account username password'.split(): 42 | if key not in config: 43 | raise unittest.SkipTest("%s not set in the test config file" % key) 44 | 45 | 46 | def get_ftp_client(config): 47 | validate_config(config) 48 | if config.get('debug'): 49 | ftplib.FTP.debugging = 5 50 | ftp = ftplib.FTP() 51 | ftp.connect(config['ftp_host'], int(config['ftp_port'])) 52 | ftp.login("%s:%s" % (config['account'], config['username']), 53 | config['password']) 54 | return ftp 55 | 56 | 57 | class BasicTests(unittest.TestCase): 58 | def test_get_client(self): 59 | ftp = get_ftp_client(conf) 60 | ftp.getwelcome() 61 | ftp.quit() 62 | 63 | def test_get_client_close(self): 64 | ftp = get_ftp_client(conf) 65 | ftp.getwelcome() 66 | ftp.close() 67 | 68 | def test_get_client_sock_close(self): 69 | for n in range(100): 70 | ftp = get_ftp_client(conf) 71 | ftp.getwelcome() 72 | ftp.sock.close() 73 | ftp.file.close() 74 | 75 | 76 | class ClientTests(unittest.TestCase): 77 | def setUp(self): 78 | self.active_connections = [] 79 | 80 | def tearDown(self): 81 | for conn in self.active_connections: 82 | try: 83 | conn.close() 84 | except: 85 | pass 86 | 87 | def get_client(self): 88 | conn = get_ftp_client(conf) 89 | self.active_connections.append(conn) 90 | return conn 91 | 92 | def test_get_many_client(self): 93 | for i in range(32): 94 | ftp = get_ftp_client(conf) 95 | ftp.close() 96 | 97 | # This test assumes sessions_per_user = 10 98 | def test_get_many_concurrent(self): 99 | validate_config(conf) 100 | for i in range(100): 101 | conn = ftplib.FTP() 102 | conn.connect(conf['ftp_host'], int(conf['ftp_port'])) 103 | self.active_connections.append(conn) 104 | time.sleep(10) 105 | 106 | # This test assumes sessions_per_user = 10 107 | def test_concurrency_limit(self): 108 | for i in range(10): 109 | self.get_client() 110 | self.assertRaises(ftplib.error_temp, self.get_client) 111 | 112 | # This test assumes sessions_per_user = 10 113 | def test_concurrency_limit_disconnect_one(self): 114 | for i in range(10): 115 | self.get_client() 116 | 117 | conn = self.active_connections.pop() 118 | conn.close() 119 | 120 | # This should not raise an error 121 | self.get_client() 122 | 123 | 124 | class RenameTests(FTPFuncTest): 125 | def test_rename_account(self): 126 | self.assertRaises(ftplib.error_perm, self.ftp.rename, '/', '/a') 127 | 128 | @defer.inlineCallbacks 129 | def test_rename_container(self): 130 | yield self.swift.put_container('ftp_tests') 131 | 132 | self.ftp.rename('ftp_tests', 'ftp_tests_2') 133 | r, listing = yield self.swift.get_account() 134 | 135 | self.assertTrue(has_item('ftp_tests_2', listing)) 136 | self.assertFalse(has_item('ftp_tests', listing)) 137 | 138 | @defer.inlineCallbacks 139 | def test_rename_container_populated(self): 140 | yield self.swift.put_container('ftp_tests') 141 | yield self.swift.put_object('ftp_tests', 'a') 142 | 143 | self.assertRaises(ftplib.error_perm, self.ftp.rename, 'ftp_tests', 144 | 'ftp_tests_2') 145 | 146 | @defer.inlineCallbacks 147 | def test_rename_object(self): 148 | yield self.swift.put_container('ftp_tests') 149 | yield self.swift.put_object('ftp_tests', 'a') 150 | yield self.swift.put_object( 151 | 'ftp_tests', 'b', 152 | headers={'Content-Type': 'application/directory'}) 153 | yield self.swift.put_object('ftp_tests', 'b/nested') 154 | yield self.swift.put_object('ftp_tests', 'c/nested') 155 | 156 | self.ftp.rename('ftp_tests/a', 'ftp_tests/a1') 157 | 158 | r, listing = yield self.swift.get_container('ftp_tests') 159 | 160 | self.assertTrue(has_item('a1', listing)) 161 | self.assertFalse(has_item('a', listing)) 162 | 163 | self.assertRaises(ftplib.error_perm, self.ftp.rename, 'ftp_tests/b', 164 | 'ftp_tests/b1') 165 | self.assertRaises(ftplib.error_perm, self.ftp.rename, 'ftp_tests/c', 166 | 'ftp_tests/c1') 167 | 168 | def test_rename_object_not_found(self): 169 | self.assertRaises(ftplib.error_perm, self.ftp.rename, 'ftp_tests/a', 170 | 'ftp_tests/b') 171 | 172 | 173 | class DownloadTests(FTPFuncTest): 174 | @defer.inlineCallbacks 175 | def _test_download(self, size, name, callback=None): 176 | yield self.swift.put_container('ftp_tests') 177 | src_path, md5 = create_test_file(self.tmpdir, size) 178 | yield upload_file(self.swift, 'ftp_tests', name, src_path, md5) 179 | dlpath = '%s/%s.dat' % (self.tmpdir, name) 180 | 181 | if not callback: 182 | resp = self.ftp.retrbinary('RETR ftp_tests/%s' % name, 183 | callback=open(dlpath, 'wb').write) 184 | self.assertEqual(os.stat(dlpath).st_size, size) 185 | self.assertEqual(md5, compute_md5(dlpath)) 186 | else: 187 | resp = self.ftp.retrbinary('RETR ftp_tests/%s' % name, 188 | callback=callback) 189 | self.assertEqual('226 Transfer Complete.', resp) 190 | 191 | def test_zero_byte_file(self): 192 | return self._test_download(0, '0b.dat') 193 | 194 | def test_32kb_file(self): 195 | return self._test_download(32 * 1024 + 1, '32kb.dat') 196 | 197 | def test_1mb_file(self): 198 | return self._test_download(1024 * 1024, '1mb.dat') 199 | 200 | def test_10mb_file(self): 201 | return self._test_download(1024 * 1024 * 10, '10mb.dat') 202 | 203 | def test_file_leak(self): 204 | class Callback(object): 205 | def __init__(self): 206 | self.i = 0 207 | 208 | def cb(self, data): 209 | self.i += 1 210 | if self.i == 2: 211 | time.sleep(5) # relatively long sleep 212 | # TODO: FIND PROCESS AND CHECK FOR MEMORY BLOAT 213 | # For now, just monitor memory usage 214 | 215 | return self._test_download(1024 * 1024 * 100, '100mb.dat', 216 | callback=Callback().cb) 217 | 218 | @defer.inlineCallbacks 219 | def test_read_timeout(self): 220 | class Callback(object): 221 | def __init__(self): 222 | self.i = 0 223 | 224 | def cb(self, data): 225 | self.i += 1 226 | if self.i == 2: 227 | time.sleep(40) # The timeout is actually 30 seconds 228 | 229 | try: 230 | yield self._test_download(1024 * 1024 * 100, '100mb.dat', 231 | callback=Callback().cb) 232 | except ftplib.error_temp: 233 | pass 234 | except: 235 | raise 236 | else: 237 | self.fail("Expected timeout error") 238 | 239 | 240 | class UploadTests(FTPFuncTest): 241 | @defer.inlineCallbacks 242 | def _test_upload(self, size, name): 243 | yield self.swift.put_container('ftp_tests') 244 | src_path, md5 = create_test_file(self.tmpdir, size) 245 | 246 | resp = self.ftp.storbinary('STOR ftp_tests/%s' % name, 247 | open(src_path, 'rb')) 248 | self.assertEqual('226 Transfer Complete.', resp) 249 | 250 | headers = yield self.swift.head_object('ftp_tests', name) 251 | self.assertEqual(md5, headers['etag']) 252 | self.assertEqual(size, int(headers['content-length'])) 253 | 254 | def test_zero_byte_file(self): 255 | return self._test_upload(0, '0b.dat') 256 | 257 | def test_32kb_file(self): 258 | return self._test_upload(1024 * 32 + 1, '32kb.dat') 259 | 260 | def test_1mb_file(self): 261 | return self._test_upload(1024 * 1024, '1mb.dat') 262 | 263 | def test_10mb_file(self): 264 | return self._test_upload(1024 * 1024 * 10, '10mb.dat') 265 | 266 | 267 | class SizeTests(FTPFuncTest): 268 | def test_size_root(self): 269 | # Testing For Error Only 270 | self.ftp.size('') 271 | 272 | @defer.inlineCallbacks 273 | def test_size_container(self): 274 | yield self.swift.put_container('ftp_tests') 275 | 276 | size = self.ftp.size('ftp_tests') 277 | self.assertEqual(0, size) 278 | 279 | @defer.inlineCallbacks 280 | def test_size_directory(self): 281 | yield self.swift.put_container('ftp_tests') 282 | yield self.swift.put_object( 283 | 'ftp_tests', 'test_size_directory', 284 | headers={'Content-Type': 'application/directory'}) 285 | 286 | size = self.ftp.size('ftp_tests/test_size_directory') 287 | self.assertEqual(0, size) 288 | 289 | @defer.inlineCallbacks 290 | def test_size_object(self): 291 | yield self.swift.put_container('ftp_tests') 292 | src_path, md5 = create_test_file(self.tmpdir, 1024) 293 | yield upload_file(self.swift, 'ftp_tests', 'test_size_object', 294 | src_path, md5) 295 | 296 | size = self.ftp.size('ftp_tests') 297 | self.assertEqual(1024, size) 298 | 299 | def test_size_container_missing(self): 300 | self.assertRaises(ftplib.error_perm, self.ftp.size, 'ftp_tests') 301 | 302 | def test_size_object_missing(self): 303 | self.assertRaises(ftplib.error_perm, self.ftp.size, 304 | 'ftp_tests/test_size_container_missing') 305 | 306 | @defer.inlineCallbacks 307 | def test_size_dir_dir(self): 308 | yield self.swift.put_container('ftp_tests') 309 | yield self.swift.put_object( 310 | 'ftp_tests', 311 | '%s/%s' % (utf8_chars.encode('utf-8'), utf8_chars.encode('utf-8'))) 312 | size = self.ftp.size('ftp_tests/%s' % utf8_chars.encode('utf-8')) 313 | self.assertEqual(0, size) 314 | 315 | 316 | class DeleteTests(FTPFuncTest): 317 | @defer.inlineCallbacks 318 | def test_delete_populated_container(self): 319 | yield self.swift.put_container('ftp_tests') 320 | yield self.swift.put_object( 321 | 'ftp_tests', 'dir1', 322 | headers={'Content-Type': 'application/directory'}) 323 | self.assertRaises(ftplib.error_perm, self.ftp.rmd, 'ftp_tests') 324 | 325 | @defer.inlineCallbacks 326 | def test_delete_populated_dir(self): 327 | yield self.swift.put_container('ftp_tests') 328 | yield self.swift.put_object( 329 | 'ftp_tests', 'dir1', 330 | headers={'Content-Type': 'application/directory'}) 331 | yield self.swift.put_object('ftp_tests', 'dir1/obj2') 332 | self.ftp.rmd('ftp_tests/dir1') 333 | 334 | @defer.inlineCallbacks 335 | def test_delete_populated_dir_not_existing(self): 336 | yield self.swift.put_container('ftp_tests') 337 | yield self.swift.put_object('ftp_tests', 'dir1/obj2') 338 | self.ftp.rmd('ftp_tests/dir1') 339 | 340 | 341 | class ListingTests(FTPFuncTest): 342 | def test_listing(self): 343 | listing = self.ftp.nlst('') 344 | self.assertNotIn('ftp_tests', listing) 345 | 346 | @defer.inlineCallbacks 347 | def test_listing_exists(self): 348 | yield self.swift.put_container('ftp_tests') 349 | listing = self.ftp.nlst('') 350 | self.assertIn('ftp_tests', listing) 351 | 352 | @defer.inlineCallbacks 353 | def test_directory_listing(self): 354 | yield self.swift.put_container('ftp_tests') 355 | yield self.swift.put_object( 356 | 'ftp_tests', 'dir1', 357 | headers={'Content-Type': 'application/directory'}) 358 | yield self.swift.put_object( 359 | 'ftp_tests', 'dir2', 360 | headers={'Content-Type': 'application/directory'}) 361 | yield self.swift.put_object('ftp_tests', 'dir2/obj1') 362 | yield self.swift.put_object('ftp_tests', 'dir3/obj2') 363 | 364 | listing = [] 365 | self.ftp.dir('ftp_tests', listing.append) 366 | self.assertIn('dir1', listing[0]) 367 | self.assertIn('dir2', listing[1]) 368 | self.assertIn('dir3', listing[2]) 369 | self.assertEqual(3, len(listing)) 370 | 371 | listing = self.ftp.nlst('ftp_tests/dir1') 372 | self.assertEqual(0, len(listing)) 373 | 374 | listing = self.ftp.nlst('ftp_tests/dir2') 375 | self.assertIn('obj1', listing) 376 | self.assertEqual(1, len(listing)) 377 | 378 | listing = self.ftp.nlst('ftp_tests/dir3') 379 | self.assertIn('obj2', listing) 380 | self.assertEqual(1, len(listing)) 381 | 382 | @defer.inlineCallbacks 383 | def test_long_listing(self): 384 | obj_count = 10010 385 | yield self.swift.put_container('ftp_tests') 386 | deferred_list = [] 387 | sem = defer.DeferredSemaphore(200) 388 | for i in range(obj_count): 389 | d = sem.run(self.swift.put_object, 'ftp_tests', str(i)) 390 | deferred_list.append(d) 391 | yield defer.DeferredList(deferred_list, consumeErrors=True) 392 | time.sleep(2) 393 | 394 | # The original FTP client can timeout while doing the setup 395 | self.ftp = self.get_ftp_client() 396 | listing = [] 397 | self.ftp.dir('ftp_tests', listing.append) 398 | self.assertTrue(len(listing) > 10000) 399 | 400 | @defer.inlineCallbacks 401 | def test_long_listing_nested(self): 402 | obj_count = 10010 403 | yield self.swift.put_container('ftp_tests') 404 | deferred_list = [] 405 | sem = defer.DeferredSemaphore(200) 406 | for i in range(obj_count): 407 | d = sem.run(self.swift.put_object, 'ftp_tests', 'subdir/' + str(i)) 408 | deferred_list.append(d) 409 | yield defer.DeferredList(deferred_list, consumeErrors=True) 410 | time.sleep(2) 411 | 412 | # The original FTP client can timeout while doing the setup 413 | self.ftp = self.get_ftp_client() 414 | listing = [] 415 | self.ftp.dir('ftp_tests/subdir', listing.append) 416 | self.assertTrue(len(listing) > 10000) 417 | 418 | 419 | class MkdirTests(FTPFuncTest): 420 | 421 | @defer.inlineCallbacks 422 | def test_make_container(self): 423 | self.ftp.mkd('ftp_tests') 424 | yield self.swift.head_container('ftp_tests') 425 | 426 | @defer.inlineCallbacks 427 | def test_make_object_dir(self): 428 | yield self.swift.put_container('ftp_tests') 429 | self.ftp.mkd('ftp_tests/mkdir') 430 | yield self.swift.head_object('ftp_tests', 'mkdir') 431 | 432 | @defer.inlineCallbacks 433 | def test_make_nested_object_dir(self): 434 | yield self.swift.put_container('ftp_tests') 435 | self.ftp.mkd('ftp_tests/nested/mkdir') 436 | yield self.swift.head_object('ftp_tests', 'nested/mkdir') 437 | 438 | 439 | class RmdirTests(FTPFuncTest): 440 | 441 | @defer.inlineCallbacks 442 | def test_rmdir_container(self): 443 | yield self.swift.put_container('ftp_tests') 444 | self.ftp.rmd('ftp_tests/nested/mkdir') 445 | resp, listing = yield self.swift.get_account('ftp_tests') 446 | self.assertNotIn('ftp_tests', listing) 447 | 448 | @defer.inlineCallbacks 449 | def test_rmdir_container_populated(self): 450 | yield self.swift.put_container('ftp_tests') 451 | yield self.swift.put_object('ftp_tests', utf8_chars.encode('utf-8')) 452 | self.assertRaises(ftplib.error_perm, self.ftp.rmd, 'ftp_tests') 453 | 454 | @defer.inlineCallbacks 455 | def test_rmdir_object_dir(self): 456 | yield self.swift.put_container('ftp_tests') 457 | yield self.swift.put_object('ftp_tests', 'nested/dir') 458 | self.ftp.rmd('ftp_tests/nested/dir') 459 | resp, listing = yield self.swift.get_container('ftp_tests') 460 | self.assertEqual(len(listing), 0) 461 | 462 | @defer.inlineCallbacks 463 | def test_rmdir_nested_object_dir(self): 464 | yield self.swift.put_container('ftp_tests') 465 | self.ftp.rmd('ftp_tests/nested/mkdir') 466 | resp, listing = yield self.swift.get_container('ftp_tests') 467 | self.assertEqual(len(listing), 0) 468 | 469 | 470 | class RemoveTests(FTPFuncTest): 471 | 472 | @defer.inlineCallbacks 473 | def test_remove_file(self): 474 | yield self.swift.put_container('ftp_tests') 475 | yield self.swift.put_object('ftp_tests', utf8_chars.encode('utf-8')) 476 | self.ftp.delete('ftp_tests/%s' % utf8_chars.encode('utf-8')) 477 | resp, listing = yield self.swift.get_container('ftp_tests') 478 | self.assertEqual(len(listing), 0) 479 | 480 | @defer.inlineCallbacks 481 | def test_remove_container(self): 482 | yield self.swift.put_container('ftp_tests') 483 | self.assertRaises(ftplib.error_perm, self.ftp.delete, 'ftp_tests') 484 | 485 | 486 | class CwdTests(FTPFuncTest): 487 | 488 | def test_cwd_root(self): 489 | self.ftp.cwd('/') 490 | 491 | @defer.inlineCallbacks 492 | def test_cwd_container(self): 493 | yield self.swift.put_container('ftp_tests') 494 | self.ftp.cwd('ftp_tests') 495 | 496 | def test_cwd_container_not_found(self): 497 | self.assertRaises( 498 | ftplib.error_perm, self.ftp.cwd, 'ftp_tests_not_found') 499 | 500 | @defer.inlineCallbacks 501 | def test_cwd_object(self): 502 | yield self.swift.put_container('ftp_tests') 503 | yield self.swift.put_object('ftp_tests', utf8_chars.encode('utf-8')) 504 | self.assertRaises( 505 | ftplib.error_perm, 506 | self.ftp.cwd, 'ftp_tests/%s' % utf8_chars.encode('utf-8')) 507 | 508 | @defer.inlineCallbacks 509 | def test_cwd_object_directory(self): 510 | yield self.swift.put_container('ftp_tests') 511 | yield self.swift.put_object( 512 | 'ftp_tests', utf8_chars.encode('utf-8'), 513 | headers={'Content-Type': 'application/directory'}) 514 | self.ftp.cwd('ftp_tests/%s' % utf8_chars.encode('utf-8')) 515 | -------------------------------------------------------------------------------- /swftp/test/functional/test_sftp.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | from twisted.trial import unittest 5 | from twisted.internet import defer, reactor 6 | from twisted.web.client import HTTPConnectionPool 7 | import paramiko 8 | 9 | import tempfile 10 | import shutil 11 | 12 | from . import get_config, has_item, create_test_file, clean_swift, \ 13 | compute_md5, upload_file, utf8_chars, get_swift_client 14 | 15 | from stat import S_ISDIR 16 | import os 17 | import time 18 | 19 | CONFIG = get_config() 20 | 21 | 22 | class SFTPFuncTest(unittest.TestCase): 23 | @defer.inlineCallbacks 24 | def setUp(self): 25 | self.active_connections = [] 26 | self.pool = HTTPConnectionPool(reactor, persistent=True) 27 | self.swift = get_swift_client(CONFIG, pool=self.pool) 28 | self.tmpdir = tempfile.mkdtemp() 29 | _, self.sftp = self.get_client() 30 | yield clean_swift(self.swift) 31 | 32 | @defer.inlineCallbacks 33 | def tearDown(self): 34 | shutil.rmtree(self.tmpdir) 35 | for (transport, conn) in self.active_connections: 36 | try: 37 | conn.close() 38 | except: 39 | pass 40 | try: 41 | transport.close() 42 | except: 43 | pass 44 | 45 | yield clean_swift(self.swift) 46 | yield self.pool.closeCachedConnections() 47 | 48 | def get_client(self): 49 | transport, conn = get_sftp_client_with_transport(CONFIG) 50 | self.active_connections.append((transport, conn)) 51 | return transport, conn 52 | 53 | 54 | def get_sftp_client(config): 55 | _, client = get_sftp_client_with_transport(config) 56 | return client 57 | 58 | 59 | def get_sftp_client_with_transport(config): 60 | for key in 'sftp_host sftp_port account username password'.split(): 61 | if key not in config: 62 | raise unittest.SkipTest("%s not set in the test config file" % key) 63 | hostname = config['sftp_host'] 64 | port = int(config['sftp_port']) 65 | username = "%s:%s" % (config['account'], config['username']) 66 | password = config['password'] 67 | 68 | t = paramiko.Transport((hostname, port)) 69 | t.connect(username=username, password=password) 70 | return t, paramiko.SFTPClient.from_transport(t) 71 | 72 | 73 | class BasicTests(unittest.TestCase): 74 | def test_get_client(self): 75 | t, client = get_sftp_client_with_transport(CONFIG) 76 | client.stat('/') 77 | client.close() 78 | t.close() 79 | 80 | 81 | class ClientTests(unittest.TestCase): 82 | def setUp(self): 83 | self.active_connections = [] 84 | 85 | def tearDown(self): 86 | for (transport, conn) in self.active_connections: 87 | try: 88 | conn.close() 89 | except: 90 | pass 91 | try: 92 | transport.close() 93 | except: 94 | pass 95 | 96 | def get_client(self): 97 | transport, conn = get_sftp_client_with_transport(CONFIG) 98 | self.active_connections.append((transport, conn)) 99 | return transport, conn 100 | 101 | def test_get_many_client(self): 102 | for i in range(100): 103 | t, client = self.get_client() 104 | client.close() 105 | t.close() 106 | 107 | # This test assumes sessions_per_user = 10 108 | def test_get_many_concurrent(self): 109 | for i in range(10): 110 | self.get_client() 111 | time.sleep(10) 112 | 113 | # This test assumes sessions_per_user = 10 114 | def test_concurrency_limit(self): 115 | for i in range(10): 116 | t, client = self.get_client() 117 | self.assertRaises(paramiko.AuthenticationException, self.get_client) 118 | 119 | # This test assumes sessions_per_user = 10 120 | def test_concurrency_limit_disconnect_one(self): 121 | for i in range(10): 122 | self.get_client() 123 | 124 | t, conn = self.active_connections.pop() 125 | conn.close() 126 | t.close() 127 | 128 | # This should not raise an error 129 | self.get_client() 130 | 131 | 132 | class RenameTests(SFTPFuncTest): 133 | def test_rename_account(self): 134 | self.assertRaises(IOError, self.sftp.rename, '/', '/a') 135 | 136 | @defer.inlineCallbacks 137 | def test_rename_container(self): 138 | yield self.swift.put_container('sftp_tests') 139 | 140 | self.sftp.rename('sftp_tests', 'sftp_tests_2') 141 | r, listing = yield self.swift.get_account() 142 | 143 | self.assertTrue(has_item('sftp_tests_2', listing)) 144 | self.assertFalse(has_item('sftp_tests', listing)) 145 | 146 | @defer.inlineCallbacks 147 | def test_rename_container_populated(self): 148 | yield self.swift.put_container('sftp_tests') 149 | yield self.swift.put_object('sftp_tests', 'a') 150 | 151 | self.assertRaises(IOError, self.sftp.rename, 'sftp_tests', 152 | 'sftp_tests_2') 153 | 154 | @defer.inlineCallbacks 155 | def test_rename_object(self): 156 | yield self.swift.put_container('sftp_tests') 157 | yield self.swift.put_object('sftp_tests', 'a') 158 | yield self.swift.put_object( 159 | 'sftp_tests', 'b', 160 | headers={'Content-Type': 'application/directory'}) 161 | yield self.swift.put_object('sftp_tests', 'b/nested') 162 | yield self.swift.put_object('sftp_tests', 'c/nested') 163 | 164 | self.sftp.rename('sftp_tests/a', 'sftp_tests/a1') 165 | 166 | r, listing = yield self.swift.get_container('sftp_tests') 167 | 168 | self.assertTrue(has_item('a1', listing)) 169 | self.assertFalse(has_item('a', listing)) 170 | 171 | self.assertRaises(IOError, self.sftp.rename, 'sftp_tests/b', 172 | 'sftp_tests/b1') 173 | self.assertRaises(IOError, self.sftp.rename, 'sftp_tests/c', 174 | 'sftp_tests/c1') 175 | 176 | def test_rename_object_not_found(self): 177 | self.assertRaises(IOError, self.sftp.rename, 'sftp_tests/a', 178 | 'sftp_tests/b') 179 | 180 | 181 | class DownloadTests(SFTPFuncTest): 182 | timeout = 240 183 | 184 | @defer.inlineCallbacks 185 | def _test_download(self, size, name, callback=None): 186 | yield self.swift.put_container('sftp_tests') 187 | src_path, md5 = create_test_file(self.tmpdir, size) 188 | yield upload_file(self.swift, 'sftp_tests', name, src_path, md5) 189 | 190 | dlpath = '%s/%s_dl' % (self.tmpdir, name) 191 | self.sftp.get('sftp_tests/%s' % name, dlpath, callback=callback) 192 | 193 | self.assertEqual(os.stat(dlpath).st_size, size) 194 | self.assertEqual(md5, compute_md5(dlpath)) 195 | 196 | def test_zero_byte_file(self): 197 | return self._test_download(0, '0b.dat') 198 | 199 | def test_32kb_file(self): 200 | return self._test_download(32 * 1024 + 1, '32kb.dat') 201 | 202 | def test_1mb_file(self): 203 | return self._test_download(1024 * 1024, '1mb.dat') 204 | 205 | def test_10mb_file(self): 206 | return self._test_download(1024 * 1024 * 10, '10mb.dat') 207 | 208 | def test_file_leak(self): 209 | class Callback(object): 210 | def __init__(self): 211 | self.i = 0 212 | 213 | def cb(self, transferred, total): 214 | self.i += 1 215 | time.sleep(0.01) # relatively long sleep 216 | # TODO: FIND PROCESS AND CHECK FOR MEMORY BLOAT 217 | # For now, just monitor memory usage 218 | 219 | return self._test_download(1024 * 1024 * 100, '100mb.dat', 220 | callback=Callback().cb) 221 | 222 | @defer.inlineCallbacks 223 | def test_read_timeout(self): 224 | class Callback(object): 225 | def __init__(self): 226 | self.i = 0 227 | 228 | def cb(self, transferred, total): 229 | self.i += 1 230 | if self.i == 2: 231 | print time.time() 232 | time.sleep(80) # The timeout is actually 60 seconds 233 | 234 | try: 235 | yield self._test_download(1024 * 1024 * 100, '100mb.dat', 236 | callback=Callback().cb) 237 | finally: 238 | print(time.time()) 239 | 240 | 241 | class UploadTests(SFTPFuncTest): 242 | timeout = 240 243 | 244 | @defer.inlineCallbacks 245 | def _test_upload(self, size, name): 246 | yield self.swift.put_container('sftp_tests') 247 | src_path, md5 = create_test_file(self.tmpdir, size) 248 | 249 | self.sftp.put(src_path, 'sftp_tests/%s' % name, confirm=False) 250 | 251 | headers = yield self.swift.head_object('sftp_tests', name) 252 | self.assertEqual(md5, headers['etag']) 253 | self.assertEqual(size, int(headers['content-length'])) 254 | 255 | def test_zero_byte_file(self): 256 | return self._test_upload(0, '0b.dat') 257 | 258 | def test_32kb_file(self): 259 | return self._test_upload(1024 * 32 + 1, '32kb.dat') 260 | 261 | def test_1mb_file(self): 262 | return self._test_upload(1024 * 1024, '1mb.dat') 263 | 264 | def test_10mb_file(self): 265 | return self._test_upload(1024 * 1024 * 10, '10mb.dat') 266 | 267 | 268 | class StatTests(SFTPFuncTest): 269 | def test_stat_root(self): 270 | stat = self.sftp.stat('/') 271 | self.assertTrue(S_ISDIR(stat.st_mode)) 272 | 273 | @defer.inlineCallbacks 274 | def test_container_stat(self): 275 | yield self.swift.put_container('sftp_tests') 276 | stat = self.sftp.stat('sftp_tests') 277 | self.assertTrue(S_ISDIR(stat.st_mode)) 278 | self.assertEqual(stat.st_size, 0) 279 | 280 | @defer.inlineCallbacks 281 | def test_dir_stat(self): 282 | yield self.swift.put_container('sftp_tests') 283 | yield self.swift.put_object('sftp_tests', utf8_chars.encode('utf-8')) 284 | stat = self.sftp.stat('sftp_tests') 285 | self.assertTrue(S_ISDIR(stat.st_mode)) 286 | self.assertEqual(stat.st_size, 0) 287 | 288 | @defer.inlineCallbacks 289 | def test_dir_dir_stat(self): 290 | yield self.swift.put_container('sftp_tests') 291 | yield self.swift.put_object( 292 | 'sftp_tests', 293 | '%s/%s' % (utf8_chars.encode('utf-8'), utf8_chars.encode('utf-8'))) 294 | stat = self.sftp.stat('sftp_tests/%s' % utf8_chars) 295 | self.assertTrue(S_ISDIR(stat.st_mode)) 296 | self.assertEqual(stat.st_size, 0) 297 | 298 | def test_stat_container_not_found(self): 299 | self.assertRaises(IOError, self.sftp.stat, 'sftp_tests') 300 | 301 | def test_stat_object_not_found(self): 302 | self.assertRaises(IOError, self.sftp.stat, 'sftp_tests/not/existing') 303 | 304 | 305 | class DeleteTests(SFTPFuncTest): 306 | @defer.inlineCallbacks 307 | def test_delete_populated_container(self): 308 | yield self.swift.put_container('sftp_tests') 309 | yield self.swift.put_object( 310 | 'sftp_tests', 'dir1', 311 | headers={'Content-Type': 'application/directory'}) 312 | self.assertRaises(IOError, self.sftp.rmdir, 'sftp_tests') 313 | 314 | @defer.inlineCallbacks 315 | def test_delete_populated_dir(self): 316 | yield self.swift.put_container('sftp_tests') 317 | yield self.swift.put_object( 318 | 'sftp_tests', 'dir1', 319 | headers={'Content-Type': 'application/directory'}) 320 | yield self.swift.put_object('sftp_tests', 'dir1/obj2') 321 | self.sftp.rmdir('sftp_tests/dir1') 322 | 323 | @defer.inlineCallbacks 324 | def test_delete_populated_dir_not_existing(self): 325 | yield self.swift.put_container('sftp_tests') 326 | yield self.swift.put_object('sftp_tests', 'dir1/obj2') 327 | self.sftp.rmdir('sftp_tests/dir1') 328 | 329 | 330 | class ListingTests(SFTPFuncTest): 331 | timeout = 360 332 | 333 | def test_listing(self): 334 | listing = self.sftp.listdir() 335 | self.assertNotIn('sftp_tests', listing) 336 | 337 | @defer.inlineCallbacks 338 | def test_listing_exists(self): 339 | yield self.swift.put_container('sftp_tests') 340 | listing = self.sftp.listdir() 341 | self.assertIn('sftp_tests', listing) 342 | 343 | @defer.inlineCallbacks 344 | def test_directory_listing(self): 345 | yield self.swift.put_container('sftp_tests') 346 | yield self.swift.put_object( 347 | 'sftp_tests', 'dir1', 348 | headers={'Content-Type': 'application/directory'}) 349 | yield self.swift.put_object( 350 | 'sftp_tests', 'dir2', 351 | headers={'Content-Type': 'application/directory'}) 352 | yield self.swift.put_object('sftp_tests', 'dir2/obj1') 353 | yield self.swift.put_object('sftp_tests', 'dir3/obj2') 354 | 355 | listing = self.sftp.listdir('sftp_tests') 356 | self.assertIn('dir1', listing) 357 | self.assertIn('dir2', listing) 358 | self.assertIn('dir3', listing) 359 | self.assertEqual(3, len(listing)) 360 | 361 | listing = self.sftp.listdir('sftp_tests/dir1') 362 | self.assertEqual(0, len(listing)) 363 | 364 | listing = self.sftp.listdir('sftp_tests/dir2') 365 | self.assertIn('obj1', listing) 366 | self.assertEqual(1, len(listing)) 367 | 368 | listing = self.sftp.listdir('sftp_tests/dir3') 369 | self.assertIn('obj2', listing) 370 | self.assertEqual(1, len(listing)) 371 | 372 | @defer.inlineCallbacks 373 | def test_long_listing(self): 374 | obj_count = 10100 375 | yield self.swift.put_container('sftp_tests') 376 | deferred_list = [] 377 | sem = defer.DeferredSemaphore(200) 378 | for i in range(obj_count): 379 | d = sem.run(self.swift.put_object, 'sftp_tests', str(i)) 380 | deferred_list.append(d) 381 | yield defer.DeferredList(deferred_list, consumeErrors=True) 382 | time.sleep(10) 383 | listing = self.sftp.listdir('sftp_tests') 384 | self.assertTrue(len(listing) > 10000) 385 | 386 | @defer.inlineCallbacks 387 | def test_long_listing_nested(self): 388 | obj_count = 10100 389 | yield self.swift.put_container('sftp_tests') 390 | deferred_list = [] 391 | sem = defer.DeferredSemaphore(200) 392 | for i in range(obj_count): 393 | d = sem.run(self.swift.put_object, 394 | 'sftp_tests', 'subdir/' + str(i)) 395 | deferred_list.append(d) 396 | yield defer.DeferredList(deferred_list, consumeErrors=True) 397 | time.sleep(10) 398 | listing = [] 399 | listing = self.sftp.listdir('sftp_tests/subdir') 400 | self.assertTrue(len(listing) > 10000) 401 | 402 | 403 | class MkdirTests(SFTPFuncTest): 404 | 405 | @defer.inlineCallbacks 406 | def test_make_container(self): 407 | self.sftp.mkdir('sftp_tests') 408 | yield self.swift.head_container('sftp_tests') 409 | 410 | @defer.inlineCallbacks 411 | def test_make_object_dir(self): 412 | yield self.swift.put_container('sftp_tests') 413 | self.sftp.mkdir('sftp_tests/mkdir') 414 | yield self.swift.head_object('sftp_tests', 'mkdir') 415 | 416 | @defer.inlineCallbacks 417 | def test_make_nested_object_dir(self): 418 | yield self.swift.put_container('sftp_tests') 419 | self.sftp.mkdir('sftp_tests/nested/mkdir') 420 | yield self.swift.head_object('sftp_tests', 'nested/mkdir') 421 | 422 | 423 | class RmdirTests(SFTPFuncTest): 424 | 425 | @defer.inlineCallbacks 426 | def test_rmdir_container(self): 427 | yield self.swift.put_container('sftp_tests') 428 | self.sftp.rmdir('sftp_tests/nested/mkdir') 429 | resp, listing = yield self.swift.get_account('sftp_tests') 430 | self.assertNotIn('sftp_tests', listing) 431 | 432 | @defer.inlineCallbacks 433 | def test_rmdir_container_populated(self): 434 | yield self.swift.put_container('sftp_tests') 435 | yield self.swift.put_object('sftp_tests', utf8_chars.encode('utf-8')) 436 | self.assertRaises(IOError, self.sftp.rmdir, 'sftp_tests') 437 | 438 | @defer.inlineCallbacks 439 | def test_rmdir_object_dir(self): 440 | yield self.swift.put_container('sftp_tests') 441 | yield self.swift.put_object('sftp_tests', 'nested/dir') 442 | self.sftp.rmdir('sftp_tests/nested/dir') 443 | resp, listing = yield self.swift.get_container('sftp_tests') 444 | self.assertEqual(len(listing), 0) 445 | 446 | @defer.inlineCallbacks 447 | def test_rmdir_nested_object_dir(self): 448 | yield self.swift.put_container('sftp_tests') 449 | self.sftp.rmdir('sftp_tests/nested/mkdir') 450 | resp, listing = yield self.swift.get_container('sftp_tests') 451 | self.assertEqual(len(listing), 0) 452 | 453 | 454 | class RemoveTests(SFTPFuncTest): 455 | 456 | @defer.inlineCallbacks 457 | def test_remove_file(self): 458 | yield self.swift.put_container('sftp_tests') 459 | yield self.swift.put_object('sftp_tests', utf8_chars.encode('utf-8')) 460 | self.sftp.remove('sftp_tests/%s' % utf8_chars.encode('utf-8')) 461 | resp, listing = yield self.swift.get_container('sftp_tests') 462 | self.assertEqual(len(listing), 0) 463 | 464 | @defer.inlineCallbacks 465 | def test_remove_container(self): 466 | yield self.swift.put_container('sftp_tests') 467 | self.assertRaises(IOError, self.sftp.remove, 'sftp_tests') 468 | -------------------------------------------------------------------------------- /swftp/test/test-ftp.conf: -------------------------------------------------------------------------------- 1 | [ftp] 2 | auth_url = http://127.0.0.1:8080/auth/v1.0 3 | host = 127.0.0.1 4 | port = 6021 5 | -------------------------------------------------------------------------------- /swftp/test/test-sftp.conf: -------------------------------------------------------------------------------- 1 | [sftp] 2 | auth_url = http://127.0.0.1:6080/auth/v1.0 3 | host = 127.0.0.1 4 | port = 6022 5 | -------------------------------------------------------------------------------- /swftp/test/test_id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEA1FIvzNpY2xEFd9vdFTlepGxa6WKL6dFORNnfoAJRnwpqQp2W 3 | I8myW9YQJ+6chzYDataHX+Imf1H/Y7TRYdxSHRj06y+fme6oDXLujXD5PpyDwu4Y 4 | 9ImFvcDUd5umumBXK+CMp6pJ804kArBGvKT9PhUR5C0XjowBdmCvyPfuLbt9ynj+ 5 | Mm856+cuR1NxjHKdilRrQiJW77pl07wzqPtwkX6OTB3F8IGt4UtqclsNlMBvs/Mh 6 | lXz2R1eauaB0Xal+3n7xdEE/gf0EXTsBHcOadCTVRJwDVRpt0WtxG3EHLNU7aUpf 7 | WR/wFyDHdskF8qsxK25VSXtXvWW7PhdeMu1psQIDAQABAoIBAGndNNI6iJVqDkTu 8 | DnT3rvXixQ2bQlHqN9FipPreTR40jjj65BKiDdl2iYWvlsJgLyIFJ2iqlTFkjCeF 9 | z+SwewzhFbVygGy7L3XNOZ6ylsQePCBkoXLEYzfyvr2IkYBaavyIsPkkkkzLSG9C 10 | v3Jt/YsH7GzOAvHjYxNIMD3PSSXpblH9Q4IvOjBlvzdgIak4FPlG8XM8BJc1Zwso 11 | 6/K8bhtLEMVHqShzWSG4KnlZvCJAtEesEKMZxpU0KMdBY1v37dvQr0TKclL+RDRG 12 | Pl4UH+7w3DV1D6QcKpNrtnb8uuXg6n8NtPdGvCtv8aRTJdl/jTD+8eTJ/gqapRNG 13 | ppHQeAECgYEA+VIvR6KnRrwL/CgdiS+DlSsuHjMTjTPwz0na7wHY+eRdKEWezS+W 14 | x7CUAVR7y6xoqkhTw9W8MDi9S0Gc+MIhMlUrGAgltIw459R85LSeWvhzf0x4ZjnU 15 | TqGSSqHPK2+djdnqcSqQDEnjjiLkRAyjB+MDJBY0d4yClYts+GEKyIECgYEA2gJC 16 | ooilZ+GRqp6hzP7Gmz7ZgQGiTvYNBfn7CME8Q1Uuhztpusah8ynN5rgigsB3vxZe 17 | ELSmDbGSsBXfYLWyUsJDPXJEfdePdLRYNq8uDbguwNV6bqHkNjUKQ570AF4KYpwm 18 | m2O8SnnjvY1cCFjS+BZZUrejNCtl30bCOxKQiTECgYAZsqni5VX+iXVmyS+6KyaR 19 | 4oB/Zl5WiIsCoqcpSXR21V3wv7RNi+ErtfybYnzcEmo7WvcfUVFzWvXP4vRuA3xu 20 | dq7ZbEWOpYFcC9/PpvpHgCJPda9XQhCrBiZRAPqcjIWd0F9EyNFOIvOMe/YxOWg4 21 | cOKq/orr45S2G0fhFgeJgQKBgGMYr+1tq7IiqLIj1Wx3SxP6Z6fqdDrsQVM8JE4A 22 | eEIrEDFMYm0SKT98+ykq/hTtJAFqS2vQkcKbuw/rgVRWgy53O1VNAbEqMP1dlNOW 23 | oZp+5FGnODXdxPyW84l1UXhxRss4K5xqo4Y/DbR18yWgKpW2oveknbzzFSV+5n8v 24 | mx+hAoGBAJDV1nSlmCPS+lgBzlV0YFDaRsn1GvScPOjZtY07/jYI7crW+dPq2SME 25 | WEHXvrrsp8DdhUvAkaCiW6MP/zTMB8+mXcLGOn76KLkHVPCzksIjYp0Wy9u6pkXa 26 | OjwUPZoI/IJW3BGAz5WqJfPeZZtjzy1hZ6E5zgmjV+Q3zBXP18kW 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /swftp/test/test_id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDUUi/M2ljbEQV3290VOV6kbFrpYovp0U5E2d+gAlGfCmpCnZYjybJb1hAn7pyHNgNq1odf4iZ/Uf9jtNFh3FIdGPTrL5+Z7qgNcu6NcPk+nIPC7hj0iYW9wNR3m6a6YFcr4IynqknzTiQCsEa8pP0+FRHkLReOjAF2YK/I9+4tu33KeP4ybznr5y5HU3GMcp2KVGtCIlbvumXTvDOo+3CRfo5MHcXwga3hS2pyWw2UwG+z8yGVfPZHV5q5oHRdqX7efvF0QT+B/QRdOwEdw5p0JNVEnANVGm3Ra3EbcQcs1TtpSl9ZH/AXIMd2yQXyqzErblVJe1e9Zbs+F14y7Wmx For Test Use 2 | -------------------------------------------------------------------------------- /swftp/test/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softlayer/swftp/6363985ed51c0d34b9d3aab9ead4f4246e93cd7b/swftp/test/unit/__init__.py -------------------------------------------------------------------------------- /swftp/test/unit/test_auth.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | from twisted.trial import unittest 5 | 6 | from mock import patch, MagicMock 7 | from twisted.cred.credentials import UsernamePassword 8 | from twisted.cred.error import UnauthorizedLogin 9 | from twisted.internet import defer 10 | 11 | from swftp.auth import SwiftBasedAuthDB 12 | from swftp.swift import UnAuthenticated 13 | 14 | 15 | def authenticate_good(ignored): 16 | return defer.succeed(None) 17 | 18 | 19 | def authenticate_bad(ignored): 20 | return defer.fail(UnAuthenticated(401, 'Not Authenticated')) 21 | 22 | 23 | class AuthenticateTest(unittest.TestCase): 24 | def setUp(self): 25 | self.auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') 26 | 27 | def test_init(self): 28 | auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') 29 | self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') 30 | self.assertEquals(auth_db.global_max_concurrency, 100) 31 | self.assertEquals(auth_db.max_concurrency, 10) 32 | self.assertEquals(auth_db.timeout, 260) 33 | self.assertEquals(auth_db.verbose, False) 34 | self.assertEquals(auth_db.rewrite_scheme, None) 35 | self.assertEquals(auth_db.rewrite_netloc, None) 36 | 37 | auth_db = SwiftBasedAuthDB( 38 | 'http://127.0.0.1:8080/v1/auth', 39 | global_max_concurrency=200, 40 | max_concurrency=20, 41 | timeout=460, 42 | verbose=True, 43 | rewrite_scheme='https', 44 | rewrite_netloc='some-hostname:1234', 45 | ) 46 | self.assertEquals(auth_db.auth_url, 'http://127.0.0.1:8080/v1/auth') 47 | self.assertEquals(auth_db.global_max_concurrency, 200) 48 | self.assertEquals(auth_db.max_concurrency, 20) 49 | self.assertEquals(auth_db.timeout, 460) 50 | self.assertEquals(auth_db.verbose, True) 51 | self.assertEquals(auth_db.rewrite_scheme, 'https') 52 | self.assertEquals(auth_db.rewrite_netloc, 'some-hostname:1234') 53 | 54 | @patch('swftp.auth.ThrottledSwiftConnection.authenticate', 55 | authenticate_good) 56 | def test_request_avatar_id(self): 57 | creds = UsernamePassword('username', 'password') 58 | return self.auth_db.requestAvatarId(creds) 59 | 60 | @patch('swftp.auth.ThrottledSwiftConnection.authenticate', 61 | authenticate_good) 62 | def test_zero_concurrency(self): 63 | auth_db = SwiftBasedAuthDB( 64 | 'http://127.0.0.1:8080/v1/auth', 65 | global_max_concurrency=0, 66 | max_concurrency=0, 67 | ) 68 | 69 | def check_connection(conn): 70 | self.assertEquals(conn.username, 'username') 71 | self.assertEquals(conn.api_key, 'password') 72 | # Default connection pool size per host is 2 73 | self.assertEquals(conn.pool.maxPersistentPerHost, 2) 74 | self.assertEquals(conn.pool.persistent, False) 75 | self.assertEquals(conn.locks, []) 76 | 77 | creds = UsernamePassword('username', 'password') 78 | d = auth_db.requestAvatarId(creds) 79 | d.addCallback(check_connection) 80 | return d 81 | 82 | @patch('swftp.auth.ThrottledSwiftConnection.authenticate', 83 | authenticate_bad) 84 | def test_request_avatar_id_fail(self): 85 | creds = UsernamePassword('username', 'password') 86 | d = self.auth_db.requestAvatarId(creds) 87 | return self.assertFailure(d, UnauthorizedLogin) 88 | 89 | def test_request_avatar_id_invalid_method(self): 90 | return self.assertFailure( 91 | self.auth_db.requestAvatarId('nope'), UnauthorizedLogin) 92 | 93 | 94 | class StorageUrlRewriteTest(unittest.TestCase): 95 | 96 | def test_no_storage_url(self): 97 | swift_conn = MagicMock() 98 | swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' 99 | auth_db = SwiftBasedAuthDB('http://127.0.0.1:8080/v1/auth') 100 | auth_db._rewrite_storage_url(swift_conn) 101 | 102 | self.assertEquals(swift_conn.storage_url, 103 | 'http://some-storage-url/v1/AUTH_12345') 104 | 105 | def test_hostname(self): 106 | swift_conn = MagicMock() 107 | swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' 108 | auth_db = SwiftBasedAuthDB( 109 | 'http://127.0.0.1:8080/v1/auth', 110 | rewrite_netloc='hostname') 111 | auth_db._rewrite_storage_url(swift_conn) 112 | 113 | self.assertEquals(swift_conn.storage_url, 114 | 'http://hostname/v1/AUTH_12345') 115 | 116 | def test_hostname_port(self): 117 | swift_conn = MagicMock() 118 | swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' 119 | auth_db = SwiftBasedAuthDB( 120 | 'http://127.0.0.1:8080/v1/auth', 121 | rewrite_netloc='hostname:1234') 122 | auth_db._rewrite_storage_url(swift_conn) 123 | 124 | self.assertEquals(swift_conn.storage_url, 125 | 'http://hostname:1234/v1/AUTH_12345') 126 | 127 | def test_scheme(self): 128 | swift_conn = MagicMock() 129 | swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' 130 | auth_db = SwiftBasedAuthDB( 131 | 'http://127.0.0.1:8080/v1/auth', 132 | rewrite_scheme='https') 133 | auth_db._rewrite_storage_url(swift_conn) 134 | 135 | self.assertEquals(swift_conn.storage_url, 136 | 'https://some-storage-url/v1/AUTH_12345') 137 | 138 | def test_all(self): 139 | swift_conn = MagicMock() 140 | swift_conn.storage_url = 'http://some-storage-url/v1/AUTH_12345' 141 | auth_db = SwiftBasedAuthDB( 142 | 'http://127.0.0.1:8080/v1/auth', 143 | rewrite_scheme='https', 144 | rewrite_netloc='hostname:1234') 145 | auth_db._rewrite_storage_url(swift_conn) 146 | 147 | self.assertEquals(swift_conn.storage_url, 148 | 'https://hostname:1234/v1/AUTH_12345') 149 | -------------------------------------------------------------------------------- /swftp/test/unit/test_ftp.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | import os.path 5 | import socket 6 | 7 | from twisted.trial import unittest 8 | 9 | from swftp.ftp.service import makeService, Options 10 | 11 | 12 | TEST_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 13 | 14 | 15 | class FTPServiceTest(unittest.TestCase): 16 | 17 | def setUp(self): 18 | opts = Options() 19 | opts.parseOptions([ 20 | '--config_file=%s' % os.path.join(TEST_PATH, 'test-ftp.conf'), 21 | ]) 22 | self.service = makeService(opts) 23 | return self.service.startService() 24 | 25 | def tearDown(self): 26 | return self.service.stopService() 27 | 28 | def test_service_listen(self): 29 | sock = socket.socket() 30 | sock.connect(('127.0.0.1', 6021)) 31 | -------------------------------------------------------------------------------- /swftp/test/unit/test_sftp.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | import os.path 5 | import socket 6 | 7 | from twisted.trial import unittest 8 | from twisted.internet import threads, defer 9 | 10 | from swftp.sftp.service import makeService, Options 11 | 12 | 13 | TEST_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 14 | 15 | 16 | class SFTPServiceTest(unittest.TestCase): 17 | 18 | @defer.inlineCallbacks 19 | def setUp(self): 20 | opts = Options() 21 | opts.parseOptions([ 22 | '--config_file=%s' % os.path.join(TEST_PATH, 'test-sftp.conf'), 23 | '--priv_key=%s' % os.path.join(TEST_PATH, 'test_id_rsa'), 24 | '--pub_key=%s' % os.path.join(TEST_PATH, 'test_id_rsa.pub'), 25 | ]) 26 | self.service = makeService(opts) 27 | yield self.service.startService() 28 | 29 | def tearDown(self): 30 | return self.service.stopService() 31 | 32 | def _defer_test_service_listen(self): 33 | for n in range(1000): 34 | sock = socket.socket() 35 | sock.connect(('127.0.0.1', 6022)) 36 | 37 | def test_service_listen(self): 38 | return threads.deferToThread(self._defer_test_service_listen) 39 | -------------------------------------------------------------------------------- /swftp/test/unit/test_swift.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | from mock import MagicMock 5 | 6 | from twisted.python.failure import Failure 7 | from twisted.trial import unittest 8 | from twisted.internet import defer, protocol 9 | from twisted.web.http_headers import Headers 10 | from twisted.web._newclient import ResponseDone 11 | from twisted.web import error 12 | 13 | from swftp.swift import ( 14 | SwiftConnection, ThrottledSwiftConnection, ResponseReceiver, 15 | ResponseIgnorer, cb_recv_resp, cb_process_resp, NotFound, UnAuthenticated, 16 | UnAuthorized, Conflict, RequestError) 17 | 18 | 19 | class StubWebAgent(protocol.Protocol): 20 | def __init__(self): 21 | self.requests = [] 22 | 23 | def request(self, *args, **kwargs): 24 | result = defer.Deferred() 25 | self.requests.append((result, args, kwargs)) 26 | return result 27 | 28 | 29 | class StubResponse(object): 30 | def __init__(self, code, headers=None, body=None): 31 | self.version = ('HTTP', 1, 1) 32 | self.code = code 33 | self.headers = headers or Headers() 34 | self.body = body or '' 35 | self.length = len(self.body) 36 | self.producing = True 37 | 38 | def deliverBody(self, receiver): 39 | receiver.makeConnection(self) 40 | if self.producing and self.body: 41 | receiver.dataReceived(self.body) 42 | receiver.connectionLost(Failure(ResponseDone())) 43 | 44 | def stopProducing(self): 45 | self.producing = False 46 | 47 | 48 | class SwiftConnectionTest(unittest.TestCase): 49 | def setUp(self): 50 | self.conn = SwiftConnection( 51 | 'http://127.0.0.1:8080/auth/v1.0', 'username', 'api_key', 52 | extra_headers={'extra': 'header'}, 53 | verbose=True) 54 | self.agent = StubWebAgent() 55 | self.conn.agent = self.agent 56 | self.conn.storage_url = 'http://127.0.0.1:8080/v1/AUTH_user' 57 | self.conn.auth_token = 'TOKEN_123' 58 | 59 | def test_init(self): 60 | conn = SwiftConnection( 61 | 'http://127.0.0.1:8080/auth/v1.0', 'username', 'api_key') 62 | self.assertEqual(conn.auth_url, 'http://127.0.0.1:8080/auth/v1.0') 63 | self.assertEqual(conn.username, 'username') 64 | self.assertEqual(conn.api_key, 'api_key') 65 | self.assertEqual(conn.auth_token, None) 66 | self.assertEqual(conn.verbose, False) 67 | self.assertIsNotNone(conn.agent) 68 | 69 | pool = MagicMock() 70 | conn = SwiftConnection( 71 | 'http://127.0.0.1:8080/auth/v1.0', 'username', 'api_key', 72 | pool=pool, 73 | verbose=True) 74 | self.assertEqual(conn.auth_url, 'http://127.0.0.1:8080/auth/v1.0') 75 | self.assertEqual(conn.username, 'username') 76 | self.assertEqual(conn.api_key, 'api_key') 77 | self.assertEqual(conn.auth_token, None) 78 | self.assertEqual(conn.verbose, True) 79 | self.assertIsNotNone(conn.agent) 80 | self.assertEqual(conn.pool, pool) 81 | 82 | def test_make_request(self): 83 | make_request = self.conn.make_request('method', 'path/to/resource', 84 | params={'param': 'value'}, 85 | headers={'header': 'value'}, 86 | body='body') 87 | self.assertEqual(len(self.agent.requests), 1) 88 | d, args, kwargs = self.agent.requests[0] 89 | self.assertEqual(args, ( 90 | 'method', 91 | 'http://127.0.0.1:8080/v1/AUTH_user/path/to/resource?param=value', 92 | Headers({ 93 | 'header': ['value'], 94 | 'user-agent': ['Twisted Swift'], 95 | 'x-auth-token': ['TOKEN_123'], 96 | 'extra': ['header']}), 97 | 'body')) 98 | 99 | response = StubResponse(200, body='some body') 100 | d.callback(response) 101 | 102 | def cbCheckResponse(resp): 103 | self.assertEqual(resp, response) 104 | return resp 105 | make_request.addCallback(cbCheckResponse) 106 | make_request.addCallback(cb_recv_resp, load_body=True) 107 | 108 | def cbCheckResponseWithBody(resp): 109 | self.assertEqual(resp, (response, 'some body')) 110 | return resp 111 | make_request.addCallback(cbCheckResponseWithBody) 112 | return make_request 113 | 114 | def test_make_request_failed_auth(self): 115 | # Make initial request 116 | make_request = self.conn.make_request('method', 'path/to/resource', 117 | params={'param': 'value'}, 118 | headers={'header': 'value'}, 119 | body='body') 120 | 121 | self.assertEqual(len(self.agent.requests), 1) 122 | d, args, kwargs = self.agent.requests[0] 123 | self.assertEqual(args, ( 124 | 'method', 125 | 'http://127.0.0.1:8080/v1/AUTH_user/path/to/resource?param=value', 126 | Headers({ 127 | 'header': ['value'], 128 | 'user-agent': ['Twisted Swift'], 129 | 'x-auth-token': ['TOKEN_123'], 130 | 'extra': ['header']}), 131 | 'body')) 132 | 133 | # Return a 401 134 | response = StubResponse(401) 135 | d.callback(response) 136 | 137 | # Check to make sure an auth request is being sent now 138 | self.assertEqual(len(self.agent.requests), 2) 139 | d, args, kwargs = self.agent.requests[1] 140 | self.assertEqual(args, ( 141 | 'GET', 142 | 'http://127.0.0.1:8080/auth/v1.0', 143 | Headers({ 144 | 'user-agent': ['Twisted Swift'], 145 | 'x-auth-user': ['username'], 146 | 'x-auth-key': ['api_key'], 147 | 'extra': ['header']}))) 148 | 149 | # Return a 200 for the auth request 150 | response = StubResponse(200, headers=Headers({ 151 | 'x-storage-url': ['AUTHED_STORAGE_URL'], 152 | 'x-auth-token': ['AUTHED_TOKEN'], 153 | })) 154 | d.callback(response) 155 | 156 | # Make sure authentication has been performed successfully 157 | self.assertEqual(self.conn.storage_url, 'AUTHED_STORAGE_URL') 158 | self.assertEqual(self.conn.auth_token, 'AUTHED_TOKEN') 159 | 160 | # Check to make sure there's a second attempt at the original request 161 | self.assertEqual(len(self.agent.requests), 3) 162 | d, args, kwargs = self.agent.requests[2] 163 | self.assertEqual(args, ( 164 | 'method', 165 | 'AUTHED_STORAGE_URL/path/to/resource?param=value', 166 | Headers({ 167 | 'header': ['value'], 168 | 'user-agent': ['Twisted Swift'], 169 | 'x-auth-token': ['AUTHED_TOKEN'], 170 | 'extra': ['header']}), 171 | 'body')) 172 | 173 | # Return a 200 for the second attempt 174 | response = StubResponse(200) 175 | d.callback(response) 176 | 177 | def cbCheckResponse(resp): 178 | self.assertEqual(resp, response) 179 | make_request.addCallback(cbCheckResponse) 180 | return make_request 181 | 182 | def test_authenticate(self): 183 | auth_d = self.conn.authenticate() 184 | self.assertEqual(len(self.agent.requests), 1) 185 | d, args, kwargs = self.agent.requests[0] 186 | self.assertEqual(args, ( 187 | 'GET', 188 | 'http://127.0.0.1:8080/auth/v1.0', 189 | Headers({ 190 | 'user-agent': ['Twisted Swift'], 191 | 'x-auth-user': ['username'], 192 | 'x-auth-key': ['api_key'], 193 | 'extra': ['header']}))) 194 | 195 | response = StubResponse(200, headers=Headers({ 196 | 'x-storage-url': ['AUTHED_STORAGE_URL'], 197 | 'x-auth-token': ['AUTHED_TOKEN'], 198 | })) 199 | d.callback(response) 200 | 201 | def cbCheckResponse(resp): 202 | self.assertEqual(self.conn.storage_url, 'AUTHED_STORAGE_URL') 203 | self.assertEqual(self.conn.auth_token, 'AUTHED_TOKEN') 204 | self.assertEqual(resp, (response, '')) 205 | auth_d.addCallback(cbCheckResponse) 206 | return auth_d 207 | 208 | def test_head_account(self): 209 | make_request = self.conn.head_account() 210 | self.assertEqual(len(self.agent.requests), 1) 211 | d, args, kwargs = self.agent.requests[0] 212 | self.assertEqual(args, ( 213 | 'HEAD', 'http://127.0.0.1:8080/v1/AUTH_user/', 214 | Headers({ 215 | 'user-agent': ['Twisted Swift'], 216 | 'x-auth-token': ['TOKEN_123'], 217 | 'extra': ['header']}), 218 | None)) 219 | 220 | response = StubResponse(204, headers=Headers({ 221 | 'X-Account-Container-Count': ['3'], 222 | 'X-Account-Bytes-Used': ['323479'], 223 | })) 224 | d.callback(response) 225 | 226 | def cbCheckResponse(resp): 227 | self.assertEqual(resp, { 228 | 'x-account-bytes-used': '323479', 229 | 'x-account-container-count': '3' 230 | }) 231 | return resp 232 | make_request.addCallback(cbCheckResponse) 233 | return make_request 234 | 235 | def test_get_account(self): 236 | make_request = self.conn.get_account(limit=10, 237 | marker='test_container_0', 238 | end_marker='test_container_3') 239 | self.assertEqual(len(self.agent.requests), 1) 240 | d, args, kwargs = self.agent.requests[0] 241 | self.assertEqual(args, ( 242 | 'GET', 243 | 'http://127.0.0.1:8080/v1/AUTH_user/?marker=test_container_0' 244 | '&limit=10&end_marker=test_container_3&format=json', 245 | Headers({ 246 | 'user-agent': ['Twisted Swift'], 247 | 'x-auth-token': ['TOKEN_123'], 248 | 'extra': ['header']}), 249 | None)) 250 | 251 | response = StubResponse(200, body='''[ 252 | {"name":"test_container_1", "count":2, "bytes":78}, 253 | {"name":"test_container_2", "count":1, "bytes":17} 254 | ]''') 255 | d.callback(response) 256 | 257 | def cbCheckResponse(resp): 258 | self.assertEqual(resp, (response, [ 259 | {u'bytes': 78, u'count': 2, u'name': u'test_container_1'}, 260 | {u'bytes': 17, u'count': 1, u'name': u'test_container_2'} 261 | ])) 262 | return resp 263 | make_request.addCallback(cbCheckResponse) 264 | return make_request 265 | 266 | def test_head_container(self): 267 | make_request = self.conn.head_container('container') 268 | self.assertEqual(len(self.agent.requests), 1) 269 | d, args, kwargs = self.agent.requests[0] 270 | self.assertEqual(args, ( 271 | 'HEAD', 'http://127.0.0.1:8080/v1/AUTH_user/container', 272 | Headers({ 273 | 'user-agent': ['Twisted Swift'], 274 | 'x-auth-token': ['TOKEN_123'], 275 | 'extra': ['header']}), 276 | None)) 277 | 278 | response = StubResponse(200, headers=Headers({ 279 | 'X-Container-Object-Count': ['7'], 280 | 'X-Container-Bytes-Used': ['413'], 281 | 'X-Container-Meta-InspectedBy': ['JackWolf'], 282 | })) 283 | d.callback(response) 284 | 285 | def cbCheckResponse(resp): 286 | self.assertEqual(resp, { 287 | 'x-container-bytes-used': '413', 288 | 'x-container-meta-inspectedby': 'JackWolf', 289 | 'x-container-object-count': '7' 290 | }) 291 | return resp 292 | make_request.addCallback(cbCheckResponse) 293 | return make_request 294 | 295 | def test_get_container(self): 296 | make_request = self.conn.get_container('container', 297 | limit=10, 298 | marker='test_obj_0', 299 | end_marker='test_obj_3', 300 | prefix='test_obj', 301 | path='path', 302 | delimiter='/') 303 | self.assertEqual(len(self.agent.requests), 1) 304 | d, args, kwargs = self.agent.requests[0] 305 | self.assertEqual(args, ( 306 | 'GET', 307 | 'http://127.0.0.1:8080/v1/AUTH_user/container' 308 | '?end_marker=test_obj_3&format=json&delimiter=/&prefix=test_obj' 309 | '&limit=10&marker=test_obj_0&path=path', 310 | Headers({ 311 | 'user-agent': ['Twisted Swift'], 312 | 'x-auth-token': ['TOKEN_123'], 313 | 'extra': ['header']}), 314 | None)) 315 | 316 | response = StubResponse(200, body='''[ 317 | {"name":"test_obj_1", 318 | "hash":"4281c348eaf83e70ddce0e07221c3d28", 319 | "bytes":14, 320 | "content_type":"application\/octet-stream", 321 | "last_modified":"2009-02-03T05:26:32.612278"}, 322 | {"name":"test_obj_2", 323 | "hash":"b039efe731ad111bc1b0ef221c3849d0", 324 | "bytes":64, 325 | "content_type":"application\/octet-stream", 326 | "last_modified":"2009-02-03T05:26:32.612278"} 327 | ]''') 328 | d.callback(response) 329 | 330 | def cbCheckResponse(resp): 331 | self.assertEqual(resp, (response, [{ 332 | u'bytes': 14, 333 | u'content_type': u'application/octet-stream', 334 | u'hash': u'4281c348eaf83e70ddce0e07221c3d28', 335 | u'last_modified': u'2009-02-03T05:26:32.612278', 336 | u'name': u'test_obj_1' 337 | }, { 338 | u'bytes': 64, 339 | u'content_type': u'application/octet-stream', 340 | u'hash': u'b039efe731ad111bc1b0ef221c3849d0', 341 | u'last_modified': u'2009-02-03T05:26:32.612278', 342 | u'name': u'test_obj_2' 343 | }])) 344 | return resp 345 | make_request.addCallback(cbCheckResponse) 346 | return make_request 347 | 348 | def test_get_container_marker(self): 349 | make_request = self.conn.get_container('container', 350 | marker='test_obj_0') 351 | self.assertEqual(len(self.agent.requests), 1) 352 | d, args, kwargs = self.agent.requests[0] 353 | self.assertEqual(args, ( 354 | 'GET', 355 | 'http://127.0.0.1:8080/v1/AUTH_user/container' 356 | '?marker=test_obj_0&format=json', 357 | Headers({ 358 | 'user-agent': ['Twisted Swift'], 359 | 'x-auth-token': ['TOKEN_123'], 360 | 'extra': ['header']}), 361 | None)) 362 | 363 | response = StubResponse(200, body='''[ 364 | {"name":"test_obj_1", 365 | "hash":"4281c348eaf83e70ddce0e07221c3d28", 366 | "bytes":14, 367 | "content_type":"application\/octet-stream", 368 | "last_modified":"2009-02-03T05:26:32.612278"}, 369 | {"name":"test_obj_2", 370 | "hash":"b039efe731ad111bc1b0ef221c3849d0", 371 | "bytes":64, 372 | "content_type":"application\/octet-stream", 373 | "last_modified":"2009-02-03T05:26:32.612278"} 374 | ]''') 375 | d.callback(response) 376 | 377 | def cbCheckResponse(resp): 378 | self.assertEqual(resp, (response, [{ 379 | u'bytes': 14, 380 | u'content_type': u'application/octet-stream', 381 | u'hash': u'4281c348eaf83e70ddce0e07221c3d28', 382 | u'last_modified': u'2009-02-03T05:26:32.612278', 383 | u'name': u'test_obj_1' 384 | }, { 385 | u'bytes': 64, 386 | u'content_type': u'application/octet-stream', 387 | u'hash': u'b039efe731ad111bc1b0ef221c3849d0', 388 | u'last_modified': u'2009-02-03T05:26:32.612278', 389 | u'name': u'test_obj_2' 390 | }])) 391 | return resp 392 | make_request.addCallback(cbCheckResponse) 393 | return make_request 394 | 395 | def test_put_container(self): 396 | make_request = self.conn.put_container('container') 397 | self.assertEqual(len(self.agent.requests), 1) 398 | d, args, kwargs = self.agent.requests[0] 399 | self.assertEqual(args, ( 400 | 'PUT', 401 | 'http://127.0.0.1:8080/v1/AUTH_user/container', 402 | Headers({ 403 | 'user-agent': ['Twisted Swift'], 404 | 'x-auth-token': ['TOKEN_123'], 405 | 'extra': ['header']}), 406 | None)) 407 | 408 | response = StubResponse(201) 409 | d.callback(response) 410 | 411 | def cbCheckResponse(resp): 412 | self.assertEqual(resp, (response, None)) 413 | return resp 414 | make_request.addCallback(cbCheckResponse) 415 | return make_request 416 | 417 | def test_delete_container(self): 418 | make_request = self.conn.delete_container('container') 419 | self.assertEqual(len(self.agent.requests), 1) 420 | d, args, kwargs = self.agent.requests[0] 421 | self.assertEqual(args, ( 422 | 'DELETE', 423 | 'http://127.0.0.1:8080/v1/AUTH_user/container', 424 | Headers({ 425 | 'user-agent': ['Twisted Swift'], 426 | 'x-auth-token': ['TOKEN_123'], 427 | 'extra': ['header']}), 428 | None)) 429 | 430 | response = StubResponse(204) 431 | d.callback(response) 432 | 433 | def cbCheckResponse(resp): 434 | self.assertEqual(resp, (response, None)) 435 | return resp 436 | make_request.addCallback(cbCheckResponse) 437 | return make_request 438 | 439 | def test_head_object(self): 440 | make_request = self.conn.head_object('container', 'object') 441 | self.assertEqual(len(self.agent.requests), 1) 442 | d, args, kwargs = self.agent.requests[0] 443 | self.assertEqual(args, ( 444 | 'HEAD', 'http://127.0.0.1:8080/v1/AUTH_user/container/object', 445 | Headers({ 446 | 'user-agent': ['Twisted Swift'], 447 | 'x-auth-token': ['TOKEN_123'], 448 | 'extra': ['header']}), 449 | None)) 450 | 451 | response = StubResponse(200, headers=Headers({ 452 | 'Last-Modified': ['Fri, 12 Jun 2010 13:40:18 GMT'], 453 | 'ETag': ['8a964ee2a5e88be344f36c22562a6486'], 454 | 'Content-Length': ['512000'], 455 | 'Content-Type': ['text/plain; charset=UTF-8'], 456 | 'X-Object-Meta-Meat': ['Bacon'], 457 | 'X-Object-Meta-Fruit': ['Bacon'], 458 | 'X-Object-Meta-Veggie': ['Bacon'], 459 | 'X-Object-Meta-Dairy': ['Bacon'], 460 | })) 461 | d.callback(response) 462 | 463 | def cbCheckResponse(resp): 464 | self.assertEqual(resp, { 465 | 'content-length': '512000', 466 | 'content-type': 'text/plain; charset=UTF-8', 467 | 'etag': '8a964ee2a5e88be344f36c22562a6486', 468 | 'last-modified': 'Fri, 12 Jun 2010 13:40:18 GMT', 469 | 'x-object-meta-dairy': 'Bacon', 470 | 'x-object-meta-fruit': 'Bacon', 471 | 'x-object-meta-meat': 'Bacon', 472 | 'x-object-meta-veggie': 'Bacon' 473 | }) 474 | return resp 475 | make_request.addCallback(cbCheckResponse) 476 | return make_request 477 | 478 | def test_get_object(self): 479 | received = defer.Deferred() 480 | receiver = ResponseReceiver(received) 481 | make_request = self.conn.get_object('container', 'object', 482 | receiver=receiver) 483 | self.assertEqual(len(self.agent.requests), 1) 484 | d, args, kwargs = self.agent.requests[0] 485 | self.assertEqual(args, ( 486 | 'GET', 'http://127.0.0.1:8080/v1/AUTH_user/container/object', 487 | Headers({ 488 | 'user-agent': ['Twisted Swift'], 489 | 'x-auth-token': ['TOKEN_123'], 490 | 'extra': ['header']}), 491 | None)) 492 | 493 | response = StubResponse(200, headers=Headers({ 494 | 'Last-Modified': ['Fri, 12 Jun 2010 13:40:18 GMT'], 495 | 'ETag': ['8a964ee2a5e88be344f36c22562a6486'], 496 | 'Content-Length': ['512000'], 497 | 'Content-Type': ['text/plain; charset=UTF-8'], 498 | 'X-Object-Meta-Meat': ['Bacon'], 499 | 'X-Object-Meta-Fruit': ['Bacon'], 500 | 'X-Object-Meta-Veggie': ['Bacon'], 501 | 'X-Object-Meta-Dairy': ['Bacon'], 502 | }), body=' ' * 512000) 503 | d.callback(response) 504 | 505 | def cbCheckResponse(resp): 506 | self.assertEqual(resp, response) 507 | return resp 508 | make_request.addCallback(cbCheckResponse) 509 | 510 | def cbCheckResponseBody(resp): 511 | self.assertEqual(resp, ' ' * 512000) 512 | return resp 513 | received.addCallback(cbCheckResponseBody) 514 | return defer.gatherResults([make_request, received]) 515 | 516 | def test_put_object(self): 517 | make_request = self.conn.put_object('container', 'object') 518 | self.assertEqual(len(self.agent.requests), 1) 519 | d, args, kwargs = self.agent.requests[0] 520 | self.assertEqual(args, ( 521 | 'PUT', 522 | 'http://127.0.0.1:8080/v1/AUTH_user/container/object', 523 | Headers({ 524 | 'content-length': ['0'], 525 | 'user-agent': ['Twisted Swift'], 526 | 'x-auth-token': ['TOKEN_123'], 527 | 'extra': ['header']}), 528 | None)) 529 | 530 | response = StubResponse(201) 531 | d.callback(response) 532 | 533 | def cbCheckResponse(resp): 534 | self.assertEqual(resp, (response, '')) 535 | return resp 536 | make_request.addCallback(cbCheckResponse) 537 | return make_request 538 | 539 | def test_delete_object(self): 540 | make_request = self.conn.delete_object('container', 'object') 541 | self.assertEqual(len(self.agent.requests), 1) 542 | d, args, kwargs = self.agent.requests[0] 543 | self.assertEqual(args, ( 544 | 'DELETE', 545 | 'http://127.0.0.1:8080/v1/AUTH_user/container/object', 546 | Headers({ 547 | 'user-agent': ['Twisted Swift'], 548 | 'x-auth-token': ['TOKEN_123'], 549 | 'extra': ['header']}), 550 | None)) 551 | 552 | response = StubResponse(204) 553 | d.callback(response) 554 | 555 | def cbCheckResponse(resp): 556 | self.assertEqual(resp, (response, None)) 557 | return resp 558 | make_request.addCallback(cbCheckResponse) 559 | return make_request 560 | 561 | 562 | class ThrottledSwiftConnectionTest(unittest.TestCase): 563 | def setUp(self): 564 | self.agent = StubWebAgent() 565 | 566 | def test_single_lock(self): 567 | lock = defer.DeferredLock() 568 | conn = ThrottledSwiftConnection( 569 | [lock], 'http://127.0.0.1:8080/auth/v1.0', 'username', 'api_key', 570 | verbose=True) 571 | conn.agent = self.agent 572 | conn.storage_url = 'http://127.0.0.1:8080/v1/AUTH_user' 573 | conn.auth_token = 'TOKEN_123' 574 | 575 | conn.make_request('method', 'path') 576 | 577 | self.assertEqual(len(self.agent.requests), 1) 578 | self.assertEqual(lock.locked, 1) 579 | 580 | conn.make_request('method', 'path2') 581 | self.assertEqual(len(self.agent.requests), 1) 582 | d, args, kwargs = self.agent.requests[0] 583 | d.callback(StubResponse(200)) 584 | 585 | self.assertEqual(len(self.agent.requests), 2) 586 | d, args, kwargs = self.agent.requests[1] 587 | d.callback(StubResponse(200)) 588 | self.assertEqual(lock.locked, 0) 589 | 590 | def test_multi_lock(self): 591 | lock = defer.DeferredLock() 592 | sem = defer.DeferredSemaphore(2) 593 | conn = ThrottledSwiftConnection( 594 | [lock, sem], 595 | 'http://127.0.0.1:8080/auth/v1.0', 'username', 'api_key', 596 | verbose=True) 597 | conn.agent = self.agent 598 | conn.storage_url = 'http://127.0.0.1:8080/v1/AUTH_user' 599 | conn.auth_token = 'TOKEN_123' 600 | 601 | conn.make_request('method', 'path') 602 | 603 | self.assertEqual(len(self.agent.requests), 1) 604 | self.assertEqual(lock.locked, 1) 605 | self.assertEqual(sem.tokens, 1) 606 | 607 | conn.make_request('method', 'path2') 608 | self.assertEqual(len(self.agent.requests), 1) 609 | d, args, kwargs = self.agent.requests[0] 610 | d.callback(StubResponse(200)) 611 | 612 | self.assertEqual(len(self.agent.requests), 2) 613 | d, args, kwargs = self.agent.requests[1] 614 | d.callback(StubResponse(200)) 615 | self.assertEqual(lock.locked, 0) 616 | self.assertEqual(sem.tokens, 2) 617 | 618 | 619 | class HelpersTest(unittest.TestCase): 620 | 621 | def test_cb_process_resp(self): 622 | resp = StubResponse(200) 623 | response, body = cb_process_resp(None, resp) 624 | self.assertEqual(response, resp) 625 | self.assertEqual(body, None) 626 | 627 | # > 404 raises NotFound 628 | self.assertRaises(NotFound, cb_process_resp, None, StubResponse(404)) 629 | 630 | # > 401 raises UnAuthenticated 631 | self.assertRaises( 632 | UnAuthenticated, cb_process_resp, None, StubResponse(401)) 633 | 634 | # > 403 raises UnAuthorized 635 | self.assertRaises( 636 | UnAuthorized, cb_process_resp, None, StubResponse(403)) 637 | 638 | # > 409 raises Conflict 639 | self.assertRaises(Conflict, cb_process_resp, None, StubResponse(409)) 640 | 641 | # > 300-399 raises a RequestError 642 | self.assertRaises( 643 | error.PageRedirect, cb_process_resp, None, StubResponse(300)) 644 | 645 | # > 400 raises a RequestError 646 | self.assertRaises( 647 | RequestError, cb_process_resp, None, StubResponse(400)) 648 | 649 | def test_response_ignorer(self): 650 | finished = defer.Deferred() 651 | ignorer = ResponseIgnorer(finished) 652 | transport = MagicMock() 653 | 654 | ignorer.makeConnection(transport) 655 | transport.stopProducing.assert_called_with() 656 | 657 | ignorer.dataReceived(None) 658 | ignorer.connectionLost(None) 659 | 660 | return finished 661 | 662 | def test_response_receiver(self): 663 | finished = defer.Deferred() 664 | recv = ResponseReceiver(finished) 665 | recv.dataReceived('bytes') 666 | recv.dataReceived(' go') 667 | recv.dataReceived('here.') 668 | 669 | err = error.Error('Something Happened') 670 | recv.connectionLost(Failure(err)) 671 | 672 | def checkError(result): 673 | result.trap(error.Error) 674 | self.assertRaises(error.Error, result.raiseException) 675 | finished.addBoth(checkError) 676 | return finished 677 | -------------------------------------------------------------------------------- /swftp/test/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | import unittest 5 | import os 6 | import time 7 | 8 | from twisted.python import log 9 | 10 | from swftp.utils import ( 11 | try_datetime_parse, MetricCollector, parse_key_value_config) 12 | 13 | 14 | class MetricCollectorTest(unittest.TestCase): 15 | def setUp(self): 16 | self.c = MetricCollector() 17 | 18 | def test_init(self): 19 | c = MetricCollector(10) 20 | self.assertEqual(c.sample_size, 10) 21 | self.assertEqual(c.current, {}) 22 | self.assertEqual(c.totals, {}) 23 | self.assertEqual(c.samples, {}) 24 | 25 | c = MetricCollector(20) 26 | self.assertEqual(c.sample_size, 20) 27 | 28 | def test_emit(self): 29 | self.c.emit({'metric': 'some_metric'}) 30 | self.assertEqual(self.c.current['some_metric'], 1) 31 | 32 | self.c.emit({'metric': 'some_metric', 'count': 10}) 33 | self.assertEqual(self.c.current['some_metric'], 11) 34 | 35 | def test_add_metric(self): 36 | self.c.add_metric('some_metric') 37 | self.assertEqual(self.c.current['some_metric'], 1) 38 | self.assertEqual(self.c.totals['some_metric'], 1) 39 | 40 | self.c.add_metric('some_metric', count=10) 41 | self.assertEqual(self.c.current['some_metric'], 11) 42 | self.assertEqual(self.c.totals['some_metric'], 11) 43 | 44 | def test_sample(self): 45 | self.c.add_metric('some_metric') 46 | self.c.sample() 47 | self.assertEqual(self.c.samples['some_metric'], [1]) 48 | 49 | self.c.add_metric('some_metric') 50 | self.c.sample() 51 | self.assertEqual(self.c.samples['some_metric'], [1, 1]) 52 | 53 | for i in range(15): 54 | self.c.add_metric('some_metric', count=i) 55 | self.c.sample() 56 | self.assertEqual(self.c.samples['some_metric'], range(4, 15)) 57 | 58 | def test_attach_logger(self): 59 | self.c.start() 60 | self.assertIn(self.c.emit, log.theLogPublisher.observers) 61 | self.c.stop() 62 | self.assertNotIn(self.c.emit, log.theLogPublisher.observers) 63 | 64 | 65 | class DateTimeParseTest(unittest.TestCase): 66 | def setUp(self): 67 | os.environ['TZ'] = 'GMT' 68 | time.tzset() 69 | 70 | def test_invalid_date(self): 71 | result = try_datetime_parse("this isn't a date!") 72 | self.assertIsNone(result) 73 | 74 | def test_RFC_1123(self): 75 | result = try_datetime_parse("Thu, 10 Apr 2008 13:30:00 GMT") 76 | self.assertEqual(result, 1207834200.0) 77 | 78 | def test_RFC_1123_subsecond(self): 79 | result = try_datetime_parse("Thu, 10 Apr 2008 13:30:00.12345 GMT") 80 | self.assertEqual(result, 1207834200.0) 81 | 82 | def test_ISO_8601(self): 83 | result = try_datetime_parse("2008-04-10T13:30:00") 84 | self.assertEqual(result, 1207834200.0) 85 | 86 | def test_ISO_8601_subsecond(self): 87 | result = try_datetime_parse("2008-04-10T13:30:00.12345") 88 | self.assertEqual(result, 1207834200.0) 89 | 90 | def test_universal_sortable(self): 91 | result = try_datetime_parse("2008-04-10 13:30:00") 92 | self.assertEqual(result, 1207834200.0) 93 | 94 | def test_universal_sortable_subsecond(self): 95 | result = try_datetime_parse("2008-04-10 13:30:00.12345") 96 | self.assertEqual(result, 1207834200.0) 97 | 98 | def test_date_short(self): 99 | result = try_datetime_parse("2012-04-10") 100 | self.assertEqual(result, 1334016000.0) 101 | 102 | 103 | class ParseKeyValueConfigTest(unittest.TestCase): 104 | def test_single(self): 105 | res = parse_key_value_config('test: 1') 106 | self.assertEqual(res, {'test': '1'}) 107 | 108 | def test_multiple(self): 109 | res = parse_key_value_config('test: 1, test2: 2') 110 | self.assertEqual(res, {'test': '1', 'test2': '2'}) 111 | 112 | def test_empty(self): 113 | res = parse_key_value_config('') 114 | self.assertEqual(res, {}) 115 | 116 | def test_duplicate(self): 117 | res = parse_key_value_config('test: 1, test: 2') 118 | self.assertEqual(res, {'test': '2'}) 119 | 120 | def test_whitespace(self): 121 | res = parse_key_value_config(' test : 1 , test2 : 2 ') 122 | self.assertEqual(res, {'test': '1', 'test2': '2'}) 123 | -------------------------------------------------------------------------------- /swftp/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | See COPYING for license information. 3 | """ 4 | from collections import defaultdict 5 | import time 6 | 7 | from twisted.python import log 8 | from twisted.internet import reactor, tcp 9 | try: 10 | from collections import OrderedDict 11 | except ImportError: 12 | from ordereddict import OrderedDict # NOQA 13 | 14 | DATE_FORMATS = [ 15 | "%a, %d %b %Y %H:%M:%S %Z", 16 | "%a, %d %b %Y %H:%M:%S.%f %Z", 17 | "%Y-%m-%dT%H:%M:%S", 18 | "%Y-%m-%dT%H:%M:%S.%f", 19 | "%Y-%m-%d %H:%M:%S", 20 | "%Y-%m-%d %H:%M:%S.%f", 21 | "%Y-%m-%d" 22 | ] 23 | 24 | GLOBAL_METRICS = [ 25 | 'num_clients', 26 | 'auth.succeed', 27 | 'auth.fail', 28 | 'transfer.egress_bytes', 29 | 'transfer.ingress_bytes', 30 | ] 31 | 32 | 33 | def try_datetime_parse(datetime_str): 34 | """ 35 | Tries to parse the datetime and return the UNIX epoch version of the time. 36 | 37 | returns timestamp(float) or None 38 | """ 39 | mtime = None 40 | if datetime_str: 41 | for date_format in DATE_FORMATS: 42 | try: 43 | mtime_tuple = time.strptime(datetime_str, date_format) 44 | mtime = time.mktime(tuple(mtime_tuple)) 45 | except ValueError: 46 | pass 47 | else: 48 | break 49 | return mtime 50 | 51 | 52 | def parse_key_value_config(config_value): 53 | """ Parses out key-value pairs from a string that has the following format: 54 | key: value, key2: value, key3: value 55 | 56 | :param string config_value: a string to parse key-value pairs from 57 | 58 | :returns dict: 59 | """ 60 | if not config_value: 61 | return {} 62 | 63 | key_values_unparsed = config_value.split(',') 64 | key_values = {} 65 | for key_value_pair in key_values_unparsed: 66 | key, value = key_value_pair.strip().split(':') 67 | key_values[key.strip()] = value.strip() 68 | return key_values 69 | 70 | 71 | class MetricCollector(object): 72 | """ Collects metrics using Twisted Logging 73 | 74 | :param int sample_size: how many samples to save. This is useful for 75 | rolling aggregates. 76 | 77 | Example: 78 | >>> h = MetricCollector() 79 | >>> h.start() 80 | >>> h.totals 81 | {} 82 | >>> log.msg(metric='my_metric') 83 | >>> h.totals 84 | {'my_metric1': 1} 85 | >>> h.samples 86 | >>> h.sample() 87 | {'my_metric1': [1]} 88 | >>> h.sample() 89 | >>> h.samples 90 | {'my_metric1': [1, 0]} 91 | >>> h.stop() 92 | 93 | """ 94 | def __init__(self, sample_size=10): 95 | self.sample_size = sample_size 96 | self.current = defaultdict(int) 97 | self.totals = defaultdict(long) 98 | self.samples = defaultdict(list) 99 | 100 | def emit(self, eventDict): 101 | " If there is a metric in the eventDict, collect that metric " 102 | if 'metric' in eventDict: 103 | self.add_metric(eventDict['metric'], eventDict.get('count', 1)) 104 | 105 | def add_metric(self, metric, count=1): 106 | " Adds a metric with the given count to the totals/current " 107 | self.current[metric] += count 108 | self.totals[metric] += count 109 | 110 | def sample(self): 111 | " Create a sample of the current metrics " 112 | keys = list( 113 | set(self.samples.keys()) | set(self.current.keys())) 114 | 115 | for key in keys: 116 | self.samples[key].append(self.current[key]) 117 | self.samples[key] = \ 118 | self.samples[key][-self.sample_size - 1:] 119 | 120 | self.current = defaultdict(int) 121 | 122 | def start(self): 123 | " Start observing log events " 124 | log.addObserver(self.emit) 125 | 126 | def stop(self): 127 | " Stop observing log events " 128 | log.removeObserver(self.emit) 129 | 130 | 131 | def runtime_info(): 132 | delayed = reactor.getDelayedCalls() 133 | readers = reactor.getReaders() 134 | writers = reactor.getWriters() 135 | servers = [] 136 | clients = [] 137 | other = [] 138 | for reader in readers: 139 | if isinstance(reader, tcp.Server): 140 | servers.append({ 141 | 'transport': reader, 142 | 'host': reader.getHost(), 143 | 'peer': reader.getPeer() 144 | }) 145 | elif isinstance(reader, tcp.Client): 146 | clients.append({ 147 | 'transport': reader, 148 | 'host': reader.getHost(), 149 | 'peer': reader.getPeer() 150 | }) 151 | else: 152 | other.append(reader) 153 | return { 154 | 'num_clients': len(clients), 155 | 'num_servers': len(servers), 156 | 'num_other': len(other), 157 | 'num_writers': len(writers), 158 | 'num_delayed': len(delayed), 159 | 'clients': clients, 160 | 'servers': servers, 161 | 'other': other, 162 | 'writers': writers, 163 | 'delayed': delayed, 164 | } 165 | 166 | 167 | def log_runtime_info(*args): 168 | info = runtime_info() 169 | log.msg("[Servers: %(num_servers)s] [Clients: %(num_clients)s] " 170 | "[Other: %(other)s] [Writers: %(num_writers)s] " 171 | "[DelayedCalls: %(num_delayed)s]" % info) 172 | 173 | for c in info['clients']: 174 | log.msg("[client]: %s" % c) 175 | 176 | for d in info['servers']: 177 | log.msg("[server]: %s" % d) 178 | 179 | for d in info['other']: 180 | log.msg("[other]: %s" % d) 181 | 182 | for d in info['delayed']: 183 | log.msg("[delayed]: %s" % d) 184 | 185 | for w in info['writers']: 186 | log.msg("[writer]: %s" % w) 187 | -------------------------------------------------------------------------------- /twisted/plugins/swftp_ftp.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines serviceMaker, which required for automatic twistd integration for 3 | swftp-ftp 4 | 5 | See COPYING for license information. 6 | """ 7 | from twisted.application.service import ServiceMaker 8 | 9 | serviceMaker = ServiceMaker( 10 | 'swftp-ftp', # name 11 | 'swftp.ftp.service', # module 12 | 'An FTP Proxy Interface for Swift', # description 13 | 'swftp-ftp' # tap name 14 | ) 15 | -------------------------------------------------------------------------------- /twisted/plugins/swftp_sftp.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines serviceMaker, which required for automatic twistd integration for 3 | swftp-sftp 4 | 5 | See COPYING for license information. 6 | """ 7 | from twisted.application.service import ServiceMaker 8 | 9 | serviceMaker = ServiceMaker( 10 | 'swftp-sftp', # name 11 | 'swftp.sftp.service', # module 12 | 'An SFTP Proxy Interface for Swift', # description 13 | 'swftp-sftp' # tap name 14 | ) 15 | --------------------------------------------------------------------------------