├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── bintray.json ├── debian ├── control └── postinst ├── install └── s3.py /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | addons: 2 | apt: 3 | packages: 4 | - fakeroot 5 | deploy: 6 | - 7 | file: bintray.json 8 | key: $BINTRAY_KEY 9 | on: 10 | tags: true 11 | provider: bintray 12 | skip_cleanup: true 13 | user: pauldraper 14 | - 15 | api_key: $GITHUB_AUTH 16 | file: target/apt_boto_s3.deb 17 | on: 18 | tags: true 19 | provider: releases 20 | skip_cleanup: true 21 | script: make dist 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | JOBS ?= 4 2 | 3 | MAKEFLAGS += -r -j $(JOBS) 4 | 5 | .ONESHELL: 6 | 7 | .PHONY: dist 8 | dist: target/apt_boto_s3.deb 9 | 10 | .PHONY: install 11 | install: target/apt_boto_s3.deb 12 | dpkg -i $< 13 | 14 | .PHONY: clean 15 | clean: 16 | rm -fr target 17 | 18 | DEBIAN_SRCS := $(wildcard debian/*) 19 | DEBIAN_TARGETS := $(DEBIAN_SRCS:debian/%=target/apt_boto_s3/DEBIAN/%) 20 | 21 | target/apt_boto_s3/usr/lib/apt/methods/s3: s3.py 22 | @mkdir -p $(@D) 23 | cp --preserve=mode $< $@ 24 | 25 | $(DEBIAN_TARGETS): target/apt_boto_s3/DEBIAN/%: debian/% 26 | @mkdir -p $(@D) 27 | cp --preserve=mode $< $@ 28 | 29 | target/apt_boto_s3.deb: $(DEBIAN_TARGETS) target/apt_boto_s3/usr/lib/apt/methods/s3 30 | fakeroot dpkg-deb --build target/apt_boto_s3 $@ 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # No longer maintained 2 | 3 | This project is no longer maintained. [apt-transport-s3](https://github.com/MayaraCloud/apt-transport-s3) solves the same problems, and is actually in the official Ubuntu repositories. In most cases it is probably a good replacement. 4 | 5 | # apt-boto-s3 6 | 7 | [![Build Status](https://travis-ci.com/lucidsoftware/apt-boto-s3.svg?branch=master)](https://travis-ci.com/lucidsoftware/apt-boto-s3) 8 | [![Package](https://img.shields.io/bintray/v/lucidsoftware/apt/apt-boto-s3.svg)](https://bintray.com/lucidsoftware/apt/apt-boto-s3/_latestVersion) 9 | 10 | The *fast* and *simple* S3 transport for apt. Access S3-hosted apt repositories via the AWS APIs. 11 | 12 | ## Why apt-boto-s3? 13 | 14 | While there are alternative apt transports for S3, like [apt-transport-s3](https://github.com/BashtonLtd/apt-transport-s3) or [apt-s3/apt-transport-s3](https://github.com/castlabs/apt-s3), this project has 15 | 16 | * standard AWS credential resolution, including environment variables and ~/.aws/credentials 17 | * pipelining requests for faster updates 18 | * Last-Modified caching 19 | * broad AWS API support, e.g. v4 credentials 20 | * operability with any S3-compatible API 21 | * works with all standard digest algorithms 22 | * Apache 2.0 license 23 | 24 | ## Install 25 | 26 | Install from the APT repository: 27 | 28 | ``` 29 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 379CE192D401AB61 30 | echo deb http://dl.bintray.com/lucidsoftware/apt/ lucid main > /etc/apt/sources.list.d/lucidsoftware-bintray.list 31 | 32 | apt-get update 33 | apt-get install apt-boto-s3 34 | ``` 35 | 36 | ## Usage 37 | 38 | ### URLs 39 | 40 | The URL in apt sources can have any of the formats [documented](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro) by AWS. 41 | 42 | ``` 43 | # path style 44 | deb s3://s3.amazonaws.com/my-bucket jessie main contrib 45 | 46 | # path style for region other than us-east-1 47 | deb s3://s3-sa-east-1.amazonaws.com/my-bucket jessie main contrib 48 | 49 | # virtual-hosted style 50 | deb s3://my-bucket.s3.amazonaws.com jessie main contrib 51 | ``` 52 | 53 | Any endpoint can be used that has an S3-compatible API. 54 | 55 | ``` 56 | deb s3://swift.example.com/my-bucket jessie main contrib 57 | ``` 58 | 59 | ### Credentials 60 | 61 | apt-boto-s3 resolves AWS credentials in the usual manner. 62 | 63 | 1. Environment variables: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` 64 | 1. Credentials file: `~/.aws/credentials` 65 | 1. Instance metadata: http://169.254.169.254 66 | 67 | Credentials may be also be specified in in the [user information](https://tools.ietf.org/html/rfc3986#section-3.2.1) of the URL. The key and secret should be [URL-encoded](https://tools.ietf.org/html/rfc3986#section-2.1). 68 | 69 | ``` 70 | deb s3://AWS_ACCESS_KEY:AWS_SECRET_KEY@my-bucket.s3.amazonaws.com jessie main contrib 71 | deb s3://AKIAIOSFODNN7EXAMPLE:wJalrXUtnFEMI%2FK7MDENG%2FbPxRfiCYEXAMPLEKEY@my-bucket.s3.amazonaws.com jessie main contrib 72 | ``` 73 | 74 | URL credentials take precendent when present. 75 | 76 | #### Signature version 77 | 78 | Hopefully, this should "just work" and you can ignore this. 79 | 80 | Some regions, e.g. eu-central-1, support only AWS version 4 signatures. However, version 4 does not work with virtual-hosted style URLs, and many S3 clones support only version 2. 81 | 82 | apt-boto-s3 uses version 4 for path style URLs with a s3*.amazonaws.com host; otherwise it uses version 2. 83 | 84 | If you need to override this default, set `S3::Signature::Version` in apt configuration, e.g. in `/etc/apt/apt.conf.d/s3`: 85 | 86 | ``` 87 | S3::Signature::Version "2"; 88 | ``` 89 | 90 | ### Instance metadata service 91 | 92 | You can also tweak the timeout and retry settings for requests to retrieve credentials from the instance metadata. 93 | 94 | ``` 95 | S3::MetadataService::Retries "5"; 96 | S3::MetadataService::Timeout "2"; 97 | ``` 98 | 99 | The default values are 5 retries with a 1 second timeout. 100 | 101 | ## Build 102 | 103 | To build and install from source, 104 | 105 | ```sh 106 | make 107 | make install # as root 108 | ``` 109 | -------------------------------------------------------------------------------- /bintray.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [ 3 | { 4 | "includePattern": "target/(apt_boto_s3)(\\.deb)", 5 | "matrixParams": { 6 | "deb_architecture": "amd64,i386", 7 | "deb_component": "main", 8 | "deb_distribution": "lucid" 9 | }, 10 | "uploadPattern": "pool/main/a/apt-boto-s3/$1_1.5$2" 11 | } 12 | ], 13 | "package": { 14 | "name": "apt-boto-s3", 15 | "repo": "apt", 16 | "subject": "lucidsoftware" 17 | }, 18 | "publish": true, 19 | "version": { 20 | "name": "1.5", 21 | "gpgSign": true, 22 | "vcs_tag": "v1.5" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Architecture: all 2 | Depends: python-pip 3 | Description: The fast and simple S3 transport for apt 4 | Maintainer: Lucid Software 5 | Package: apt-boto-s3 6 | Priority: optional 7 | Section: base 8 | Version: 1.3 9 | -------------------------------------------------------------------------------- /debian/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | pip install boto3 3 | -------------------------------------------------------------------------------- /install: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | apt-get install python python-pip 3 | pip install boto3 4 | cp s3.py /usr/lib/apt/methods/s3 5 | chmod 755 /usr/lib/apt/methods/s3 6 | -------------------------------------------------------------------------------- /s3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | import boto3 3 | import botocore 4 | import collections 5 | import hashlib 6 | import os 7 | import Queue 8 | import re 9 | import signal 10 | import socket 11 | import sys 12 | import threading 13 | import urllib 14 | import urlparse 15 | 16 | class Settings(object): 17 | def __init__(self): 18 | self.metadata_service_num_attempts = 5 19 | self.metadata_service_timeout = 1 20 | self.signature_version = None 21 | def botocore_session(self): 22 | session = botocore.session.get_session() 23 | if self.metadata_service_num_attempts is not None: 24 | session.set_config_variable( 25 | 'metadata_service_num_attempts', 26 | self.metadata_service_num_attempts, 27 | ) 28 | if self.metadata_service_timeout is not None: 29 | session.set_config_variable( 30 | 'metadata_service_timeout', 31 | self.metadata_service_timeout, 32 | ) 33 | return session 34 | settings = Settings() 35 | 36 | class Interrupt(): 37 | def __init__(self): 38 | self.lock = threading.Lock() 39 | self.interrupted = False 40 | 41 | def __nonzero__(self): 42 | return self.interrupted 43 | 44 | def interupt(self): 45 | with self.lock: 46 | if not self.interrupted: 47 | self.interrupted = True 48 | return True 49 | return False 50 | 51 | class MessageHeader(collections.namedtuple('MessageHeader_', ['status_code', 'status_info'])): 52 | def __str__(self): 53 | return '{} {}'.format(self.status_code, self.status_info) 54 | 55 | @staticmethod 56 | def parse(line): 57 | status_code, status_info = line.split(' ', 1) 58 | return MessageHeader(int(status_code), status_info) 59 | 60 | class MessageHeaders: 61 | CAPABILITIES = MessageHeader(100, 'Capabilities') 62 | STATUS = MessageHeader(102, 'Status') 63 | URI_FAILURE = MessageHeader(400, 'URI Failure') 64 | GENERAL_FAILURE = MessageHeader(401, 'General Failure') 65 | URI_START = MessageHeader(200, 'URI Start') 66 | URI_DONE = MessageHeader(201, 'URI Done') 67 | URI_ACQUIRE = MessageHeader(600, 'URI Acquire') 68 | CONFIGURATION = MessageHeader(601, 'Configuration') 69 | 70 | class Message(collections.namedtuple('Message_', ['header', 'fields'])): 71 | @staticmethod 72 | def parse_lines(lines): 73 | return Message(MessageHeader.parse(lines[0]), tuple(re.split(': *', line, 1) for line in lines[1:])) 74 | 75 | def get_field(self, field_name): 76 | return next(self.get_fields(field_name), None) 77 | 78 | def get_fields(self, field_name): 79 | return (value for name, value in self.fields if name.lower() == field_name.lower()) 80 | 81 | def __str__(self): 82 | lines = [str(self.header)] 83 | lines.extend('{}: {}'.format(name, value) for name, value in self.fields) 84 | lines.append('\n') 85 | return '\n'.join(lines) 86 | 87 | Pipes = collections.namedtuple('Pipes', ['input', 'output']) 88 | 89 | class AptIO(object): 90 | @staticmethod 91 | def input(input): 92 | def read_one(): 93 | lines = [] 94 | while True: 95 | line = input.readline() 96 | if not line: 97 | return None 98 | line = line.rstrip('\n') 99 | if line: 100 | lines.append(line) 101 | elif lines: 102 | return Message.parse_lines(lines) 103 | return iter(read_one, None) 104 | 105 | @staticmethod 106 | def output(output): 107 | def send_one(message): 108 | output.write(str(message)) 109 | output.flush() 110 | return send_one 111 | 112 | class AptMethod(object): 113 | def __init__(self, pipes): 114 | self.input = AptIO.input(pipes.input) 115 | self.output = AptIO.output(pipes.output) 116 | 117 | class AptRequest(collections.namedtuple('AptRequest_', ['output'])): 118 | def handle_message(self, message): 119 | try: 120 | self._handle_message(message) 121 | except Exception as ex: 122 | exc_tb = sys.exc_info()[2] 123 | message = '{} ({}, line {})'.format(ex, exc_tb.tb_frame.f_code.co_filename, exc_tb.tb_lineno) 124 | self.output(Message(MessageHeaders.GENERAL_FAILURE, (('Message', message),))) 125 | 126 | class PipelinedAptMethod(AptMethod): 127 | 128 | class Output(object): 129 | def __init__(self, method): 130 | self.method = method 131 | self.queue = Queue.Queue() 132 | self.method.queues.put(self.queue) 133 | 134 | def __enter__(self): 135 | return self.queue.put 136 | 137 | def __exit__(self, type, value, traceback): 138 | self.queue.put(None) 139 | 140 | def send(self, message): 141 | if message.header != MessageHeaders.GENERAL_FAILURE: 142 | self.queue.put(message) 143 | elif self.method.interrupt: 144 | self.queue.put(message) 145 | 146 | def __init__(self, method_type, pipes): 147 | super(PipelinedAptMethod, self).__init__(pipes) 148 | self.interrupt = Interrupt() 149 | self.method_type = method_type 150 | self.queues = Queue.Queue() 151 | 152 | def _send_queue_thread(self): 153 | def f(): 154 | # try: 155 | for queue in iter(self.queues.get, None): 156 | for message in iter(queue.get, None): 157 | self.output(message) 158 | # except IOError: 159 | # pass 160 | thread = threading.Thread(target=f) 161 | thread.start() 162 | return thread 163 | 164 | def _handle_message_thread(self, message): 165 | pipelined_output = self.Output(self) 166 | def f(): 167 | with pipelined_output as output: 168 | self.method_type.request(output).handle_message(message) 169 | thread = threading.Thread(target=f) 170 | thread.start() 171 | return thread 172 | 173 | def run(self): 174 | self.output(Message(MessageHeaders.CAPABILITIES, self.method_type.capabilities())) 175 | # TODO: Use a proper executor. concurrent.futures has them, but only in Python 3.2+. 176 | threads = [self._send_queue_thread()] 177 | for message in self.input: 178 | if self.interrupt: 179 | break 180 | threads.append(self._handle_message_thread(message)) 181 | self.queues.put(None) 182 | for thread in threads: 183 | thread.join() 184 | 185 | class S3AptMethodType(object): 186 | def request(self, output): 187 | return S3AptRequest(output) 188 | 189 | def capabilities(self): 190 | return ( 191 | ('Send-Config', 'true'), 192 | ('Pipeline', 'true'), 193 | ('Single-Instance', 'yes'), 194 | ) 195 | 196 | class S3AptRequest(AptRequest): 197 | def __init__(self, output): 198 | super(S3AptRequest, self).__init__(output) 199 | 200 | class S3Uri: 201 | def __init__(self, request, raw_uri): 202 | self.request = request 203 | self.uri = urlparse.urlparse(raw_uri) 204 | # parse host as if it were an AWS host 205 | match = re.match('(.+\.|)?s3(?:[-.]([^.]*))?.amazonaws.com', self.uri.hostname) 206 | self.virtual_host_bucket, self.region = (match.groups() if match else (None, None)) 207 | 208 | def user_host(self): 209 | parts = self.uri.netloc.split('@', 1) 210 | return parts if len(parts) == 2 else (None, parts[0]) 211 | 212 | def endpoint_url(self): 213 | return 'https://{}/'.format(self.user_host()[1]) 214 | 215 | def credentials(self): 216 | user, _ = self.user_host() 217 | if user: 218 | user_parts = user.split(':', 1) 219 | if len(user_parts) == 2: 220 | return map(urllib.unquote, user_parts) 221 | else: 222 | raise Exception('Access key and secret are specified improperly in the URL') 223 | 224 | role_arn = os.environ.get("role_arn", None) 225 | if role_arn: 226 | creds_rsp = boto3.client('sts').assume_role( 227 | RoleArn=role_arn, 228 | RoleSessionName=socket.gethostname().replace('.', '-'), 229 | ) 230 | if "Credentials" in creds_rsp: 231 | return creds_rsp["Credentials"]["AccessKeyId"], 232 | creds_rsp["Credentials"]["SecretAccessKey"], 233 | creds_rsp["Credentials"]["SessionToken"] 234 | 235 | return None, None, None 236 | 237 | def bucket_key(self): 238 | if self.virtual_host_bucket: 239 | key = self.uri.path[1:] 240 | else: 241 | _, bucket, key = map(urllib.unquote, self.uri.path.split('/', 2)) 242 | return bucket, key 243 | 244 | def signature_version(self): 245 | global settings 246 | if settings.signature_version: 247 | return settings.signature_version 248 | elif self.virtual_host_bucket == '': 249 | return 's3v4' 250 | 251 | def _handle_message(self, message): 252 | global settings 253 | if message.header.status_code == MessageHeaders.CONFIGURATION.status_code: 254 | for config in message.get_fields('Config-Item'): 255 | key, value = config.split('=', 1) 256 | if key == 'S3::Signature::Version': 257 | try: 258 | settings.signature_version = {'2':'s3', '4':'s3v4'}[value] 259 | except KeyError: 260 | raise Exception('Invalid value for S3::Signature::Version') 261 | elif key == 'S3::Credentials::RoleArn': 262 | os.environ["role_arn"] = value 263 | elif key == 'S3::MetadataService::Retries': 264 | try: 265 | metadata_service_num_attempts = int(value) + 1 266 | if metadata_service_num_attempts < 1: 267 | metadata_service_num_attempts = 1 268 | settings.metadata_service_num_attempts = metadata_service_num_attempts 269 | except ValueError: 270 | raise Exception('Invalid value for S3::MetadataService::Retries') 271 | elif key == 'S3::MetadataService::Retries': 272 | try: 273 | metadata_service_timeout = int(value) 274 | if metadata_service_timeout < 1: 275 | metadata_service_timeout = None 276 | settings.metadata_service_timeout = metadata_service_timeout 277 | except ValueError: 278 | raise Exception('Invalid value for S3::MetadataService::Timeout') 279 | elif message.header.status_code == MessageHeaders.URI_ACQUIRE.status_code: 280 | uri = message.get_field('URI') 281 | filename = message.get_field('Filename') 282 | s3_uri = self.S3Uri(self, uri) 283 | 284 | access_key, access_secret, token = s3_uri.credentials() 285 | bucket, key = s3_uri.bucket_key() 286 | 287 | region = s3_uri.region 288 | botocore_session = settings.botocore_session() 289 | if not region and s3_uri.virtual_host_bucket: 290 | # find bucket's region 291 | session = boto3.session.Session( 292 | aws_access_key_id=access_key, 293 | aws_secret_access_key=access_secret, 294 | aws_session_token=token, 295 | region_name='us-east-1', 296 | botocore_session=botocore_session, 297 | ) 298 | s3_client = session.client('s3') 299 | region = s3_client.get_bucket_location(Bucket=bucket)['LocationConstraint'] or 'us-east-1' 300 | session = boto3.session.Session( 301 | aws_access_key_id=access_key, 302 | aws_secret_access_key=access_secret, 303 | aws_session_token=token, 304 | region_name=region or 'us-east-1', 305 | botocore_session=botocore_session, 306 | ) 307 | s3 = session.resource('s3', 308 | config=botocore.client.Config(signature_version=s3_uri.signature_version()), 309 | endpoint_url=s3_uri.endpoint_url(), 310 | ) 311 | s3_object = s3.Bucket(bucket).Object(key) 312 | 313 | self.output(Message(MessageHeaders.STATUS, ( 314 | ('Message', 'Requesting {}/{}'.format(bucket, key)), 315 | ('URI', uri), 316 | ))) 317 | try: 318 | s3_request = {} 319 | last_modified = message.get_field('Last-Modified') 320 | if last_modified: 321 | s3_request['IfModifiedSince'] = last_modified 322 | s3_response = s3_object.get(**s3_request) 323 | except botocore.exceptions.ClientError as error: 324 | if error.response['Error']['Code'] == '304': 325 | self.output(Message(MessageHeaders.URI_DONE, ( 326 | ('Filename', filename), 327 | ('IMS-Hit', 'true'), 328 | ('URI', uri), 329 | ))) 330 | else: 331 | self.output(Message(MessageHeaders.URI_FAILURE, ( 332 | ('Message', error.response['Error']['Message']), 333 | ('URI', uri), 334 | ))) 335 | else: 336 | self.output(Message(MessageHeaders.URI_START, ( 337 | ('Last-Modified', s3_response['LastModified'].isoformat()), 338 | ('Size', s3_response['ContentLength']), 339 | ('URI', uri), 340 | ))) 341 | 342 | md5 = hashlib.md5() 343 | sha1 = hashlib.sha1() 344 | sha256 = hashlib.sha256() 345 | sha512 = hashlib.sha512() 346 | with open(filename, 'wb') as f: 347 | while True: 348 | bytes = s3_response['Body'].read(16 * 1024) 349 | if not bytes: 350 | break 351 | f.write(bytes) 352 | md5.update(bytes) 353 | sha1.update(bytes) 354 | sha256.update(bytes) 355 | sha512.update(bytes) 356 | self.output(Message(MessageHeaders.URI_DONE, ( 357 | ('Filename', filename), 358 | ('Last-Modified', s3_response['LastModified'].isoformat()), 359 | ('MD5-Hash', md5.hexdigest()), 360 | ('MD5Sum-Hash', md5.hexdigest()), 361 | ('SHA1-Hash', sha1.hexdigest()), 362 | ('SHA256-Hash', sha256.hexdigest()), 363 | ('SHA512-Hash', sha512.hexdigest()), 364 | ('Size', s3_response['ContentLength']), 365 | ('URI', uri), 366 | ))) 367 | 368 | if __name__ == '__main__': 369 | # interrupt signals are sometimes sent 370 | def signal_handler(signal, frame): 371 | pass 372 | signal.signal(signal.SIGINT, signal_handler) 373 | 374 | PipelinedAptMethod(S3AptMethodType(), Pipes(sys.stdin, sys.stdout)).run() 375 | --------------------------------------------------------------------------------