├── src ├── main │ ├── resources │ │ ├── chameleon.png │ │ ├── swiftproxy.conf │ │ ├── swiftproxy-saio.conf │ │ ├── copyright_header.txt │ │ ├── logback.xml │ │ └── checkstyle.xml │ ├── java │ │ └── com │ │ │ └── bouncestorage │ │ │ └── swiftproxy │ │ │ ├── BlobStoreLocator.java │ │ │ ├── COPY.java │ │ │ ├── ExceptionLogger.java │ │ │ ├── ContainerNotFoundExceptionMapper.java │ │ │ ├── HttpResponseExceptionMapper.java │ │ │ ├── PlainTextMessageBodyWriter.java │ │ │ ├── ContentLengthAddOn.java │ │ │ ├── v1 │ │ │ ├── InfoResource.java │ │ │ ├── AuthResource.java │ │ │ ├── AccountResource.java │ │ │ └── ContainerResource.java │ │ │ ├── OptionalParamProvider.java │ │ │ ├── Main.java │ │ │ ├── SwiftProxy.java │ │ │ ├── BlobStoreResource.java │ │ │ ├── RuntimeDelegateImpl.java │ │ │ ├── Utils.java │ │ │ ├── BounceResourceConfig.java │ │ │ └── v2 │ │ │ └── Identity.java │ └── assembly │ │ └── jar-with-dependencies.xml └── test │ ├── resources │ ├── swiftproxy.conf │ ├── run-tests.sh │ ├── logback.xml │ ├── run-swiftclient-tests.sh │ ├── run-swiftproxy.sh │ ├── run-swiftclient-python-tests.sh │ └── run-swift-tests.sh │ └── java │ └── com │ └── bouncestorage │ └── swiftproxy │ ├── JcloudsIntegrationTest.java │ ├── v1 │ ├── AuthResourceTest.java │ ├── ObjectResourceTest.java │ ├── AccountResourceTest.java │ └── ContainerResourceTest.java │ ├── BlobStoreLocatorTest.java │ └── TestUtils.java ├── .gitmodules ├── .gitignore ├── .travis.yml ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── .idea └── codeStyleSettings.xml ├── README.md ├── LICENSE └── pom.xml /src/main/resources/chameleon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bouncestorage/swiftproxy/HEAD/src/main/resources/chameleon.png -------------------------------------------------------------------------------- /src/test/resources/swiftproxy.conf: -------------------------------------------------------------------------------- 1 | swiftproxy.endpoint=http://127.0.0.1:0 2 | 3 | jclouds.provider=transient 4 | jclouds.identity=test:tester 5 | jclouds.credential=testing 6 | -------------------------------------------------------------------------------- /src/main/resources/swiftproxy.conf: -------------------------------------------------------------------------------- 1 | swiftproxy.endpoint=http://127.0.0.1:8080 2 | 3 | jclouds.provider=transient 4 | jclouds.identity=test:tester 5 | jclouds.credential=testing 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "swift-tests"] 2 | path = swift-tests 3 | url = https://github.com/bouncestorage/swift.git 4 | [submodule "swiftclient-tests"] 5 | path = swiftclient-tests 6 | url = https://github.com/openstack/python-swiftclient 7 | -------------------------------------------------------------------------------- /src/main/resources/swiftproxy-saio.conf: -------------------------------------------------------------------------------- 1 | swiftproxy.endpoint=http://127.0.0.1:8080 2 | 3 | jclouds.provider=openstack-swift 4 | jclouds.endpoint=http://docker-swift:8080/auth/v1.0 5 | jclouds.keystone.credential-type=tempAuthCredentials 6 | jclouds.identity=test:tester 7 | jclouds.credential=testing 8 | -------------------------------------------------------------------------------- /src/test/resources/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o xtrace 4 | set -o errexit 5 | set -o nounset 6 | 7 | 8 | source $(dirname $0)/run-swiftproxy.sh 9 | 10 | wait_for_swiftproxy 11 | 12 | export SKIP_PROXY=1 13 | 14 | src/test/resources/run-swift-tests.sh 15 | src/test/resources/run-swiftclient-python-tests.sh 16 | exit $? 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | swift-proxy.iml 2 | .idea/ 3 | target/ 4 | virtualenv/ 5 | *~ 6 | 7 | # below is default github .ignore for java 8 | *.class 9 | # Mobile Tools for Java (J2ME) 10 | .mtj.tmp/ 11 | # Package Files # 12 | *.jar 13 | *.war 14 | *.ear 15 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 16 | hs_err_pid* 17 | target/ 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: java 3 | addons: 4 | apt_packages: 5 | - python-pip 6 | - python-virtualenv 7 | - python-dev 8 | - liberasurecode-dev 9 | - libffi-dev 10 | jdk: 11 | - oraclejdk8 12 | # TODO: work around travis-ci/travis-ci#4629 13 | before_install: 14 | - sed -i.bak -e 's|https://nexus.codehaus.org/snapshots/|https://oss.sonatype.org/content/repositories/codehaus-snapshots/|g' ~/.m2/settings.xml 15 | script: 16 | - mvn test 17 | - ./src/test/resources/run-tests.sh 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: "monthly" 12 | - package-ecosystem: "maven" # See documentation for possible values 13 | directory: "/" # Location of package manifests 14 | schedule: 15 | interval: "monthly" 16 | -------------------------------------------------------------------------------- /src/main/resources/copyright_header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Java CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | push: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | java: [ '11' ] 17 | maven: [ '3.6.3' ] 18 | steps: 19 | - uses: actions/checkout@v6 20 | - name: Set up JDK 21 | uses: actions/setup-java@v5 22 | with: 23 | distribution: 'zulu' 24 | java-version: ${{ matrix.java }} 25 | - name: Set up Maven 26 | run: mvn -e -B -V org.apache.maven.plugins:maven-wrapper-plugin:3.3.1:wrapper "-Dmaven=${{ matrix.maven }}" -Dtype=only-script 27 | - name: Build with Maven 28 | run: ./mvnw -e -B -V clean verify 29 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/BlobStoreLocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import java.util.Map; 20 | 21 | import org.jclouds.blobstore.BlobStore; 22 | 23 | public interface BlobStoreLocator { 24 | Map.Entry locateBlobStore(String identity, String container, String blob); 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/COPY.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | import javax.ws.rs.HttpMethod; 25 | 26 | @Target({ElementType.METHOD}) 27 | @Retention(RetentionPolicy.RUNTIME) 28 | @HttpMethod("COPY") 29 | public @interface COPY { 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/ExceptionLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import javax.ws.rs.NotFoundException; 20 | import javax.ws.rs.core.Response; 21 | import javax.ws.rs.ext.Provider; 22 | 23 | import org.glassfish.jersey.spi.ExtendedExceptionMapper; 24 | 25 | @Provider 26 | public final class ExceptionLogger implements ExtendedExceptionMapper { 27 | @Override 28 | public Response toResponse(Throwable t) { 29 | return null; 30 | } 31 | 32 | @Override 33 | public boolean isMappable(Throwable t) { 34 | if (!(t instanceof NotFoundException)) { 35 | t.printStackTrace(); 36 | } 37 | return false; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /.idea/codeStyleSettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 30 | 32 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/ContainerNotFoundExceptionMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import javax.ws.rs.core.Response; 20 | import javax.ws.rs.ext.Provider; 21 | 22 | import org.glassfish.jersey.spi.ExtendedExceptionMapper; 23 | import org.jclouds.blobstore.ContainerNotFoundException; 24 | 25 | @Provider 26 | public class ContainerNotFoundExceptionMapper implements ExtendedExceptionMapper { 27 | @Override 28 | public final Response toResponse(ContainerNotFoundException exception) { 29 | return Response.status(Response.Status.NOT_FOUND).build(); 30 | } 31 | 32 | @Override 33 | public final boolean isMappable(ContainerNotFoundException e) { 34 | return true; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | %.-1p %d{MM-dd HH:mm:ss.SSS} %t %c{30}:%L %X{clientId}|%X{sessionId}:%X{messageId}:%X{fileId}] %m%n 21 | 22 | 23 | ${LOG_LEVEL:-debug} 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | %.-1p %d{MM-dd HH:mm:ss.SSS} %t %c{30}:%L %X{clientId}|%X{sessionId}:%X{messageId}:%X{fileId}] %m%n 21 | 22 | 23 | ${LOG_LEVEL:-debug} 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/HttpResponseExceptionMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import javax.ws.rs.core.Response; 20 | import javax.ws.rs.ext.Provider; 21 | 22 | import org.glassfish.jersey.spi.ExtendedExceptionMapper; 23 | import org.jclouds.http.HttpResponseException; 24 | 25 | @Provider 26 | public final class HttpResponseExceptionMapper implements ExtendedExceptionMapper { 27 | @Override 28 | public Response toResponse(HttpResponseException exception) { 29 | return Response.status(exception.getResponse().getStatusCode()) 30 | .build(); 31 | } 32 | 33 | @Override 34 | public boolean isMappable(HttpResponseException exception) { 35 | return exception.getResponse() != null; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/resources/run-swiftclient-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o xtrace 4 | set -o errexit 5 | set -o nounset 6 | 7 | source $(dirname $0)/run-swiftproxy.sh 8 | 9 | wait_for_swiftproxy 10 | 11 | export PYTHONUNBUFFERED=1 12 | export NOSE_NOCAPTURE=1 13 | export NOSE_NOLOGCAPTURE=1 14 | CURL="stdbuf -oL -eL curl" 15 | 16 | function login { 17 | AUTH=$($CURL -i http://127.0.0.1:8080/auth/v1.0 -X GET \ 18 | -H 'X-Auth-User: test:tester' -H 'X-Auth-Key: testing' \ 19 | | grep -i x-storage-token | sed -e 's/.*: //') 20 | echo $AUTH 21 | } 22 | 23 | AUTH=$(login) 24 | if [ "$AUTH" = "" ]; then 25 | sleep 10 26 | AUTH=$(login) 27 | fi 28 | AUTH=$(login) 29 | if [ "$AUTH" = "" ]; then 30 | echo "Failed to authenticate" 31 | exit 1 32 | fi 33 | 34 | 35 | SWIFT="stdbuf -oL -eL swift --debug -v -v --os-auth-token $AUTH --os-storage-url http://127.0.0.1:8080/v1/AUTH_test:tester" 36 | 37 | $SWIFT capabilities 38 | $SWIFT list 39 | $SWIFT post test_container 40 | $SWIFT stat test_container 41 | $SWIFT list 42 | $SWIFT stat test_container README.md 43 | $SWIFT upload test_container README.md 44 | $SWIFT stat test_container README.md 45 | $SWIFT list test_container 46 | $SWIFT list test_container --lh 47 | $SWIFT delete test_container README.md 48 | $SWIFT list test_container --lh 49 | 50 | dd if=/dev/zero of=BIG bs=1M count=$((32 * 2 + 1)) 51 | $SWIFT upload --segment-container test_container --segment-size $((32 * 1024 * 1024)) test_container BIG 52 | $SWIFT stat test_container BIG 53 | $SWIFT download -o BIG-2 test_container BIG 54 | diff --speed-large-files -q BIG BIG-2 55 | $SWIFT delete test_container BIG 56 | 57 | $SWIFT delete test_container 58 | $SWIFT list 59 | -------------------------------------------------------------------------------- /src/main/assembly/jar-with-dependencies.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 20 | jar-with-dependencies 21 | 22 | jar 23 | 24 | false 25 | 26 | 27 | metaInf-services 28 | 29 | 30 | 31 | 32 | / 33 | true 34 | true 35 | runtime 36 | 37 | 38 | 39 | 40 | ${project.basedir}/src/main/config 41 | / 42 | 43 | logback.xml 44 | 45 | true 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/test/resources/run-swiftproxy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | : ${SKIP_PROXY:="0"} 4 | : ${TRAVIS:="false"} 5 | 6 | function wait_for_swiftproxy 7 | { 8 | 9 | for i in $(seq 30); 10 | do 11 | if exec 3<>"/dev/tcp/localhost/8080"; 12 | then 13 | exec 3<&- # Close for read 14 | exec 3>&- # Close for write 15 | return 16 | fi 17 | sleep 1 18 | done 19 | 20 | # we didn't start correctly 21 | exit 22 | } 23 | 24 | if [ "$SKIP_PROXY" = "1" ]; then 25 | return 26 | fi 27 | 28 | set -o xtrace 29 | set -o errexit 30 | set -o nounset 31 | 32 | function cleanup { 33 | if [ "$DOCKER" != "" ]; then 34 | sudo docker stop $DOCKER 35 | sudo docker rm $DOCKER 36 | fi 37 | if [ "$PROXY_PID" != "" ]; then 38 | kill $PROXY_PID 39 | fi 40 | } 41 | 42 | trap cleanup EXIT 43 | 44 | if [ "$TRAVIS" != "true" ]; then 45 | DOCKER=$(sudo docker run -d kahing/docker-swift) 46 | DOCKER_IP=$(sudo docker inspect -f '{{ .NetworkSettings.IPAddress }}' $DOCKER) 47 | export NOSE_NOCAPTURE=1 48 | export NOSE_NOLOGCAPTURE=1 49 | export MAVEN_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005" 50 | 51 | cat > target/swiftproxy-saio.conf < target/swiftproxy-saio.conf < ./virtualenv/etc/swift/test.conf < 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import java.io.IOException; 20 | import java.io.OutputStream; 21 | import java.lang.annotation.Annotation; 22 | import java.lang.reflect.Type; 23 | import java.nio.charset.StandardCharsets; 24 | import java.util.Collection; 25 | 26 | import javax.ws.rs.Produces; 27 | import javax.ws.rs.WebApplicationException; 28 | import javax.ws.rs.core.MediaType; 29 | import javax.ws.rs.core.MultivaluedMap; 30 | import javax.ws.rs.ext.MessageBodyWriter; 31 | import javax.ws.rs.ext.Provider; 32 | 33 | @Provider 34 | @Produces(MediaType.TEXT_PLAIN) 35 | public final class PlainTextMessageBodyWriter implements MessageBodyWriter { 36 | @Override 37 | public boolean isWriteable(Class aClass, Type type, Annotation[] annotations, MediaType mediaType) { 38 | return Collection.class.isAssignableFrom(aClass); 39 | } 40 | 41 | @Override 42 | public long getSize(Collection collection, Class aClass, Type type, Annotation[] annotations, MediaType mediaType) { 43 | return -1; 44 | } 45 | 46 | @Override 47 | public void writeTo(Collection collection, Class aClass, Type type, Annotation[] annotations, MediaType mediaType, MultivaluedMap multivaluedMap, OutputStream outputStream) throws IOException, WebApplicationException { 48 | for (Object o: collection) { 49 | outputStream.write(o.toString().getBytes(StandardCharsets.UTF_8)); 50 | outputStream.write('\n'); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/com/bouncestorage/swiftproxy/JcloudsIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import static com.google.common.base.Throwables.propagate; 20 | 21 | import java.util.Properties; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import com.google.common.util.concurrent.Uninterruptibles; 25 | 26 | import org.jclouds.Constants; 27 | import org.jclouds.openstack.keystone.config.KeystoneProperties; 28 | import org.jclouds.openstack.swift.v1.blobstore.integration.SwiftBlobIntegrationLiveTest; 29 | import org.testng.annotations.AfterSuite; 30 | 31 | public final class JcloudsIntegrationTest extends SwiftBlobIntegrationLiveTest { 32 | protected static final int AWAIT_CONSISTENCY_TIMEOUT_SECONDS = Integer.parseInt(System.getProperty( 33 | "test.blobstore.await-consistency-timeout-seconds", "0")); 34 | private SwiftProxy proxy; 35 | 36 | public JcloudsIntegrationTest() throws Exception { 37 | } 38 | 39 | @AfterSuite 40 | @Override 41 | public void destroyResources() { 42 | proxy.stop(); 43 | } 44 | 45 | @Override 46 | protected void awaitConsistency() { 47 | Uninterruptibles.sleepUninterruptibly(AWAIT_CONSISTENCY_TIMEOUT_SECONDS, TimeUnit.SECONDS); 48 | } 49 | 50 | @Override 51 | protected Properties setupProperties() { 52 | try { 53 | proxy = TestUtils.setupAndStartProxy(); 54 | } catch (Exception e) { 55 | throw propagate(e); 56 | } 57 | Properties props = super.setupProperties(); 58 | identity = "test:tester"; 59 | credential = "testing"; 60 | endpoint = proxy.getEndpoint().toString() + "/auth/v1.0"; 61 | props.setProperty(KeystoneProperties.CREDENTIAL_TYPE, "tempAuthCredentials"); 62 | props.setProperty(Constants.PROPERTY_IDENTITY, identity); 63 | props.setProperty(Constants.PROPERTY_CREDENTIAL, credential); 64 | props.setProperty(Constants.PROPERTY_ENDPOINT, endpoint); 65 | return props; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/ContentLengthAddOn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import javax.ws.rs.core.MediaType; 20 | import javax.ws.rs.core.Response; 21 | 22 | import org.glassfish.grizzly.filterchain.FilterChainBuilder; 23 | import org.glassfish.grizzly.filterchain.FilterChainContext; 24 | import org.glassfish.grizzly.http.HttpHeader; 25 | import org.glassfish.grizzly.http.HttpResponsePacket; 26 | import org.glassfish.grizzly.http.HttpServerFilter; 27 | import org.glassfish.grizzly.http.server.AddOn; 28 | import org.glassfish.grizzly.http.server.NetworkListener; 29 | 30 | public final class ContentLengthAddOn implements AddOn { 31 | @Override 32 | public void setup(NetworkListener networkListener, FilterChainBuilder builder) { 33 | 34 | // Get the index of HttpCodecFilter in the HttpServer filter chain 35 | final int httpCodecFilterIdx = builder.indexOfType(HttpServerFilter.class); 36 | 37 | if (httpCodecFilterIdx >= 0) { 38 | // Insert the WebSocketFilter right after HttpCodecFilter 39 | HttpServerFilter originalFilter = (HttpServerFilter) builder.get(httpCodecFilterIdx); 40 | builder.set(httpCodecFilterIdx, new ContentLengthFilter(originalFilter)); 41 | } 42 | } 43 | 44 | private static class ContentLengthFilter extends HttpServerFilter { 45 | ContentLengthFilter(HttpServerFilter original) { 46 | setAllowPayloadForUndefinedHttpMethods(original.isAllowPayloadForUndefinedHttpMethods()); 47 | } 48 | 49 | @Override 50 | protected void onInitialLineEncoded(HttpHeader header, FilterChainContext ctx) { 51 | super.onInitialLineEncoded(header, ctx); 52 | 53 | if (!header.isCommitted()) { 54 | final HttpResponsePacket response = (HttpResponsePacket) header; 55 | if (response.getStatus() == Response.Status.NO_CONTENT.getStatusCode()) { 56 | response.getHeaders().setValue("Content-Length").setString("0"); 57 | response.getHeaders().setValue("Content-Type").setString(MediaType.TEXT_PLAIN); 58 | } 59 | } 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/com/bouncestorage/swiftproxy/v1/AuthResourceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import java.io.InputStream; 20 | import java.net.URI; 21 | import java.util.Properties; 22 | 23 | import com.bouncestorage.swiftproxy.SwiftProxy; 24 | import com.bouncestorage.swiftproxy.TestUtils; 25 | import com.google.common.io.Resources; 26 | 27 | import org.jclouds.ContextBuilder; 28 | import org.jclouds.blobstore.BlobStoreContext; 29 | import org.junit.After; 30 | import org.junit.Before; 31 | import org.junit.Test; 32 | 33 | public final class AuthResourceTest { 34 | private SwiftProxy proxy; 35 | private URI endpoint; 36 | private Properties properties; 37 | 38 | @Before 39 | public void setup() throws Exception { 40 | properties = new Properties(); 41 | try (InputStream is = Resources.asByteSource(Resources.getResource( 42 | "swiftproxy.conf")).openStream()) { 43 | properties.load(is); 44 | } 45 | proxy = SwiftProxy.Builder.builder() 46 | .overrides(properties) 47 | .build(); 48 | proxy.start(); 49 | proxy = TestUtils.setupAndStartProxy(); 50 | endpoint = proxy.getEndpoint(); 51 | } 52 | 53 | @After 54 | public void tearDown() throws Exception { 55 | if (proxy != null) { 56 | proxy.stop(); 57 | } 58 | } 59 | 60 | @Test 61 | public void testV1() throws Exception { 62 | properties.setProperty("jclouds.keystone.credential-type", "tempAuthCredentials"); 63 | BlobStoreContext context = ContextBuilder.newBuilder("openstack-swift") 64 | .endpoint(endpoint.toString() + "/auth/v1.0") 65 | .overrides(properties) 66 | .build(BlobStoreContext.class); 67 | context.getBlobStore().list(); 68 | } 69 | 70 | @Test 71 | public void testV2() throws Exception { 72 | BlobStoreContext context = ContextBuilder.newBuilder("openstack-swift") 73 | .endpoint(endpoint.toString() + "/v2.0") 74 | .overrides(properties) 75 | .build(BlobStoreContext.class); 76 | context.getBlobStore().list(); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/v1/InfoResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import javax.ws.rs.GET; 20 | import javax.ws.rs.Path; 21 | import javax.ws.rs.Produces; 22 | import javax.ws.rs.core.MediaType; 23 | 24 | import com.fasterxml.jackson.annotation.JsonProperty; 25 | 26 | @Path("/info") 27 | public final class InfoResource { 28 | public static final ServerConfiguration CONFIG = new ServerConfiguration(); 29 | 30 | @GET 31 | @Produces(MediaType.APPLICATION_JSON) 32 | public ServerConfiguration getInfo() { 33 | return CONFIG; 34 | } 35 | 36 | //CHECKSTYLE:OFF 37 | public static final class ServerConfiguration { 38 | @JsonProperty public final SwiftConfiguration swift = new SwiftConfiguration(); 39 | @JsonProperty public final SLOConfiguration slo = new SLOConfiguration(); 40 | @JsonProperty public final TempAuthConfiguration tempauth = new TempAuthConfiguration(); 41 | 42 | static final class SwiftConfiguration { 43 | @JsonProperty final int account_listing_limit = 10000; 44 | @JsonProperty final boolean allow_account_management = false; 45 | @JsonProperty final int container_listing_limit = 10000; 46 | @JsonProperty final int max_account_name_length = 256; 47 | @JsonProperty final int max_container_name_length = 256; 48 | @JsonProperty final long max_file_size = 5368709122L; 49 | @JsonProperty final int max_header_size = 8192; 50 | @JsonProperty final int max_meta_name_length = 128; 51 | @JsonProperty final int max_meta_value_length = 256; 52 | @JsonProperty final int max_meta_count = 90; 53 | @JsonProperty final int max_meta_overall_size = 2048; 54 | @JsonProperty final int max_object_name_length = 1024; 55 | @JsonProperty final boolean strict_cors_mode = true; 56 | } 57 | 58 | static final class SLOConfiguration { 59 | @JsonProperty final int max_manifest_segments = 1000; 60 | } 61 | 62 | public static final class TempAuthConfiguration { 63 | @JsonProperty public final int token_life = 85400; 64 | } 65 | } 66 | //CHECKSTYLE:ON 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/v1/AuthResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import java.util.Optional; 20 | 21 | import javax.ws.rs.GET; 22 | import javax.ws.rs.HeaderParam; 23 | import javax.ws.rs.Path; 24 | import javax.ws.rs.core.Context; 25 | import javax.ws.rs.core.Response; 26 | 27 | import com.bouncestorage.swiftproxy.BlobStoreResource; 28 | import com.bouncestorage.swiftproxy.BounceResourceConfig; 29 | 30 | import org.glassfish.grizzly.http.server.Request; 31 | 32 | /** 33 | * Implements TempAuth (V1 Auth) for Swift. Documentations: 34 | * http://docs.openstack.org/developer/swift/overview_auth.html 35 | * https://swiftstack.com/docs/cookbooks/swift_usage/auth.html 36 | * http://docs.openstack.org/developer/swift/deployment_guide.html 37 | */ 38 | @Path("/auth/v1.0") 39 | public final class AuthResource extends BlobStoreResource { 40 | @GET 41 | public Response auth(@HeaderParam("X-Auth-User") Optional authUser, 42 | @HeaderParam("X-Auth-Key") Optional authKey, 43 | @HeaderParam("X-Storage-User") Optional storageUser, 44 | @HeaderParam("X-Storage-Pass") Optional storagePass, 45 | @HeaderParam("Host") Optional host, 46 | @Context Request request) { 47 | String identity = authUser.orElseGet(storageUser::get); 48 | String credential = authKey.orElseGet(storagePass::get); 49 | String authToken = null; 50 | try { 51 | authToken = ((BounceResourceConfig) application).authenticate(identity, credential); 52 | } catch (Throwable e) { 53 | e.printStackTrace(); 54 | } 55 | if (authToken == null) { 56 | return notAuthorized(); 57 | } 58 | 59 | String storageURL = host.orElseGet(() -> request.getLocalAddr() + ":" + request.getLocalPort()); 60 | String scheme = request.getScheme(); 61 | storageURL = scheme + "://" + storageURL + "/v1/AUTH_" + identity; 62 | 63 | return Response.ok() 64 | .header("x-storage-url", storageURL) 65 | .header("x-auth-token", authToken) 66 | .header("x-storage-token", authToken) 67 | .build(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/OptionalParamProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import java.lang.annotation.Annotation; 20 | import java.lang.reflect.Method; 21 | import java.lang.reflect.ParameterizedType; 22 | import java.lang.reflect.Type; 23 | import java.util.Optional; 24 | 25 | import javax.ws.rs.ext.ParamConverter; 26 | import javax.ws.rs.ext.ParamConverterProvider; 27 | import javax.ws.rs.ext.Provider; 28 | 29 | @Provider 30 | public final class OptionalParamProvider implements ParamConverterProvider { 31 | @Override 32 | public ParamConverter getConverter(Class aClass, Type type, Annotation[] annotations) { 33 | if (aClass == Optional.class) { 34 | try { 35 | return (ParamConverter) new OptionalParamConverter(type); 36 | } catch (NoSuchMethodException | IllegalArgumentException e) { 37 | return null; 38 | } 39 | } 40 | 41 | return null; 42 | } 43 | 44 | private static class OptionalParamConverter implements ParamConverter { 45 | Method valueOf; 46 | OptionalParamConverter(Type type) throws NoSuchMethodException { 47 | Class classType; 48 | if (type instanceof Class) { 49 | classType = (Class) type; 50 | } else if (type instanceof ParameterizedType) { 51 | classType = (Class) ((ParameterizedType) type).getActualTypeArguments()[0]; 52 | } else { 53 | throw new IllegalArgumentException("type"); 54 | } 55 | 56 | try { 57 | valueOf = classType.getDeclaredMethod("valueOf", String.class); 58 | } catch (NoSuchMethodException e) { 59 | valueOf = classType.getDeclaredMethod("valueOf", Object.class); 60 | } 61 | } 62 | 63 | @Override 64 | public Optional fromString(String s) { 65 | if (s == null) { 66 | return Optional.empty(); 67 | } 68 | 69 | try { 70 | Object o = valueOf.invoke(null, s); 71 | return Optional.of(o); 72 | } catch (ReflectiveOperationException e) { 73 | throw new IllegalArgumentException(e); 74 | } 75 | } 76 | 77 | @Override 78 | public String toString(Optional optional) { 79 | return String.valueOf(optional.orElse(null)); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/test/java/com/bouncestorage/swiftproxy/BlobStoreLocatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import static org.assertj.core.api.Assertions.assertThat; 20 | 21 | import javax.ws.rs.client.Client; 22 | import javax.ws.rs.client.ClientBuilder; 23 | import javax.ws.rs.client.WebTarget; 24 | import javax.ws.rs.core.Response; 25 | 26 | import com.google.common.base.Joiner; 27 | import com.google.common.collect.ImmutableMap; 28 | import com.google.common.collect.Maps; 29 | 30 | import org.jclouds.ContextBuilder; 31 | import org.jclouds.blobstore.BlobStore; 32 | import org.jclouds.blobstore.BlobStoreContext; 33 | import org.junit.After; 34 | import org.junit.Before; 35 | import org.junit.Test; 36 | 37 | public final class BlobStoreLocatorTest { 38 | private SwiftProxy proxy; 39 | private WebTarget target; 40 | private ImmutableMap blobStoreMap; 41 | 42 | @Before 43 | public void setUp() throws Exception { 44 | proxy = TestUtils.setupAndStartProxy(); 45 | // create the client 46 | Client c = ClientBuilder.newClient(); 47 | 48 | // uncomment the following line if you want to enable 49 | // support for JSON in the client (you also have to uncomment 50 | // dependency on jersey-media-json module in pom.xml and Main.startServer()) 51 | // -- 52 | // c.configuration().enable(new org.glassfish.jersey.media.json.JsonJaxbFeature()); 53 | 54 | target = c.target(proxy.getEndpoint()); 55 | } 56 | 57 | @After 58 | public void tearDown() throws Exception { 59 | proxy.stop(); 60 | if (blobStoreMap != null) { 61 | blobStoreMap.forEach((key, value) -> { 62 | if (value != null) { 63 | value.getContext().close(); 64 | } 65 | }); 66 | } 67 | } 68 | 69 | @Test 70 | public void testBlobStoreLocator() throws Exception { 71 | ContextBuilder context = ContextBuilder.newBuilder("transient"); 72 | BlobStore foo = context.build(BlobStoreContext.class).getBlobStore(); 73 | ContextBuilder otherContext = ContextBuilder.newBuilder("transient"); 74 | BlobStore bar = otherContext.build(BlobStoreContext.class).getBlobStore(); 75 | blobStoreMap = ImmutableMap.of("foo", foo, "bar", bar); 76 | proxy.setBlobStoreLocator((identity, container, blob) -> { 77 | if (blobStoreMap.containsKey(identity)) { 78 | return Maps.immutableEntry(identity, blobStoreMap.get(identity)); 79 | } 80 | return null; 81 | }); 82 | 83 | String path = "/auth/v1.0"; 84 | Response resp = target.path(path).request().header("X-auth-user", "foo").header("X-auth-key", "foo") 85 | .get(); 86 | String storageURL = resp.getHeaderString("x-storage-url"); 87 | String authToken = resp.getHeaderString("x-auth-token"); 88 | assertThat(storageURL).isNotNull(); 89 | assertThat(authToken).isNotNull(); 90 | String container = "container"; 91 | path = Joiner.on("/").join(TestUtils.ACCOUNT_PATH, container); 92 | target.path(path).request().header("X-auth-token", authToken) 93 | .post(null); 94 | assertThat(foo.containerExists(container)).isTrue(); 95 | assertThat(bar.containerExists(container)).isFalse(); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | SwiftProxy ![logo](src/main/resources/chameleon.png) 2 | ======= 3 | SwiftProxy allows applications using the 4 | [Swift API](https://wiki.openstack.org/wiki/Swift) 5 | to access other object stores, 6 | e.g., Amazon S3, EMC Atmos, Google Cloud Storage, Microsoft Azure. 7 | It also allows local testing of Swift without the complication of actually setting up Swift. 8 | Finally users can extend SwiftProxy with custom middlewares, e.g., caching, 9 | encryption, tiering. 10 | 11 | Features 12 | -------- 13 | * create, remove, and list containers 14 | * put, get, delete, and list objects 15 | * large objects (static and dynamic) 16 | * copy objects 17 | * store and retrieve object metadata, including user metadata 18 | * authorization via V1 Auth 19 | 20 | Supported object stores: 21 | 22 | * atmos 23 | * aws-s3 24 | * azureblob 25 | * filesystem (on-disk storage) 26 | * google-cloud-storage 27 | * hpcloud-objectstorage 28 | * openstack-swift 29 | * rackspace-cloudfiles-uk and rackspace-cloudfiles-us 30 | * s3 31 | * swift and swift-keystone (legacy) 32 | * transient (in-memory storage) 33 | 34 | Installation 35 | ------------ 36 | 37 | Users can 38 | [download releases](https://github.com/bouncestorage/swiftproxy/releases) 39 | from GitHub. One can also build the project by running `mvn package` 40 | which produces a binary at 41 | `target/swift-proxy-1.0.0-jar-with-dependencies.jar`. 42 | SwiftProxy requires Java 8 to run. 43 | 44 | Examples 45 | -------- 46 | 47 | ``` 48 | java -jar ./swift-proxy-1.0.0-jar-with-dependencies.jar --properties swiftproxy.conf 49 | ``` 50 | 51 | Users can configure SwiftProxy via a properties file. An example 52 | using Amazon S3 as the backing store: 53 | 54 | ``` 55 | swiftproxy.endpoint=http://0.0.0.0:8080 56 | 57 | jclouds.provider=aws-s3 58 | jclouds.identity=AWS_ACCESSKEY 59 | jclouds.credential=AWS_CREDENTIAL 60 | jclouds.region=us-west-2 61 | ``` 62 | 63 | SwiftProxy forwards authentication to the underlying object store, 64 | with the above configuration you can access Amazon S3 with: 65 | 66 | ``` 67 | $ swift -A http://127.0.0.1:8080/auth/v1.0 -U AWS_ACCESSKEY -K AWS_CREDENTIAL list 68 | ``` 69 | 70 | Another example using the local file system as the backing store: 71 | 72 | ``` 73 | swiftproxy.endpoint=http://127.0.0.1:8080 74 | jclouds.provider=filesystem 75 | jclouds.identity=test:tester 76 | jclouds.credential=testing 77 | jclouds.filesystem.basedir=/tmp/swiftproxy 78 | ``` 79 | 80 | Users can also set other Java and 81 | [jclouds](https://github.com/jclouds/jclouds/blob/master/core/src/main/java/org/jclouds/Constants.java) 82 | properties. 83 | 84 | Limitations 85 | ----------- 86 | 87 | SwiftProxy does not support: 88 | 89 | * object metadata with filesystem provider on Mac OS X 90 | ([OpenJDK issue](https://bugs.openjdk.java.net/browse/JDK-8030048)) 91 | * object versioning 92 | * ACLs, container metadata and container syncing 93 | * object auto delete (`X-Delete-At` or `X-Delete-After`) 94 | * HTTPS frontend (Connecting to HTTPS object store is supported) 95 | * TempURL 96 | 97 | Testing 98 | ------- 99 | 100 | SwiftProxy itself has limited tests and those can be run via `mvn 101 | test`. We use Swift's functional tests to catch incompatibilities with 102 | the Swift API. SwiftProxy passes a large subset of Swift tests, and 103 | there's a helper script to run them against SwiftProxy 104 | (`src/test/resources/run-swift-tests.sh`). 105 | 106 | References 107 | ---------- 108 | 109 | * Apache [jclouds](http://jclouds.apache.org/) provides object store 110 | support for SwiftProxy 111 | * [OpenStack Swift tests](https://github.com/openstack/swift/tree/master/test/functional) 112 | used to maintain and improve compatibility with the Swift API 113 | * [Docker Swift](https://github.com/ualbertalib/docker-swift) provides 114 | functionality similar to SwiftProxy when using the filesystem 115 | provider 116 | * [S3Proxy](https://github.com/andrewgaul/s3proxy) provided inspiration 117 | for this project 118 | 119 | 120 | License 121 | ------- 122 | Copyright (C) 2015 Bounce Storage 123 | 124 | Licensed under the Apache License, Version 2.0 125 | -------------------------------------------------------------------------------- /src/test/java/com/bouncestorage/swiftproxy/v1/ObjectResourceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import static org.assertj.core.api.Assertions.assertThat; 20 | 21 | import java.util.Random; 22 | 23 | import javax.ws.rs.client.Client; 24 | import javax.ws.rs.client.ClientBuilder; 25 | import javax.ws.rs.client.Entity; 26 | import javax.ws.rs.client.WebTarget; 27 | import javax.ws.rs.core.MediaType; 28 | import javax.ws.rs.core.Response; 29 | 30 | import com.bouncestorage.swiftproxy.SwiftProxy; 31 | import com.bouncestorage.swiftproxy.TestUtils; 32 | import com.google.common.base.Joiner; 33 | 34 | import org.junit.After; 35 | import org.junit.Before; 36 | import org.junit.Test; 37 | 38 | public final class ObjectResourceTest { 39 | private static final String CONTAINER = "swiftproxy-test-container-" + new Random().nextInt(Integer.MAX_VALUE); 40 | private static final String BLOB_NAME = "blob"; 41 | private final String path; 42 | 43 | private SwiftProxy proxy; 44 | private WebTarget target; 45 | private String authToken; 46 | 47 | public ObjectResourceTest() { 48 | String[] parts = {TestUtils.ACCOUNT_PATH, CONTAINER, BLOB_NAME}; 49 | path = Joiner.on("/").join(parts); 50 | } 51 | 52 | @Before 53 | public void setup() throws Exception { 54 | proxy = TestUtils.setupAndStartProxy(); 55 | Client c = ClientBuilder.newClient(); 56 | target = c.target(proxy.getEndpoint()); 57 | 58 | authToken = TestUtils.createContainer(target, CONTAINER); 59 | } 60 | 61 | @After 62 | public void tearDown() throws Exception { 63 | if (proxy != null) { 64 | proxy.stop(); 65 | } 66 | } 67 | 68 | @Test 69 | public void testPut() throws Exception { 70 | String data = "foo"; 71 | putObject(target.path(path), data.getBytes()); 72 | 73 | Response resp = target.path(path).request().header("x-auth-token", authToken).get(); 74 | assertThat(resp.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); 75 | assertThat(resp.readEntity(String.class)).isEqualTo(data); 76 | assertThat(resp.getLength()).isEqualTo(data.length()); 77 | assertThat(resp.getMediaType().toString()).isEqualTo(MediaType.APPLICATION_OCTET_STREAM); 78 | } 79 | 80 | @Test 81 | public void testMissingObject() throws Exception { 82 | Response resp = target.path(path).request().header("x-auth-token", authToken).get(); 83 | assertThat(resp.getStatus()).isEqualTo(Response.Status.NOT_FOUND.getStatusCode()); 84 | } 85 | 86 | @Test 87 | public void testHead() throws Exception { 88 | String data = "foo"; 89 | putObject(target.path(path), data.getBytes()); 90 | 91 | Response resp = target.path(path).request().header("x-auth-token", authToken).head(); 92 | // TODO: this should be fixed once the Jersey issue is resolved and we return NO_CONTENT 93 | assertThat(resp.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); 94 | assertThat(resp.getLength()).isEqualTo(data.length()); 95 | assertThat(resp.getMediaType().toString()).isEqualTo(MediaType.APPLICATION_OCTET_STREAM); 96 | } 97 | 98 | Response putObject(WebTarget putTarget, byte[] data) throws Exception { 99 | Response resp = target.path(path).request() 100 | .header("x-auth-token", authToken) 101 | .put(Entity.entity(data, MediaType.APPLICATION_OCTET_STREAM)); 102 | assertThat(resp.getStatus()).isEqualTo(Response.Status.CREATED.getStatusCode()); 103 | assertThat(resp.getMediaType().toString()).isEqualTo(MediaType.APPLICATION_OCTET_STREAM); 104 | return resp; 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/Main.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import java.io.InputStream; 20 | import java.net.URI; 21 | import java.nio.charset.StandardCharsets; 22 | import java.nio.file.FileSystem; 23 | import java.nio.file.FileSystems; 24 | import java.nio.file.Files; 25 | import java.nio.file.Path; 26 | import java.util.Properties; 27 | 28 | import com.google.common.collect.ImmutableList; 29 | import com.google.inject.Module; 30 | 31 | import org.jclouds.Constants; 32 | import org.jclouds.ContextBuilder; 33 | import org.jclouds.blobstore.BlobStoreContext; 34 | import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; 35 | 36 | /** 37 | * Main class. 38 | * 39 | */ 40 | public final class Main { 41 | private Main() { 42 | // Hide the main method 43 | } 44 | 45 | /** 46 | * Main method. 47 | * @param args 48 | */ 49 | public static void main(String[] args) throws Exception { 50 | if (args.length == 1 && args[0].equals("--version")) { 51 | System.err.println( 52 | Main.class.getPackage().getImplementationVersion()); 53 | System.exit(0); 54 | } else if (args.length != 2) { 55 | System.err.println("Usage: swiftproxy --properties FILE"); 56 | System.exit(1); 57 | } 58 | 59 | FileSystem fs = FileSystems.getDefault(); 60 | Properties properties = new Properties(); 61 | try (InputStream is = Files.newInputStream(fs.getPath(args[1]))) { 62 | properties.load(is); 63 | } 64 | properties.putAll(System.getProperties()); 65 | 66 | String provider = properties.getProperty(Constants.PROPERTY_PROVIDER); 67 | String identity = properties.getProperty(Constants.PROPERTY_IDENTITY); 68 | String credential = properties.getProperty(Constants.PROPERTY_CREDENTIAL); 69 | String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT); 70 | String proxyEndpoint = properties.getProperty(SwiftProxy.PROPERTY_ENDPOINT); 71 | if (provider == null || identity == null || credential == null || proxyEndpoint == null) { 72 | System.err.format("Properties file must contain:%n" + 73 | Constants.PROPERTY_PROVIDER + "%n" + 74 | Constants.PROPERTY_IDENTITY + "%n" + 75 | Constants.PROPERTY_CREDENTIAL + "%n" + 76 | SwiftProxy.PROPERTY_ENDPOINT + "%n"); 77 | System.exit(1); 78 | } 79 | 80 | if (provider.equals("google-cloud-storage")) { 81 | Path credentialPath = fs.getPath(credential); 82 | if (Files.exists(credentialPath)) { 83 | credential = new String(Files.readAllBytes(credentialPath), 84 | StandardCharsets.UTF_8); 85 | } 86 | properties.remove(Constants.PROPERTY_CREDENTIAL); 87 | } 88 | 89 | ContextBuilder builder = ContextBuilder 90 | .newBuilder(provider) 91 | .credentials(identity, credential) 92 | .modules(ImmutableList.of(new SLF4JLoggingModule())) 93 | .overrides(properties); 94 | if (endpoint != null) { 95 | builder = builder.endpoint(endpoint); 96 | } 97 | BlobStoreContext context = builder.build(BlobStoreContext.class); 98 | 99 | SwiftProxy proxy = SwiftProxy.Builder.builder() 100 | .overrides(properties) 101 | .endpoint(new URI(proxyEndpoint)) 102 | .build(); 103 | proxy.start(); 104 | System.out.format("Swift proxy listening on port %d%n", proxy.getPort()); 105 | Thread.currentThread().join(); 106 | } 107 | } 108 | 109 | -------------------------------------------------------------------------------- /src/test/java/com/bouncestorage/swiftproxy/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import static org.assertj.core.api.Assertions.assertThat; 20 | 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.nio.charset.StandardCharsets; 24 | import java.nio.file.FileSystem; 25 | import java.nio.file.FileSystems; 26 | import java.nio.file.Files; 27 | import java.nio.file.Path; 28 | import java.util.Optional; 29 | import java.util.Properties; 30 | 31 | import javax.ws.rs.client.WebTarget; 32 | import javax.ws.rs.core.Response; 33 | 34 | import com.google.common.io.Resources; 35 | 36 | import org.jclouds.Constants; 37 | 38 | public final class TestUtils { 39 | public static final String ACCOUNT_PATH = "/v1/testAccount"; 40 | 41 | private TestUtils() { 42 | // Hide the constructor for a Utils class 43 | } 44 | 45 | public static SwiftProxy setupAndStartProxy() throws Exception { 46 | Properties properties = new Properties(); 47 | try (InputStream is = Resources.asByteSource(Resources.getResource( 48 | "swiftproxy.conf")).openStream()) { 49 | properties.load(is); 50 | } 51 | 52 | String provider = properties.getProperty(Constants.PROPERTY_PROVIDER); 53 | String credential = properties.getProperty(Constants.PROPERTY_CREDENTIAL); 54 | if (provider != null && credential != null && provider.equals("google-cloud-storage")) { 55 | FileSystem fs = FileSystems.getDefault(); 56 | Path credentialPath = fs.getPath(credential); 57 | if (Files.exists(credentialPath)) { 58 | credential = new String(Files.readAllBytes(credentialPath), 59 | StandardCharsets.UTF_8); 60 | } 61 | properties.put(Constants.PROPERTY_CREDENTIAL, credential); 62 | } 63 | 64 | SwiftProxy proxy = SwiftProxy.Builder.builder() 65 | .overrides(properties) 66 | .build(); 67 | proxy.start(); 68 | return proxy; 69 | } 70 | 71 | public static String getAuthToken(WebTarget target) { 72 | Properties properties = new Properties(); 73 | try (InputStream is = Resources.asByteSource(Resources.getResource( 74 | "swiftproxy.conf")).openStream()) { 75 | properties.load(is); 76 | } catch (IOException e) { 77 | return null; 78 | } 79 | 80 | Response resp = target.path("/auth/v1.0").request() 81 | .header("X-auth-user", properties.getProperty(Constants.PROPERTY_IDENTITY)) 82 | .header("X-auth-key", properties.getProperty(Constants.PROPERTY_CREDENTIAL)) 83 | .get(); 84 | return resp.getHeaderString("x-auth-token"); 85 | } 86 | 87 | public static Response deleteContainer(WebTarget target, String container) throws Exception { 88 | return deleteContainer(target, container, Optional.empty()); 89 | } 90 | 91 | public static Response deleteContainer(WebTarget target, String container, 92 | Optional authToken) throws Exception { 93 | return target.path(ACCOUNT_PATH + "/" + container).request() 94 | .header("x-auth-token", authToken.orElseGet(() -> getAuthToken(target))) 95 | .delete(); 96 | } 97 | 98 | public static String createContainer(WebTarget target, String container) throws Exception { 99 | String authToken = getAuthToken(target); 100 | Response resp = target.path(ACCOUNT_PATH + "/" + container).request() 101 | .header("x-auth-token", authToken) 102 | .post(null); 103 | assertThat(resp.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); 104 | return authToken; 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/test/java/com/bouncestorage/swiftproxy/v1/AccountResourceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import static org.assertj.core.api.Assertions.assertThat; 20 | 21 | import java.util.List; 22 | import java.util.Optional; 23 | import java.util.Random; 24 | 25 | import javax.ws.rs.client.Client; 26 | import javax.ws.rs.client.ClientBuilder; 27 | import javax.ws.rs.client.Entity; 28 | import javax.ws.rs.client.WebTarget; 29 | import javax.ws.rs.core.GenericType; 30 | import javax.ws.rs.core.MediaType; 31 | import javax.ws.rs.core.Response; 32 | 33 | import com.bouncestorage.swiftproxy.SwiftProxy; 34 | import com.bouncestorage.swiftproxy.TestUtils; 35 | import com.google.common.base.Joiner; 36 | 37 | import org.junit.After; 38 | import org.junit.Before; 39 | import org.junit.Test; 40 | 41 | public final class AccountResourceTest { 42 | private static final String CONTAINER = "swiftproxy-test-container-" + new Random().nextInt(Integer.MAX_VALUE); 43 | private SwiftProxy proxy; 44 | private WebTarget target; 45 | 46 | @Before 47 | public void setup() throws Exception { 48 | proxy = TestUtils.setupAndStartProxy(); 49 | Client c = ClientBuilder.newClient(); 50 | target = c.target(proxy.getEndpoint()); 51 | } 52 | 53 | @After 54 | public void tearDown() throws Exception { 55 | if (proxy != null) { 56 | proxy.stop(); 57 | } 58 | } 59 | 60 | @Test 61 | public void testEmptyContainers() throws Exception { 62 | List entries = listContainers(); 63 | assertThat(entries).isEmpty(); 64 | } 65 | 66 | @Test 67 | public void testOneContainer() throws Exception { 68 | String authToken = TestUtils.createContainer(target, CONTAINER); 69 | 70 | List entries = listContainers(Optional.of(authToken)); 71 | assertThat(entries).containsOnly(new AccountResource.ContainerEntry(CONTAINER)); 72 | } 73 | 74 | @Test 75 | public void testHead() throws Exception { 76 | Response response = target.path(TestUtils.ACCOUNT_PATH).queryParam("format", "json").request().head(); 77 | assertThat(response.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); 78 | } 79 | 80 | @Test 81 | public void testBulkDelete() throws Exception { 82 | String authToken = TestUtils.createContainer(target, CONTAINER); 83 | 84 | String[] removeObjects = {"/test/bar", "/test"}; 85 | Response response = target.path(TestUtils.ACCOUNT_PATH) 86 | // swift actually sends ?bulk-delete and this sends ?bulk-delete=, but 87 | // that's the closest we can get 88 | .queryParam("bulk-delete", "") 89 | .request() 90 | .header("X-Auth-Token", authToken) 91 | .post(Entity.entity(Joiner.on("\n").join(removeObjects), MediaType.TEXT_PLAIN)); 92 | assertThat(response.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); 93 | AccountResource.BulkDeleteResult result = response.readEntity(AccountResource.BulkDeleteResult.class); 94 | assertThat(result.numberDeleted).isEqualTo(1); 95 | assertThat(result.numberNotFound).isEqualTo(1); 96 | } 97 | 98 | List listContainers() throws Exception { 99 | return listContainers(Optional.empty()); 100 | } 101 | 102 | List listContainers(Optional authToken) throws Exception { 103 | return target.path(TestUtils.ACCOUNT_PATH) 104 | .queryParam("format", "json") 105 | .request() 106 | .header("x-auth-token", authToken.orElseGet(() -> TestUtils.getAuthToken(target))) 107 | .get(new GenericType>() { 108 | }); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/test/java/com/bouncestorage/swiftproxy/v1/ContainerResourceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import static org.assertj.core.api.Assertions.assertThat; 20 | 21 | import java.util.Optional; 22 | import java.util.Random; 23 | 24 | import javax.ws.rs.client.Client; 25 | import javax.ws.rs.client.ClientBuilder; 26 | import javax.ws.rs.client.Entity; 27 | import javax.ws.rs.client.WebTarget; 28 | import javax.ws.rs.core.MediaType; 29 | import javax.ws.rs.core.Response; 30 | 31 | import com.bouncestorage.swiftproxy.SwiftProxy; 32 | import com.bouncestorage.swiftproxy.TestUtils; 33 | 34 | import org.junit.After; 35 | import org.junit.Before; 36 | import org.junit.Test; 37 | 38 | public final class ContainerResourceTest { 39 | private static final String CONTAINER = "swiftproxy-test-container-" + new Random().nextInt(Integer.MAX_VALUE); 40 | private SwiftProxy proxy; 41 | private WebTarget target; 42 | 43 | @Before 44 | public void setup() throws Exception { 45 | proxy = TestUtils.setupAndStartProxy(); 46 | Client c = ClientBuilder.newClient(); 47 | target = c.target(proxy.getEndpoint()); 48 | } 49 | 50 | @After 51 | public void tearDown() throws Exception { 52 | if (proxy != null) { 53 | proxy.stop(); 54 | } 55 | } 56 | 57 | @Test 58 | public void testDeleteContainer() throws Exception { 59 | Response resp = TestUtils.deleteContainer(target, CONTAINER); 60 | assertThat(resp.getStatus()).isEqualTo(Response.Status.NOT_FOUND.getStatusCode()); 61 | 62 | String authToken = TestUtils.createContainer(target, CONTAINER); 63 | 64 | resp = TestUtils.deleteContainer(target, CONTAINER, Optional.of(authToken)); 65 | assertThat(resp.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); 66 | } 67 | 68 | @Test 69 | public void testHeadContainer() throws Exception { 70 | Response resp = headContainer(CONTAINER, Optional.empty()); 71 | assertThat(resp.getStatus()).isEqualTo(Response.Status.NOT_FOUND.getStatusCode()); 72 | 73 | String authToken = TestUtils.createContainer(target, CONTAINER); 74 | 75 | resp = headContainer(CONTAINER, Optional.of(authToken)); 76 | assertThat(resp.getStatus()).isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); 77 | } 78 | 79 | @Test 80 | public void testGetContainer() throws Exception { 81 | Response resp = getContainer(CONTAINER, Optional.empty()); 82 | assertThat(resp.getStatus()).isEqualTo(Response.Status.NOT_FOUND.getStatusCode()); 83 | 84 | String authToken = TestUtils.createContainer(target, CONTAINER); 85 | 86 | String data = "foo"; 87 | resp = target.path(TestUtils.ACCOUNT_PATH + "/" + CONTAINER + "/blob").request() 88 | .header("x-auth-token", authToken) 89 | .put(Entity.entity(data.getBytes(), MediaType.APPLICATION_OCTET_STREAM)); 90 | assertThat(resp.getStatus()).isEqualTo(Response.Status.CREATED.getStatusCode()); 91 | 92 | resp = getContainer(CONTAINER, Optional.of(authToken)); 93 | assertThat(resp.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); 94 | assertThat(resp.readEntity(String.class)).isEqualTo("blob\n"); 95 | } 96 | 97 | Response headContainer(String container, Optional authToken) { 98 | return target.path(TestUtils.ACCOUNT_PATH + "/" + container).request() 99 | .header("x-auth-token", authToken.orElseGet(() -> TestUtils.getAuthToken(target))) 100 | .head(); 101 | } 102 | 103 | Response getContainer(String container, Optional authToken) { 104 | return target.path(TestUtils.ACCOUNT_PATH + "/" + container).request() 105 | .header("x-auth-token", authToken.orElseGet(() -> TestUtils.getAuthToken(target))) 106 | .get(); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/SwiftProxy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | import static com.google.common.base.Throwables.propagate; 22 | 23 | import java.io.IOException; 24 | import java.net.URI; 25 | import java.net.URISyntaxException; 26 | import java.util.Properties; 27 | 28 | import javax.ws.rs.ext.RuntimeDelegate; 29 | 30 | import org.glassfish.grizzly.http.server.HttpServer; 31 | import org.glassfish.jersey.filter.LoggingFilter; 32 | import org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpServerFactory; 33 | import org.slf4j.Logger; 34 | import org.slf4j.LoggerFactory; 35 | 36 | public final class SwiftProxy { 37 | public static final String PROPERTY_ENDPOINT = "swiftproxy.endpoint"; 38 | private Logger logger = LoggerFactory.getLogger(getClass()); 39 | private HttpServer server; 40 | private URI endpoint; 41 | private final BounceResourceConfig rc; 42 | 43 | public SwiftProxy(Properties properties, BlobStoreLocator locator, URI endpoint) { 44 | this.endpoint = requireNonNull(endpoint); 45 | 46 | rc = new BounceResourceConfig(properties, locator); 47 | if (logger.isDebugEnabled()) { 48 | rc.register(new LoggingFilter(java.util.logging.Logger.getGlobal(), false)); 49 | } 50 | server = GrizzlyHttpServerFactory.createHttpServer(endpoint, rc, false); 51 | server.getListeners().forEach(listener -> { 52 | listener.registerAddOn(new ContentLengthAddOn()); 53 | }); 54 | 55 | // allow HTTP DELETE to have payload for multi-object delete 56 | server.getServerConfiguration().setAllowPayloadForUndefinedHttpMethods(true); 57 | 58 | RuntimeDelegate.setInstance(new RuntimeDelegateImpl(RuntimeDelegate.getInstance())); 59 | } 60 | 61 | public void setBlobStoreLocator(BlobStoreLocator locator) { 62 | rc.setBlobStoreLocator(locator); 63 | } 64 | 65 | public URI getEndpoint() { 66 | return endpoint; 67 | } 68 | 69 | public void start() throws IOException, URISyntaxException { 70 | server.start(); 71 | endpoint = new URI(endpoint.getScheme(), endpoint.getUserInfo(), endpoint.getHost(), 72 | getPort(), endpoint.getPath(), endpoint.getQuery(), endpoint.getFragment()); 73 | rc.setEndPoint(endpoint); 74 | } 75 | 76 | public void stop() { 77 | server.shutdownNow(); 78 | } 79 | 80 | public int getPort() { 81 | return server.getListeners().stream().findAny().map(n -> n.getPort()).orElse(0); 82 | } 83 | 84 | public boolean isStarted() { 85 | return server.isStarted(); 86 | } 87 | 88 | public static final class Builder { 89 | private BlobStoreLocator locator; 90 | private URI endpoint; 91 | private Properties properties; 92 | 93 | Builder() { 94 | } 95 | 96 | public static Builder builder() { 97 | return new Builder(); 98 | } 99 | 100 | public Builder endpoint(URI newEndpoint) { 101 | this.endpoint = requireNonNull(newEndpoint); 102 | return this; 103 | } 104 | 105 | public Builder locator(BlobStoreLocator newLocator) { 106 | this.locator = newLocator; 107 | return this; 108 | } 109 | 110 | public Builder overrides(Properties prop) { 111 | this.properties = requireNonNull(prop); 112 | 113 | String proxyEndpoint = properties.getProperty(SwiftProxy.PROPERTY_ENDPOINT); 114 | if (proxyEndpoint != null) { 115 | try { 116 | endpoint(new URI(proxyEndpoint)); 117 | } catch (URISyntaxException e) { 118 | throw propagate(e); 119 | } 120 | } 121 | 122 | return this; 123 | } 124 | 125 | public SwiftProxy build() { 126 | return new SwiftProxy(properties, locator, endpoint); 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/BlobStoreResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import static com.google.common.base.Throwables.propagate; 20 | 21 | import java.io.ByteArrayOutputStream; 22 | import java.lang.annotation.Annotation; 23 | import java.util.Collection; 24 | 25 | import javax.ws.rs.ClientErrorException; 26 | import javax.ws.rs.core.Application; 27 | import javax.ws.rs.core.Context; 28 | import javax.ws.rs.core.MediaType; 29 | import javax.ws.rs.core.MultivaluedHashMap; 30 | import javax.ws.rs.core.Response; 31 | import javax.ws.rs.ext.MessageBodyWriter; 32 | 33 | import org.glassfish.jersey.message.MessageBodyWorkers; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | public abstract class BlobStoreResource { 38 | private static final String UNAUTHORIZED_BODY = 39 | "

Unauthorized

This server could not verify that you are authorized " + 40 | "to access the document you requested.

"; 41 | 42 | protected Logger logger = LoggerFactory.getLogger(getClass()); 43 | @Context 44 | protected Application application; 45 | @Context 46 | private MessageBodyWorkers workers; 47 | 48 | protected final BounceResourceConfig.AuthenticatedBlobStore getBlobStore(String authToken) { 49 | if (authToken == null) { 50 | throw new ClientErrorException(UNAUTHORIZED_BODY, Response.Status.UNAUTHORIZED); 51 | } 52 | BounceResourceConfig.AuthenticatedBlobStore blobStore = 53 | ((BounceResourceConfig) application).getBlobStore(authToken); 54 | if (blobStore == null) { 55 | throw new ClientErrorException(UNAUTHORIZED_BODY, Response.Status.UNAUTHORIZED); 56 | } 57 | 58 | return blobStore; 59 | } 60 | 61 | protected static Response notAuthorized() { 62 | return Response.status(Response.Status.UNAUTHORIZED).entity(UNAUTHORIZED_BODY).build(); 63 | } 64 | 65 | protected static Response notFound() { 66 | return Response.status(Response.Status.NOT_FOUND) 67 | .entity("

Not Found

The resource could not be found.

") 68 | .build(); 69 | } 70 | 71 | protected static Response badRequest() { 72 | return Response.status(Response.Status.BAD_REQUEST).build(); 73 | } 74 | 75 | private void debugWrite(Object root, MediaType format) { 76 | 77 | MessageBodyWriter messageBodyWriter = 78 | workers.getMessageBodyWriter(root.getClass(), root.getClass(), 79 | new Annotation[]{}, format); 80 | ByteArrayOutputStream baos = new ByteArrayOutputStream(); 81 | try { 82 | // use the MBW to serialize myBean into baos 83 | messageBodyWriter.writeTo(root, 84 | root.getClass(), root.getClass(), new Annotation[]{}, 85 | format, new MultivaluedHashMap(), 86 | baos); 87 | } catch (Throwable e) { 88 | logger.error(String.format("could not serialize %s to format %s", root, format), e); 89 | throw propagate(e); 90 | } 91 | 92 | logger.info("{}", baos); 93 | } 94 | 95 | protected final Response.ResponseBuilder output(Object root, Object value, MediaType 96 | format) { 97 | if (value instanceof Collection) { 98 | Collection entries = (Collection) value; 99 | if (format == MediaType.TEXT_PLAIN_TYPE && entries.isEmpty()) { 100 | return Response.noContent(); 101 | } 102 | if (format == MediaType.APPLICATION_XML_TYPE) { 103 | debugWrite(root, format); 104 | return Response.ok(root, format); 105 | } 106 | if (format == MediaType.APPLICATION_JSON_TYPE) { 107 | return Response.ok(entries, format); 108 | } 109 | } 110 | 111 | debugWrite(value, format); 112 | return Response.ok(value, format); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/RuntimeDelegateImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | import java.util.Map; 22 | import java.util.Objects; 23 | 24 | import javax.ws.rs.core.Application; 25 | import javax.ws.rs.core.Link; 26 | import javax.ws.rs.core.MediaType; 27 | import javax.ws.rs.core.Response; 28 | import javax.ws.rs.core.UriBuilder; 29 | import javax.ws.rs.core.Variant; 30 | import javax.ws.rs.ext.RuntimeDelegate; 31 | 32 | import com.google.common.base.Joiner; 33 | 34 | public final class RuntimeDelegateImpl extends RuntimeDelegate { 35 | private static final MediaTypeProvider MEDIA_TYPE_PROVIDER = new MediaTypeProvider(); 36 | private final RuntimeDelegate delegate; 37 | 38 | RuntimeDelegateImpl(RuntimeDelegate delegate) { 39 | this.delegate = requireNonNull(delegate); 40 | } 41 | 42 | @Override 43 | public UriBuilder createUriBuilder() { 44 | return delegate.createUriBuilder(); 45 | } 46 | 47 | @Override 48 | public Response.ResponseBuilder createResponseBuilder() { 49 | return delegate.createResponseBuilder(); 50 | } 51 | 52 | @Override 53 | public Variant.VariantListBuilder createVariantListBuilder() { 54 | return delegate.createVariantListBuilder(); 55 | } 56 | 57 | @Override 58 | public T createEndpoint(Application application, Class endpointType) throws IllegalArgumentException, UnsupportedOperationException { 59 | return delegate.createEndpoint(application, endpointType); 60 | } 61 | 62 | @Override 63 | public HeaderDelegate createHeaderDelegate(Class type) { 64 | if (type.equals(MediaType.class)) { 65 | return (HeaderDelegate) MEDIA_TYPE_PROVIDER; 66 | } else { 67 | return delegate.createHeaderDelegate(type); 68 | } 69 | } 70 | 71 | @Override 72 | public Link.Builder createLinkBuilder() { 73 | return delegate.createLinkBuilder(); 74 | } 75 | 76 | private static final class InvalidMediaType extends MediaType { 77 | private final String type; 78 | 79 | InvalidMediaType(String type) { 80 | this.type = requireNonNull(type); 81 | } 82 | 83 | @Override 84 | public boolean equals(Object object) { 85 | if (object == this) { 86 | return true; 87 | } 88 | if (!(object instanceof InvalidMediaType)) { 89 | return false; 90 | } 91 | InvalidMediaType other = (InvalidMediaType) object; 92 | return super.equals(other) && 93 | Objects.equals(type, other.type); 94 | } 95 | 96 | @Override 97 | public int hashCode() { 98 | return Objects.hash(super.hashCode(), type); 99 | } 100 | 101 | @Override 102 | public String toString() { 103 | StringBuilder res = new StringBuilder(type); 104 | Map params = getParameters(); 105 | if (params != null && !params.isEmpty()) { 106 | res.append("; "); 107 | res.append(Joiner.on("; ").withKeyValueSeparator("=").join(params)); 108 | } 109 | return res.toString(); 110 | } 111 | } 112 | 113 | private static final class MediaTypeProvider extends org.glassfish.jersey.message.internal.MediaTypeProvider { 114 | @Override 115 | public MediaType fromString(String header) { 116 | try { 117 | return super.fromString(header); 118 | } catch (IllegalArgumentException e) { 119 | return new InvalidMediaType(header); 120 | } 121 | } 122 | 123 | @Override 124 | public String toString(MediaType type) { 125 | StringBuilder res = new StringBuilder(); 126 | res.append(type.getType()).append('/').append(type.getSubtype()); 127 | Map params = type.getParameters(); 128 | if (params != null && !params.isEmpty()) { 129 | res.append("; "); 130 | res.append(Joiner.on("; ").withKeyValueSeparator("=").join(params)); 131 | } 132 | return res.toString(); 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import java.util.Iterator; 20 | import java.util.Objects; 21 | import java.util.Properties; 22 | 23 | import com.google.common.collect.AbstractIterator; 24 | import com.google.common.collect.ImmutableList; 25 | import com.google.inject.Module; 26 | 27 | import org.jclouds.Constants; 28 | import org.jclouds.ContextBuilder; 29 | import org.jclouds.blobstore.BlobStore; 30 | import org.jclouds.blobstore.BlobStoreContext; 31 | import org.jclouds.blobstore.domain.PageSet; 32 | import org.jclouds.blobstore.domain.StorageMetadata; 33 | import org.jclouds.blobstore.domain.StorageType; 34 | import org.jclouds.blobstore.options.ListContainerOptions; 35 | import org.jclouds.javax.annotation.Nullable; 36 | import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; 37 | 38 | public final class Utils { 39 | private Utils() { 40 | throw new AssertionError("intentionally unimplemented"); 41 | } 42 | 43 | public static Iterable crawlBlobStore( 44 | BlobStore blobStore, String containerName) { 45 | return crawlBlobStore(blobStore, containerName, 46 | new ListContainerOptions()); 47 | } 48 | 49 | public static Iterable crawlBlobStore( 50 | BlobStore blobStore, String containerName, 51 | ListContainerOptions options) { 52 | return new CrawlBlobStoreIterable(blobStore, containerName, options); 53 | } 54 | 55 | public static BlobStore storeFromProperties(Properties properties) { 56 | String provider = properties.getProperty(Constants.PROPERTY_PROVIDER); 57 | ContextBuilder builder = ContextBuilder 58 | .newBuilder(provider) 59 | .modules(ImmutableList.of(new SLF4JLoggingModule())) 60 | .overrides(properties); 61 | BlobStoreContext context = builder.build(BlobStoreContext.class); 62 | return context.getBlobStore(); 63 | } 64 | 65 | private static class CrawlBlobStoreIterable 66 | implements Iterable { 67 | private final BlobStore blobStore; 68 | private final String containerName; 69 | private final ListContainerOptions options; 70 | 71 | CrawlBlobStoreIterable(BlobStore blobStore, String containerName, 72 | ListContainerOptions options) { 73 | this.blobStore = Objects.requireNonNull(blobStore); 74 | this.containerName = Objects.requireNonNull(containerName); 75 | this.options = Objects.requireNonNull(options).clone(); 76 | } 77 | 78 | @Override 79 | public Iterator iterator() { 80 | return new CrawlBlobStoreIterator(blobStore, containerName, 81 | options); 82 | } 83 | } 84 | 85 | private static class CrawlBlobStoreIterator 86 | extends AbstractIterator { 87 | private final BlobStore blobStore; 88 | private final String containerName; 89 | private final ListContainerOptions options; 90 | private Iterator iterator; 91 | private String marker; 92 | 93 | CrawlBlobStoreIterator(BlobStore blobStore, String containerName, 94 | ListContainerOptions options) { 95 | this.blobStore = Objects.requireNonNull(blobStore); 96 | this.containerName = Objects.requireNonNull(containerName); 97 | this.options = Objects.requireNonNull(options); 98 | if (options.getDelimiter() == null && options.getDir() == null) { 99 | this.options.recursive(); 100 | } 101 | advance(); 102 | } 103 | 104 | private void advance() { 105 | if (marker != null) { 106 | options.afterMarker(marker); 107 | } 108 | PageSet set = blobStore.list( 109 | containerName, options); 110 | marker = set.getNextMarker(); 111 | iterator = set.iterator(); 112 | } 113 | 114 | @Override 115 | protected StorageMetadata computeNext() { 116 | while (true) { 117 | if (!iterator.hasNext()) { 118 | if (marker == null) { 119 | return endOfData(); 120 | } 121 | advance(); 122 | continue; 123 | } 124 | 125 | StorageMetadata metadata = iterator.next(); 126 | // filter out folders with atmos and filesystem providers 127 | // accept metadata == null for Google Cloud Storage folders 128 | if (metadata == null || metadata.getType() == StorageType.RELATIVE_PATH) { 129 | continue; 130 | } 131 | return metadata; 132 | } 133 | } 134 | } 135 | 136 | public static String trimETag(@Nullable String eTag) { 137 | if (eTag == null) { 138 | return null; 139 | } 140 | int begin = 0; 141 | int end = eTag.length(); 142 | if (eTag.startsWith("\"")) { 143 | begin = 1; 144 | } 145 | if (eTag.endsWith("\"")) { 146 | end = eTag.length() - 1; 147 | } 148 | return eTag.substring(begin, end); 149 | } 150 | 151 | public static boolean eTagsEqual(@Nullable String eTag1, @Nullable String eTag2) { 152 | return Objects.equals(trimETag(eTag1), trimETag(eTag2)); 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/BounceResourceConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy; 18 | 19 | import static com.google.common.base.Throwables.propagate; 20 | 21 | import java.net.URI; 22 | import java.util.Map; 23 | import java.util.Properties; 24 | import java.util.concurrent.TimeUnit; 25 | 26 | import javax.ws.rs.core.MediaType; 27 | 28 | import com.bouncestorage.swiftproxy.v1.InfoResource; 29 | import com.google.common.cache.Cache; 30 | import com.google.common.cache.CacheBuilder; 31 | import com.google.common.collect.ImmutableMap; 32 | import com.google.common.collect.ImmutableSet; 33 | import com.google.inject.Module; 34 | 35 | import org.apache.commons.lang3.RandomStringUtils; 36 | import org.glassfish.jersey.server.ResourceConfig; 37 | import org.jclouds.Constants; 38 | import org.jclouds.ContextBuilder; 39 | import org.jclouds.blobstore.BlobStore; 40 | import org.jclouds.blobstore.BlobStoreContext; 41 | import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; 42 | import org.slf4j.Logger; 43 | import org.slf4j.LoggerFactory; 44 | 45 | public final class BounceResourceConfig extends ResourceConfig { 46 | private static final Map swiftFormatToMediaType = ImmutableMap.of( 47 | "json", MediaType.APPLICATION_JSON_TYPE, 48 | "application/json", MediaType.APPLICATION_JSON_TYPE, 49 | "xml", MediaType.APPLICATION_XML_TYPE, 50 | "plain", MediaType.TEXT_PLAIN_TYPE 51 | ); 52 | 53 | private final Logger logger = LoggerFactory.getLogger(getClass()); 54 | private final Properties properties; 55 | private URI endPoint; 56 | private BlobStoreLocator locator; 57 | private Cache tokensToIdentities = CacheBuilder.newBuilder() 58 | .expireAfterWrite(InfoResource.CONFIG.tempauth.token_life, TimeUnit.SECONDS) 59 | .build(); 60 | private Cache identitiesToBlobStore = CacheBuilder.newBuilder() 61 | .expireAfterWrite(InfoResource.CONFIG.tempauth.token_life, TimeUnit.SECONDS) 62 | .build(); 63 | 64 | public interface AuthenticatedBlobStore { 65 | BlobStore get(String container, String key); 66 | default BlobStore get(String container) { 67 | return get(container, null); 68 | } 69 | default BlobStore get() { 70 | return get(null, null); 71 | } 72 | } 73 | 74 | BounceResourceConfig(Properties properties, BlobStoreLocator locator) { 75 | if (properties == null && locator == null) { 76 | throw new NullPointerException("One of properties or locator must be set"); 77 | } 78 | this.properties = properties; 79 | this.locator = locator; 80 | packages(getClass().getPackage().getName()); 81 | } 82 | 83 | public String authenticate(String identity, String credential) { 84 | AuthenticatedBlobStore blobStore = tryAuthenticate(identity, credential); 85 | if (blobStore != null) { 86 | String token = "AUTH_tk" + RandomStringUtils.randomAlphanumeric(32); 87 | tokensToIdentities.put(token, identity); 88 | identitiesToBlobStore.put(identity, blobStore); 89 | return token; 90 | } 91 | 92 | return null; 93 | } 94 | 95 | private AuthenticatedBlobStore tryAuthenticate(String identity, String credential) { 96 | if (locator != null) { 97 | Map.Entry entry = locator.locateBlobStore(identity, null, null); 98 | if (entry != null && entry.getKey().equals(credential)) { 99 | logger.debug("blob store for {} found", identity); 100 | return (container, key) -> locator.locateBlobStore(identity, container, key).getValue(); 101 | } else { 102 | logger.debug("blob store for {} not found", identity); 103 | } 104 | } else { 105 | logger.debug("fallback to authenticate with configured provider"); 106 | String provider = properties.getProperty(Constants.PROPERTY_PROVIDER); 107 | if (provider.equals("transient")) { 108 | /* there's no authentication for transient blobstores, so simply re-use 109 | the previous blobstore so that multiple authentication will reuse the 110 | same namespace */ 111 | AuthenticatedBlobStore blobStore = identitiesToBlobStore.getIfPresent(identity); 112 | if (blobStore != null) { 113 | return blobStore; 114 | } 115 | } 116 | 117 | try { 118 | BlobStoreContext context = ContextBuilder 119 | .newBuilder(provider) 120 | .overrides(properties) 121 | .credentials(identity, credential) 122 | .modules(ImmutableSet.of(new SLF4JLoggingModule())) 123 | .build(BlobStoreContext.class); 124 | return (container, key) -> context.getBlobStore(); 125 | } catch (Throwable e) { 126 | throw propagate(e); 127 | } 128 | } 129 | 130 | return null; 131 | } 132 | 133 | public AuthenticatedBlobStore getBlobStore(String authToken) { 134 | String identity = tokensToIdentities.getIfPresent(authToken); 135 | return identitiesToBlobStore.getIfPresent(identity); 136 | } 137 | 138 | public static MediaType getMediaType(String format) { 139 | return swiftFormatToMediaType.get(format); 140 | } 141 | 142 | public void setEndPoint(URI endPoint) { 143 | this.endPoint = endPoint; 144 | } 145 | 146 | public URI getEndPoint() { 147 | return endPoint; 148 | } 149 | 150 | public boolean isLocatorSet() { 151 | return locator != null; 152 | } 153 | 154 | public void setBlobStoreLocator(BlobStoreLocator newLocator) { 155 | locator = newLocator; 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/main/resources/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 33 | 34 | 35 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 83 | 84 | 85 | 86 | 87 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 116 | 117 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/v2/Identity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v2; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | import static com.google.common.base.Throwables.propagate; 22 | 23 | import java.net.MalformedURLException; 24 | import java.net.URL; 25 | import java.time.Instant; 26 | import java.util.ArrayList; 27 | import java.util.List; 28 | import java.util.Optional; 29 | 30 | import javax.ws.rs.HeaderParam; 31 | import javax.ws.rs.POST; 32 | import javax.ws.rs.Path; 33 | import javax.ws.rs.Produces; 34 | import javax.ws.rs.core.Context; 35 | import javax.ws.rs.core.MediaType; 36 | import javax.ws.rs.core.Response; 37 | 38 | import com.bouncestorage.swiftproxy.BlobStoreResource; 39 | import com.bouncestorage.swiftproxy.BounceResourceConfig; 40 | import com.bouncestorage.swiftproxy.v1.InfoResource; 41 | import com.fasterxml.jackson.annotation.JsonProperty; 42 | import com.fasterxml.jackson.databind.annotation.JsonSerialize; 43 | import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; 44 | import com.google.common.base.Strings; 45 | 46 | import org.glassfish.grizzly.http.server.Request; 47 | 48 | @Path("/v2.0") 49 | public final class Identity extends BlobStoreResource { 50 | @Path("tokens") 51 | @POST 52 | @Produces(MediaType.APPLICATION_JSON) 53 | public Response auth(AuthV2Request payload, 54 | @HeaderParam("Host") Optional host, 55 | @Context Request request) { 56 | String tenant = payload.auth.tenantName; 57 | String identity = payload.auth.passwordCredentials.username; 58 | if (Strings.isNullOrEmpty(tenant)) { 59 | tenant = identity; 60 | } 61 | String credential = payload.auth.passwordCredentials.password; 62 | String authToken = null; 63 | try { 64 | authToken = ((BounceResourceConfig) application).authenticate(tenant + ":" + identity, credential); 65 | } catch (Throwable e) { 66 | e.printStackTrace(); 67 | } 68 | if (authToken == null) { 69 | return notAuthorized(); 70 | } 71 | 72 | String storageURL = host.orElseGet(() -> request.getLocalAddr() + ":" + request.getLocalPort()); 73 | String scheme = request.getScheme(); 74 | tenant = "AUTH_" + tenant; 75 | storageURL = scheme + "://" + storageURL + "/v1/" + tenant; 76 | 77 | AuthV2Response resp = new AuthV2Response(); 78 | resp.access.token.id = authToken; 79 | resp.access.token.expires = Instant.now().plusSeconds(InfoResource.CONFIG.tempauth.token_life); 80 | resp.access.token.tenant = new IDAndName(tenant, tenant); 81 | resp.access.user = new AuthV2Response.Access.User(identity, identity); 82 | resp.access.user.roles.add(new IDAndName(identity, identity)); 83 | if (!identity.equals(tenant)) { 84 | resp.access.user.roles.add(new IDAndName(tenant, tenant)); 85 | } 86 | 87 | AuthV2Response.Access.ServiceCatalog.Endpoint endpoint = new AuthV2Response.Access.ServiceCatalog.Endpoint(); 88 | try { 89 | endpoint.publicURL = new URL(storageURL); 90 | } catch (MalformedURLException e) { 91 | e.printStackTrace(); 92 | throw propagate(e); 93 | } 94 | endpoint.tenantId = tenant; 95 | resp.access.serviceCatalog[0].endpoints.add(endpoint); 96 | org.jclouds.Context c = getBlobStore(authToken).get().getContext().unwrap(); 97 | resp.access.serviceCatalog[0].name += String.format(" (%s %s)", 98 | c.getId(), c.getProviderMetadata().getEndpoint()); 99 | return Response.ok(resp).build(); 100 | } 101 | 102 | static class AuthV2Request { 103 | @JsonProperty 104 | Auth auth = new Auth(); 105 | static class Auth { 106 | @JsonProperty 107 | PasswordCredentials passwordCredentials = new PasswordCredentials(); 108 | @JsonProperty 109 | String tenantName; 110 | static class PasswordCredentials { 111 | @JsonProperty 112 | String username; 113 | @JsonProperty 114 | String password; 115 | } 116 | } 117 | } 118 | 119 | static class IDAndName { 120 | @JsonProperty 121 | String id; 122 | @JsonProperty 123 | String name; 124 | 125 | IDAndName(String id, String name) { 126 | this.id = requireNonNull(id); 127 | this.name = requireNonNull(name); 128 | } 129 | } 130 | 131 | static class AuthV2Response { 132 | @JsonProperty 133 | Access access = new Access(); 134 | static class Access { 135 | @JsonProperty 136 | Token token = new Token(); 137 | @JsonProperty 138 | User user; 139 | @JsonProperty 140 | ServiceCatalog[] serviceCatalog = {new ServiceCatalog()}; 141 | 142 | static class Token { 143 | @JsonProperty 144 | String id; 145 | @JsonSerialize(using = ToStringSerializer.class) 146 | Instant expires; 147 | @JsonProperty 148 | IDAndName tenant; 149 | } 150 | 151 | static class User extends IDAndName { 152 | @JsonProperty 153 | List roles = new ArrayList<>(); 154 | 155 | User(String id, String name) { 156 | super(id, name); 157 | } 158 | } 159 | 160 | static class ServiceCatalog { 161 | @JsonProperty 162 | List endpoints = new ArrayList<>(); 163 | @JsonProperty 164 | String name = "Swift Object Storage"; 165 | @JsonProperty 166 | String type = "object-store"; 167 | 168 | static class Endpoint { 169 | @JsonProperty 170 | String region = "default"; 171 | @JsonProperty 172 | URL publicURL; 173 | @JsonProperty 174 | String versionId = "1"; 175 | @JsonProperty 176 | String tenantId; 177 | } 178 | } 179 | } 180 | } 181 | 182 | } 183 | -------------------------------------------------------------------------------- /src/test/resources/run-swift-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o xtrace 4 | set -o errexit 5 | set -o nounset 6 | 7 | source $(dirname $0)/run-swiftproxy.sh 8 | 9 | : ${SKIP_TESTS:=""} 10 | 11 | SWIFT_DOCKER_TESTS=$(echo \ 12 | test.functional.tests:TestContainerPaths \ 13 | test.functional.tests:TestFile.testCopyAccount \ 14 | test.functional.tests:TestFile.testMetadataNumberLimit \ 15 | test.functional.tests:TestFile.testMetadataLengthLimits \ 16 | test.functional.tests:TestFileComparison \ 17 | test.functional.tests:TestSlo.test_slo_copy_account \ 18 | test.functional.tests:TestSlo.test_slo_copy_the_manifest_account \ 19 | ) 20 | 21 | 22 | SWIFT_TESTS=$(echo \ 23 | test.functional.tests:TestAccountEnv \ 24 | test.functional.tests:TestAccountDev \ 25 | test.functional.tests:TestAccountDevUTF8 \ 26 | test.functional.tests:TestAccountNoContainersEnv \ 27 | test.functional.tests:TestAccountNoContainers \ 28 | test.functional.tests:TestAccountNoContainersUTF8 \ 29 | test.functional.tests:TestContainerEnv \ 30 | test.functional.tests:TestContainerDev \ 31 | test.functional.tests:TestContainerDevUTF8 \ 32 | test.functional.tests:TestContainer.testContainerNameLimit \ 33 | test.functional.tests:TestContainer.testFileThenContainerDelete \ 34 | test.functional.tests:TestContainer.testPrefixAndLimit \ 35 | test.functional.tests:TestContainer.testCreate \ 36 | test.functional.tests:TestContainer.testContainerFileListOnContainerThatDoesNotExist \ 37 | test.functional.tests:TestContainer.testCreateOnExisting \ 38 | test.functional.tests:TestContainer.testSlashInName \ 39 | test.functional.tests:TestContainer.testDelete \ 40 | test.functional.tests:TestContainer.testDeleteOnContainerThatDoesNotExist \ 41 | test.functional.tests:TestContainer.testDeleteOnContainerWithFiles \ 42 | test.functional.tests:TestContainer.testFileCreateInContainerThatDoesNotExist \ 43 | test.functional.tests:TestContainer.testLastFileMarker \ 44 | test.functional.tests:TestContainer.testContainerFileList \ 45 | test.functional.tests:TestContainer.testMarkerLimitFileList \ 46 | test.functional.tests:TestContainer.testFileOrder \ 47 | test.functional.tests:TestContainer.testContainerInfoOnContainerThatDoesNotExist \ 48 | test.functional.tests:TestContainer.testContainerFileListWithLimit \ 49 | test.functional.tests:TestContainer.testTooLongName \ 50 | test.functional.tests:TestContainer.testContainerExistenceCachingProblem \ 51 | test.functional.tests:TestFile.testCopy \ 52 | test.functional.tests:TestFile.testCopy404s \ 53 | test.functional.tests:TestFile.testCopyNoDestinationHeader \ 54 | test.functional.tests:TestFile.testCopyDestinationSlashProblems \ 55 | test.functional.tests:TestFile.testCopyFromHeader \ 56 | test.functional.tests:TestFile.testCopyFromHeader404s \ 57 | test.functional.tests:TestFile.testNameLimit \ 58 | test.functional.tests:TestFile.testQuestionMarkInName \ 59 | test.functional.tests:TestFile.testDeleteThen404s \ 60 | test.functional.tests:TestFile.testBlankMetadataName \ 61 | test.functional.tests:TestFile.testRangedGets \ 62 | test.functional.tests:TestFile.testRangedGetsWithLWSinHeader \ 63 | test.functional.tests:TestFile.testNoContentLengthForPut \ 64 | test.functional.tests:TestFile.testDelete \ 65 | test.functional.tests:TestFile.testBadHeaders \ 66 | test.functional.tests:TestFile.testEtagWayoff \ 67 | test.functional.tests:TestFile.testFileCreate \ 68 | test.functional.tests:TestFile.testDeleteOfFileThatDoesNotExist \ 69 | test.functional.tests:TestFile.testHeadOnFileThatDoesNotExist \ 70 | test.functional.tests:TestFile.testMetadataOnPost \ 71 | test.functional.tests:TestFile.testGetOnFileThatDoesNotExist \ 72 | test.functional.tests:TestFile.testPostOnFileThatDoesNotExist \ 73 | test.functional.tests:TestFile.testMetadataOnPut \ 74 | test.functional.tests:TestFile.testStackedOverwrite \ 75 | test.functional.tests:TestFile.testTooLongName \ 76 | test.functional.tests:TestFile.testZeroByteFile \ 77 | test.functional.tests:TestFile.testEtagResponse \ 78 | test.functional.tests:TestFile.testChunkedPut \ 79 | test.functional.tests:TestDlo.test_get_manifest \ 80 | test.functional.tests:TestDlo.test_get_manifest_document_itself \ 81 | test.functional.tests:TestDlo.test_get_range \ 82 | test.functional.tests:TestDlo.test_get_range_out_of_range \ 83 | test.functional.tests:TestDlo.test_copy \ 84 | test.functional.tests:TestDlo.test_copy_account \ 85 | test.functional.tests:TestDlo.test_copy_manifest \ 86 | test.functional.tests:TestDlo.test_dlo_if_match_get \ 87 | test.functional.tests:TestDlo.test_dlo_if_none_match_get \ 88 | test.functional.tests:TestSlo.test_slo_get_simple_manifest \ 89 | test.functional.tests:TestSlo.test_slo_get_nested_manifest \ 90 | test.functional.tests:TestSlo.test_slo_ranged_get \ 91 | test.functional.tests:TestSlo.test_slo_ranged_submanifest \ 92 | test.functional.tests:TestSlo.test_slo_etag_is_hash_of_etags \ 93 | test.functional.tests:TestSlo.test_slo_etag_is_hash_of_etags_submanifests \ 94 | test.functional.tests:TestSlo.test_slo_etag_mismatch \ 95 | test.functional.tests:TestSlo.test_slo_size_mismatch \ 96 | test.functional.tests:TestSlo.test_slo_copy \ 97 | test.functional.tests:TestSlo.test_slo_copy_the_manifest \ 98 | test.functional.tests:TestSlo.test_slo_get_the_manifest \ 99 | test.functional.tests:TestSlo.test_slo_head_the_manifest \ 100 | test.functional.tests:TestSlo.test_slo_if_match_get \ 101 | test.functional.tests:TestSlo.test_slo_if_none_match_get \ 102 | ) 103 | 104 | if [ "$TRAVIS" != "true" ]; then 105 | export NOSE_NOCAPTURE=1 106 | export NOSE_NOLOGCAPTURE=1 107 | SWIFT_TESTS="$SWIFT_TESTS $SWIFT_DOCKER_TESTS" 108 | fi 109 | 110 | mkdir -p virtualenv 111 | if [ ! -e ./virtualenv/bin/pip ]; then 112 | virtualenv --no-site-packages --distribute virtualenv 113 | fi 114 | 115 | # avoid pip bugs 116 | ./virtualenv/bin/pip install --upgrade pip 117 | 118 | # work-around change in pip 1.5 119 | ./virtualenv/bin/pip install setuptools --no-use-wheel --upgrade 120 | 121 | ./virtualenv/bin/pip install -r swift-tests/requirements.txt 122 | ./virtualenv/bin/pip install -r swift-tests/test-requirements.txt 123 | 124 | wait_for_swiftproxy 125 | 126 | 127 | mkdir -p ./virtualenv/etc/swift 128 | cat > ./virtualenv/etc/swift/test.conf < 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | import static com.google.common.base.Throwables.propagate; 22 | 23 | import java.io.BufferedReader; 24 | import java.io.IOException; 25 | import java.io.InputStreamReader; 26 | import java.net.URLDecoder; 27 | import java.nio.charset.StandardCharsets; 28 | import java.util.ArrayList; 29 | import java.util.List; 30 | import java.util.Objects; 31 | import java.util.Optional; 32 | import java.util.stream.Collectors; 33 | 34 | import javax.inject.Singleton; 35 | import javax.validation.constraints.NotNull; 36 | import javax.ws.rs.Consumes; 37 | import javax.ws.rs.DELETE; 38 | import javax.ws.rs.DefaultValue; 39 | import javax.ws.rs.GET; 40 | import javax.ws.rs.HEAD; 41 | import javax.ws.rs.HeaderParam; 42 | import javax.ws.rs.POST; 43 | import javax.ws.rs.Path; 44 | import javax.ws.rs.PathParam; 45 | import javax.ws.rs.Produces; 46 | import javax.ws.rs.QueryParam; 47 | import javax.ws.rs.WebApplicationException; 48 | import javax.ws.rs.core.Context; 49 | import javax.ws.rs.core.MediaType; 50 | import javax.ws.rs.core.Response; 51 | import javax.xml.bind.annotation.XmlAttribute; 52 | import javax.xml.bind.annotation.XmlElement; 53 | import javax.xml.bind.annotation.XmlRootElement; 54 | import javax.xml.bind.annotation.XmlType; 55 | 56 | import com.bouncestorage.swiftproxy.BlobStoreResource; 57 | import com.bouncestorage.swiftproxy.BounceResourceConfig; 58 | import com.fasterxml.jackson.annotation.JsonCreator; 59 | import com.fasterxml.jackson.annotation.JsonProperty; 60 | import com.fasterxml.jackson.core.JsonProcessingException; 61 | import com.fasterxml.jackson.databind.ObjectMapper; 62 | import com.google.common.net.PercentEscaper; 63 | 64 | import org.glassfish.grizzly.http.server.Request; 65 | import org.jclouds.blobstore.BlobStore; 66 | import org.jclouds.blobstore.ContainerNotFoundException; 67 | import org.jclouds.blobstore.domain.StorageMetadata; 68 | 69 | @Singleton 70 | @Path("/v1/{account}") 71 | public final class AccountResource extends BlobStoreResource { 72 | private static final PercentEscaper BULK_DELETE_ESCAPER = new PercentEscaper("-_.~:/", false); 73 | 74 | @GET 75 | public Response getAccount(@NotNull @PathParam("account") String account, 76 | @QueryParam("limit") Optional limit, 77 | @QueryParam("marker") Optional marker, 78 | @QueryParam("end_marker") Optional endMarker, 79 | @QueryParam("format") Optional format, 80 | @QueryParam("prefix") Optional prefix, 81 | @QueryParam("delimiter") Optional delimiter, 82 | @HeaderParam("X-Auth-Token") String authToken, 83 | @HeaderParam("X-Newest") @DefaultValue("false") boolean newest, 84 | @HeaderParam("Accept") Optional accept) { 85 | delimiter.ifPresent(x -> logger.info("delimiter not supported yet")); 86 | 87 | BlobStore blobStore = getBlobStore(authToken).get(); 88 | ArrayList entries = blobStore.list() 89 | .stream() 90 | .map(StorageMetadata::getName) 91 | .filter(name -> marker.map(m -> name.compareTo(m) > 0).orElse(true)) 92 | .filter(name -> endMarker.map(m -> name.compareTo(m) < 0).orElse(true)) 93 | .filter(name -> prefix.map(name::startsWith).orElse(true)) 94 | .map(ContainerEntry::new) 95 | .collect(Collectors.toCollection(ArrayList::new)); 96 | 97 | MediaType formatType; 98 | if (format.isPresent()) { 99 | formatType = BounceResourceConfig.getMediaType(format.get()); 100 | } else if (accept.isPresent()) { 101 | formatType = MediaType.valueOf(accept.get()); 102 | } else { 103 | formatType = MediaType.TEXT_PLAIN_TYPE; 104 | } 105 | 106 | if (blobStore.getContext().unwrap().getId().equals("transient")) { 107 | entries.sort((a, b) -> a.getName().compareTo(b.getName())); 108 | } 109 | 110 | long count = entries.size(); 111 | limit.ifPresent(max -> { 112 | if (entries.size() > max) { 113 | entries.subList(max, entries.size()).clear(); 114 | } 115 | }); 116 | 117 | Account root = new Account(); 118 | root.name = account; 119 | root.container = entries; 120 | return output(root, entries, formatType) 121 | .header("X-Account-Container-Count", count) 122 | .header("X-Account-Object-Count", -1) 123 | .header("X-Account-Bytes-Used", -1) 124 | .header("X-Timestamp", -1) 125 | .header("X-Trans-Id", -1) 126 | .header("Accept-Ranges", "bytes") 127 | .build(); 128 | } 129 | 130 | @HEAD 131 | public Response headAccount(@NotNull @PathParam("account") String account, 132 | @HeaderParam("X-Auth-Token") String authToken, 133 | @HeaderParam("X-Newest") boolean newest) { 134 | return Response.noContent() 135 | .header("X-Account-Container-Count", -1) 136 | .header("X-Account-Object-Count", -1) 137 | .header("X-Account-Bytes-Used", -1) 138 | .header("X-Timestamp", -1) 139 | .header("X-Trans-Id", -1) 140 | .header("Accept-Ranges", "bytes") 141 | .build(); 142 | } 143 | 144 | @DELETE 145 | @Consumes(MediaType.TEXT_PLAIN) 146 | @Produces(MediaType.APPLICATION_JSON) 147 | public BulkDeleteResult bulkDeleteDelete(@NotNull @PathParam("account") String account, 148 | @QueryParam("bulk-delete") String bulkDelete, 149 | @HeaderParam("X-Auth-Token") String authToken, 150 | @Context Request request) throws JsonProcessingException { 151 | return bulkDelete(account, bulkDelete, authToken, request); 152 | } 153 | 154 | @POST 155 | @Consumes(MediaType.TEXT_PLAIN) 156 | @Produces(MediaType.APPLICATION_JSON) 157 | public BulkDeleteResult bulkDelete(@NotNull @PathParam("account") String account, 158 | @QueryParam("bulk-delete") String bulkDelete, 159 | @HeaderParam("X-Auth-Token") String authToken, 160 | @Context Request request) throws JsonProcessingException { 161 | if (bulkDelete == null) { 162 | // TODO: Currently this will match the account delete request as well, which we do not implement 163 | throw new WebApplicationException(Response.Status.NOT_IMPLEMENTED); 164 | } 165 | 166 | BlobStore blobStore = getBlobStore(authToken).get(); 167 | if (blobStore == null) { 168 | throw new WebApplicationException(Response.Status.BAD_REQUEST); 169 | } 170 | 171 | String line; 172 | ArrayList objects = new ArrayList<>(); 173 | 174 | boolean isTransient = blobStore.getContext().unwrap().getId().equals("transient"); 175 | try (BufferedReader in = new BufferedReader(new InputStreamReader(request.getInputStream(), StandardCharsets.UTF_8))) { 176 | while ((line = in.readLine()) != null) { 177 | if (isTransient) { 178 | // jclouds does not escape things correctly 179 | line = BULK_DELETE_ESCAPER.escape(URLDecoder.decode(line, "UTF-8")); 180 | } 181 | objects.add(line); 182 | } 183 | } catch (IOException e) { 184 | throw propagate(e); 185 | } 186 | 187 | BulkDeleteResult result = new BulkDeleteResult(); 188 | for (String objectContainer : objects) { 189 | try { 190 | if (objectContainer.startsWith("/")) { 191 | objectContainer = objectContainer.substring(1); 192 | } 193 | int separatorIndex = objectContainer.indexOf('/'); 194 | if (separatorIndex < 0) { 195 | blobStore.deleteContainer(objectContainer.substring(1)); 196 | result.numberDeleted += 1; 197 | continue; 198 | } 199 | String container = objectContainer.substring(0, separatorIndex); 200 | String object = objectContainer.substring(separatorIndex + 1); 201 | 202 | if (!blobStore.blobExists(container, object)) { 203 | result.numberNotFound += 1; 204 | } else { 205 | blobStore.removeBlob(container, object); 206 | result.numberDeleted += 1; 207 | } 208 | } catch (ContainerNotFoundException e) { 209 | result.numberNotFound += 1; 210 | } catch (Exception e) { 211 | e.printStackTrace(); 212 | result.errors.add(objectContainer); 213 | } 214 | } 215 | 216 | if (result.errors.isEmpty()) { 217 | result.responseStatus = Response.Status.OK.toString(); 218 | return result; 219 | } else { 220 | ObjectMapper mapper = new ObjectMapper(); 221 | result.responseStatus = Response.Status.BAD_GATEWAY.toString(); 222 | throw new WebApplicationException(mapper.writeValueAsString(result), Response.Status.BAD_GATEWAY); 223 | } 224 | } 225 | 226 | @XmlRootElement(name = "account") 227 | @XmlType 228 | static class Account { 229 | @XmlElement 230 | List container; 231 | @XmlAttribute 232 | private String name; 233 | } 234 | 235 | @XmlRootElement(name = "container") 236 | @XmlType 237 | static class ContainerEntry { 238 | @XmlElement 239 | private String name; 240 | @XmlElement 241 | private long count; 242 | @XmlElement 243 | private long bytes; 244 | 245 | // for jackson XML 246 | ContainerEntry() { 247 | 248 | } 249 | 250 | @JsonCreator 251 | ContainerEntry(@JsonProperty("name") String name) { 252 | this.name = requireNonNull(name); 253 | } 254 | 255 | @Override 256 | public boolean equals(Object other) { 257 | return other instanceof ContainerEntry && 258 | name.equals(((ContainerEntry) other).name); 259 | } 260 | 261 | @Override 262 | public int hashCode() { 263 | return Objects.hash(name, count, bytes); 264 | } 265 | 266 | public String getName() { 267 | return name; 268 | } 269 | 270 | @Override 271 | public String toString() { 272 | return name; 273 | } 274 | } 275 | 276 | static class BulkDeleteResult { 277 | @JsonProperty("Response Status") 278 | String responseStatus; 279 | @JsonProperty("Errors") 280 | ArrayList errors; 281 | @JsonProperty("Number Deleted") 282 | int numberDeleted; 283 | @JsonProperty("Number Not Found") 284 | int numberNotFound; 285 | 286 | BulkDeleteResult() { 287 | errors = new ArrayList<>(); 288 | numberDeleted = 0; 289 | numberNotFound = 0; 290 | } 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /src/main/java/com/bouncestorage/swiftproxy/v1/ContainerResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Bounce Storage, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bouncestorage.swiftproxy.v1; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | import static com.google.common.base.Throwables.propagate; 22 | 23 | import java.io.UnsupportedEncodingException; 24 | import java.net.URLDecoder; 25 | import java.text.SimpleDateFormat; 26 | import java.time.Instant; 27 | import java.util.Date; 28 | import java.util.List; 29 | import java.util.Optional; 30 | import java.util.stream.Collectors; 31 | import java.util.stream.StreamSupport; 32 | 33 | import javax.validation.constraints.NotNull; 34 | import javax.ws.rs.BadRequestException; 35 | import javax.ws.rs.DELETE; 36 | import javax.ws.rs.DefaultValue; 37 | import javax.ws.rs.GET; 38 | import javax.ws.rs.HEAD; 39 | import javax.ws.rs.HeaderParam; 40 | import javax.ws.rs.POST; 41 | import javax.ws.rs.PUT; 42 | import javax.ws.rs.Path; 43 | import javax.ws.rs.PathParam; 44 | import javax.ws.rs.QueryParam; 45 | import javax.ws.rs.core.HttpHeaders; 46 | import javax.ws.rs.core.MediaType; 47 | import javax.ws.rs.core.Response; 48 | import javax.xml.bind.annotation.XmlAttribute; 49 | import javax.xml.bind.annotation.XmlElement; 50 | import javax.xml.bind.annotation.XmlRootElement; 51 | import javax.xml.bind.annotation.XmlType; 52 | 53 | import com.bouncestorage.swiftproxy.BlobStoreResource; 54 | import com.bouncestorage.swiftproxy.BounceResourceConfig; 55 | import com.bouncestorage.swiftproxy.Utils; 56 | import com.fasterxml.jackson.annotation.JsonCreator; 57 | import com.fasterxml.jackson.annotation.JsonProperty; 58 | import com.fasterxml.jackson.databind.annotation.JsonSerialize; 59 | import com.fasterxml.jackson.databind.ser.std.DateSerializer; 60 | import com.google.common.base.Strings; 61 | 62 | import org.jclouds.blobstore.BlobStore; 63 | import org.jclouds.blobstore.domain.BlobMetadata; 64 | import org.jclouds.blobstore.domain.StorageMetadata; 65 | import org.jclouds.blobstore.domain.StorageType; 66 | import org.jclouds.blobstore.options.ListContainerOptions; 67 | 68 | @Path("/v1/{account}/{container}") 69 | public final class ContainerResource extends BlobStoreResource { 70 | 71 | private void createContainer(String authToken, String container) { 72 | if (container.length() > InfoResource.CONFIG.swift.max_container_name_length) { 73 | throw new BadRequestException("container name too long"); 74 | } 75 | 76 | getBlobStore(authToken).get(container).createContainerInLocation(null, container); 77 | } 78 | 79 | @POST 80 | public Response postContainer(@NotNull @PathParam("container") String container, 81 | @HeaderParam("X-Auth-Token") String authToken, 82 | @HeaderParam("X-Container-Read") String readACL, 83 | @HeaderParam("X-Container-write") String writeACL, 84 | @HeaderParam("X-Container-Sync-To") String syncTo, 85 | @HeaderParam("X-Container-Sync-Key") String syncKey, 86 | @HeaderParam("X-Versions-Location") String versionsLocation, 87 | @HeaderParam(HttpHeaders.CONTENT_TYPE) String contentType, 88 | @HeaderParam("X-Detect-Content-Type") boolean detectContentType, 89 | @HeaderParam(HttpHeaders.IF_NONE_MATCH) String ifNoneMatch) { 90 | createContainer(authToken, container); 91 | return Response.status(Response.Status.NO_CONTENT).build(); 92 | } 93 | 94 | @PUT 95 | public Response putContainer(@NotNull @PathParam("container") String container, 96 | @HeaderParam("X-Auth-Token") String authToken, 97 | @HeaderParam("X-Container-Read") String readACL, 98 | @HeaderParam("X-Container-write") String writeACL, 99 | @HeaderParam("X-Container-Sync-To") String syncTo, 100 | @HeaderParam("X-Container-Sync-Key") String syncKey, 101 | @HeaderParam("X-Versions-Location") String versionsLocation, 102 | @HeaderParam(HttpHeaders.CONTENT_TYPE) String contentType, 103 | @HeaderParam("X-Detect-Content-Type") boolean detectContentType, 104 | @HeaderParam(HttpHeaders.IF_NONE_MATCH) String ifNoneMatch) { 105 | Response.Status status; 106 | BlobStore store = getBlobStore(authToken).get(container); 107 | 108 | if (store.containerExists(container)) { 109 | status = Response.Status.ACCEPTED; 110 | } else { 111 | createContainer(authToken, container); 112 | status = Response.Status.CREATED; 113 | } 114 | 115 | return Response.status(status).build(); 116 | } 117 | 118 | @DELETE 119 | public Response deleteContainer(@NotNull @PathParam("container") String container, 120 | @HeaderParam("X-Auth-Token") String authToken) { 121 | BlobStore store = getBlobStore(authToken).get(container); 122 | if (!store.containerExists(container)) { 123 | return Response.status(Response.Status.NOT_FOUND).build(); 124 | } 125 | 126 | if (store.deleteContainerIfEmpty(container)) { 127 | return Response.noContent().build(); 128 | } else { 129 | return Response.status(Response.Status.CONFLICT) 130 | .entity("

Conflict

There was a conflict when trying to complete your request.

") 131 | .build(); 132 | } 133 | } 134 | 135 | @HEAD 136 | public Response headContainer(@NotNull @PathParam("container") String container, 137 | @HeaderParam("X-Auth-Token") String authToken, 138 | @HeaderParam("X-Newest") @DefaultValue("false") boolean newest) { 139 | BlobStore store = getBlobStore(authToken).get(container); 140 | if (!store.containerExists(container)) { 141 | return Response.status(Response.Status.NOT_FOUND).build(); 142 | } 143 | 144 | long objectCount = -1; 145 | String provider = store.getContext().unwrap().getId(); 146 | if (provider.equals("transient") || provider.equals("openstack-swift")) { 147 | objectCount = store.countBlobs(container); 148 | } 149 | 150 | return Response.status(Response.Status.NO_CONTENT).entity("") 151 | .header("X-Container-Object-Count", objectCount) 152 | .header("X-Container-Bytes-Used", 0) // TODO: bogus value 153 | .header("X-Versions-Location", "") 154 | .header("X-Timestamp", -1) 155 | .header("X-Trans-Id", -1) 156 | .header("Accept-Ranges", "bytes") 157 | .build(); 158 | } 159 | 160 | private String contentType(StorageMetadata meta) { 161 | if (meta instanceof BlobMetadata) { 162 | String contentType = ((BlobMetadata) meta).getContentMetadata().getContentType(); 163 | if (contentType != null && !contentType.isEmpty()) { 164 | return contentType; 165 | } 166 | } 167 | if (meta.getType().equals(StorageType.RELATIVE_PATH) || meta.getName().endsWith("/")) { 168 | return "application/directory"; 169 | } 170 | return MediaType.APPLICATION_OCTET_STREAM; 171 | } 172 | 173 | @GET 174 | public Response listContainer(@NotNull @PathParam("container") String container, 175 | @HeaderParam("X-Auth-Token") String authToken, 176 | @QueryParam("limit") Integer limit, 177 | @QueryParam("marker") String marker, 178 | @QueryParam("end_marker") String endMarker, 179 | @QueryParam("format") Optional format, 180 | @QueryParam("prefix") String prefixParam, 181 | @QueryParam("delimiter") String delimiterParam, 182 | @QueryParam("path") String path, 183 | @HeaderParam("X-Newest") @DefaultValue("false") boolean newest, 184 | @HeaderParam("Accept") Optional accept) { 185 | BlobStore store = getBlobStore(authToken).get(container); 186 | if (!store.containerExists(container)) { 187 | return Response.status(Response.Status.NOT_FOUND).build(); 188 | } 189 | 190 | ListContainerOptions options = new ListContainerOptions(); 191 | if (!Strings.isNullOrEmpty(marker)) { 192 | options.afterMarker(marker); 193 | } 194 | 195 | if (Strings.isNullOrEmpty(delimiterParam) && path == null) { 196 | options.recursive(); 197 | } 198 | 199 | if (!Strings.isNullOrEmpty(delimiterParam)) { 200 | options.delimiter(delimiterParam); 201 | } 202 | 203 | if (!Strings.isNullOrEmpty(prefixParam)) { 204 | options.prefix(prefixParam); 205 | } 206 | 207 | if (path != null) { 208 | if (path.equals("/")) { 209 | options.prefix("/"); 210 | options.delimiter("/"); 211 | } else { 212 | options.inDirectory(path); 213 | } 214 | } 215 | 216 | logger.info("list: {} marker={} prefix={}", options, options.getMarker(), prefixParam); 217 | List entries = StreamSupport.stream( 218 | Utils.crawlBlobStore(store, container, options).spliterator(), false) 219 | .peek(meta -> logger.debug("meta: {}", meta)) 220 | //.filter(meta -> (prefix == null || meta.getName().startsWith(prefix))) 221 | //.filter(meta -> delimFilter(meta.getName(), delim_filter)) 222 | .filter(meta -> endMarker == null || meta.getName().compareTo(endMarker) < 0) 223 | .limit(limit == null ? InfoResource.CONFIG.swift.container_listing_limit : limit) 224 | .map(meta -> new ObjectEntry(meta.getName(), meta.getETag(), 225 | meta.getSize() == null ? 0 : meta.getSize(), 226 | contentType(meta), meta.getLastModified())) 227 | .collect(Collectors.toList()); 228 | 229 | MediaType formatType; 230 | if (format.isPresent()) { 231 | formatType = BounceResourceConfig.getMediaType(format.get()); 232 | } else if (accept.isPresent()) { 233 | formatType = MediaType.valueOf(accept.get()); 234 | } else { 235 | formatType = MediaType.TEXT_PLAIN_TYPE; 236 | } 237 | 238 | if (store.getContext().unwrap().getId().equals("transient")) { 239 | entries.forEach(entry -> { 240 | try { 241 | entry.name = URLDecoder.decode(entry.name, "UTF-8"); 242 | } catch (UnsupportedEncodingException e) { 243 | throw propagate(e); 244 | } 245 | }); 246 | } 247 | 248 | // XXX semi-bogus value 249 | long totalBytes = entries.stream().mapToLong(e -> e.bytes).sum(); 250 | 251 | ContainerRoot root = new ContainerRoot(); 252 | root.name = container; 253 | root.object = entries; 254 | return output(root, entries, formatType) 255 | .header("X-Container-Object-Count", entries.size()) 256 | .header("X-Container-Bytes-Used", totalBytes) 257 | .header("X-Timestamp", -1) 258 | .header("X-Trans-Id", -1) 259 | .header("Accept-Ranges", "bytes") 260 | .build(); 261 | 262 | } 263 | 264 | @XmlRootElement(name = "container") 265 | @XmlType 266 | static class ContainerRoot { 267 | @XmlElement 268 | List object; 269 | @XmlAttribute 270 | private String name; 271 | } 272 | 273 | @XmlRootElement(name = "object") 274 | @XmlType 275 | static class ObjectEntry { 276 | static class SwiftDateSerializer extends DateSerializer { 277 | SwiftDateSerializer() { 278 | super(false, new SimpleDateFormat("yyyy-MM-dd'T'kk:mm:ss.SSSSSS")); 279 | } 280 | } 281 | 282 | @XmlElement 283 | String name; 284 | @XmlElement 285 | String hash; 286 | @XmlElement 287 | long bytes; 288 | @XmlElement 289 | String content_type; 290 | @JsonSerialize(using = SwiftDateSerializer.class) 291 | @XmlElement 292 | Date last_modified; 293 | 294 | // dummy 295 | ObjectEntry() { 296 | } 297 | 298 | @JsonCreator 299 | ObjectEntry(@JsonProperty("name") String name, 300 | @JsonProperty("hash") String hash, 301 | @JsonProperty("bytes") long bytes, 302 | @JsonProperty("content_type") String content_type, 303 | @JsonProperty("last_modified") Date last_modified) { 304 | this.name = requireNonNull(name); 305 | this.hash = hash == null ? "" : Utils.trimETag(hash); 306 | this.bytes = bytes; 307 | this.content_type = requireNonNull(content_type); 308 | this.last_modified = last_modified == null ? Date.from(Instant.EPOCH) : last_modified; 309 | } 310 | 311 | @Override 312 | public String toString() { 313 | return name; 314 | } 315 | } 316 | } 317 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 4.0.0 5 | 6 | 7 | org.sonatype.oss 8 | oss-parent 9 | 7 10 | 11 | 12 | com.bouncestorage 13 | swift-proxy 14 | 1.2.0-SNAPSHOT 15 | jar 16 | 17 | swift-proxy 18 | https://github.com/bouncestorage/swiftproxy 19 | Access other object stores via the Swift API 20 | 21 | 22 | 23 | The Apache Software License, Version 2.0 24 | http://www.apache.org/licenses/LICENSE-2.0.txt 25 | repo 26 | 27 | 28 | 29 | 30 | scm:git:git@github.com:bouncestorage/swiftproxy.git 31 | scm:git:git@github.com:bouncestorage/swiftproxy.git 32 | git@github.com:bouncestorage/swiftproxy.git 33 | 34 | 35 | 36 | 37 | Ka-Hing Cheung 38 | khc 39 | khc@bouncestorage.com 40 | 41 | 42 | 43 | 44 | 45 | ossrh 46 | https://oss.sonatype.org/content/repositories/snapshots 47 | 48 | 49 | 50 | 51 | 52 | release 53 | 54 | 55 | 56 | org.apache.maven.plugins 57 | maven-gpg-plugin 58 | 1.5 59 | 60 | 61 | sign-artifacts 62 | verify 63 | 64 | sign 65 | 66 | 67 | 68 | 69 | 70 | org.apache.maven.plugins 71 | maven-source-plugin 72 | 2.2.1 73 | 74 | 75 | attach-sources 76 | 77 | jar-no-fork 78 | 79 | 80 | 81 | 82 | 83 | org.apache.maven.plugins 84 | maven-javadoc-plugin 85 | 2.9.1 86 | 87 | 88 | attach-javadocs 89 | 90 | jar 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | org.glassfish.jersey 104 | jersey-bom 105 | ${jersey.version} 106 | pom 107 | import 108 | 109 | 110 | 111 | 112 | 113 | 3.0.5 114 | 115 | 116 | 117 | 118 | apache-snapshots 119 | https://repository.apache.org/content/repositories/snapshots 120 | 121 | true 122 | 123 | 124 | 125 | 126 | 127 | 128 | javax.xml.bind 129 | jaxb-api 130 | 2.3.1 131 | 132 | 133 | org.glassfish.jersey.containers 134 | jersey-container-grizzly2-http 135 | 136 | 137 | org.glassfish.jersey.media 138 | jersey-media-json-jackson 139 | ${jersey.version} 140 | 141 | 148 | 149 | junit 150 | junit 151 | 4.13.2 152 | test 153 | 154 | 155 | org.apache.jclouds 156 | jclouds-allblobstore 157 | ${jclouds.version} 158 | 159 | 160 | org.apache.jclouds.api 161 | filesystem 162 | ${jclouds.version} 163 | 164 | 165 | org.apache.jclouds.driver 166 | jclouds-slf4j 167 | ${jclouds.version} 168 | 169 | 170 | ch.qos.logback 171 | logback-classic 172 | 1.5.21 173 | 174 | 175 | org.assertj 176 | assertj-core 177 | test 178 | 179 | 3.27.6 180 | 181 | 182 | commons-io 183 | commons-io 184 | 2.7 185 | 186 | 187 | org.apache.commons 188 | commons-lang3 189 | 3.4 190 | 191 | 192 | 193 | org.apache.jclouds 194 | jclouds-blobstore 195 | ${jclouds.version} 196 | test-jar 197 | test 198 | 199 | 200 | org.apache.jclouds.api 201 | openstack-swift 202 | ${jclouds.version} 203 | test-jar 204 | test 205 | 206 | 207 | org.apache.jclouds 208 | jclouds-core 209 | ${jclouds.version} 210 | test-jar 211 | test 212 | 213 | 214 | org.testng 215 | testng 216 | 7.5.1 217 | test 218 | 219 | 220 | 221 | 222 | 223 | 224 | org.apache.maven.plugins 225 | maven-checkstyle-plugin 226 | 3.1.0 227 | 228 | 229 | com.puppycrawl.tools 230 | checkstyle 231 | 8.29 232 | 233 | 234 | 235 | 236 | checkstyle 237 | verify 238 | 239 | check 240 | 241 | 242 | 243 | 244 | src/main/resources/checkstyle.xml 245 | src/main/resources/copyright_header.txt 246 | true 247 | warning 248 | 249 | 250 | 251 | org.apache.maven.plugins 252 | maven-compiler-plugin 253 | 3.5.1 254 | true 255 | 256 | 1.8 257 | 1.8 258 | 259 | 260 | 261 | org.codehaus.mojo 262 | exec-maven-plugin 263 | 3.6.2 264 | 265 | 266 | 267 | java 268 | 269 | 270 | 271 | 272 | com.bouncestorage.swiftproxy.Main 273 | --properties ${swiftproxy.conf} 274 | 275 | 276 | 277 | org.codehaus.mojo 278 | findbugs-maven-plugin 279 | 3.0.5 280 | 281 | Max 282 | FindDeadLocalStores,UnreadFields 283 | 284 | 285 | 286 | org.apache.maven.plugins 287 | maven-assembly-plugin 288 | 2.6 289 | 290 | 291 | src/main/assembly/jar-with-dependencies.xml 292 | 293 | 294 | 295 | com.bouncestorage.swiftproxy.Main 296 | 297 | 298 | 299 | 300 | 301 | make-assembly 302 | package 303 | 304 | single 305 | 306 | 307 | 308 | 309 | 310 | org.gaul 311 | modernizer-maven-plugin 312 | 3.2.0 313 | 314 | 315 | modernizer 316 | verify 317 | 318 | modernizer 319 | 320 | 321 | 322 | 323 | 1.8 324 | 325 | 326 | 327 | org.sonatype.plugins 328 | nexus-staging-maven-plugin 329 | 1.7.0 330 | true 331 | 332 | ossrh 333 | https://oss.sonatype.org/ 334 | true 335 | 336 | 337 | 338 | org.apache.maven.plugins 339 | maven-surefire-plugin 340 | ${surefire.version} 341 | 342 | 343 | org.apache.maven.surefire 344 | surefire-junit47 345 | ${surefire.version} 346 | 347 | 348 | org.apache.maven.surefire 349 | surefire-testng 350 | ${surefire.version} 351 | 352 | 353 | 354 | methods 355 | 1 356 | -Xmx512m 357 | true 358 | 300 359 | random 360 | false 361 | 362 | 363 | junit 364 | false 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 2.18 374 | UTF-8 375 | 2.7.0 376 | 3.1.0 377 | src/main/resources/swiftproxy.conf 378 | 379 | 380 | --------------------------------------------------------------------------------