├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── acceptance_test ├── acceptance_test.go ├── ca_cert ├── cert_file ├── config.yml ├── helpers.go ├── key_file ├── with-http │ ├── acceptance_test.go │ └── config.yml └── with-identical-registry-and-public │ ├── acceptance_test.go │ └── config.yml ├── api-doc ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── README.md ├── config.rb ├── deploy.sh ├── font-selection.json └── source │ ├── fonts │ ├── slate.eot │ ├── slate.svg │ ├── slate.ttf │ ├── slate.woff │ └── slate.woff2 │ ├── images │ ├── api_doc_logo.png │ ├── logo.png │ └── navbar.png │ ├── index.html.md │ ├── javascripts │ ├── all.js │ ├── all_nosearch.js │ ├── app │ │ ├── _lang.js │ │ ├── _search.js │ │ └── _toc.js │ └── lib │ │ ├── _energize.js │ │ ├── _imagesloaded.min.js │ │ ├── _jquery.highlight.js │ │ ├── _jquery.js │ │ ├── _jquery.tocify.js │ │ ├── _jquery_ui.js │ │ └── _lunr.js │ ├── layouts │ └── layout.erb │ └── stylesheets │ ├── _icon-font.scss │ ├── _normalize.scss │ ├── _variables.scss │ ├── print.css.scss │ └── screen.css.scss ├── app_stash_handler.go ├── app_stash_handler_test.go ├── assets ├── lots-of-files.zip └── test-file.zip ├── bitsgo_suite_test.go ├── blobstore.go ├── blobstores ├── alibaba │ └── alibaba_blobstore.go ├── azure │ └── azure_blobstore.go ├── blobstore_test.go ├── contract_integ_test │ ├── blobstore_integ_test.go │ ├── contract_integ_test_suite_test.go │ ├── mock_metricsservice_test.go │ ├── run-contract-integ-tests.sh │ └── slow_integ_test.go ├── decorator │ ├── metrics_emitting_blobstore_decorator.go │ ├── partitioning_path_blobstore_decorator.go │ └── prefixing_path_blobstore_decorator.go ├── gcp │ ├── gcp_blobstore.go │ └── timeout_retry_test.go ├── inmemory │ └── inmemory_blobstore.go ├── local │ ├── local_blobstore.go │ └── signed_urls.go ├── openstack │ ├── openstack_blobstore.go │ ├── openstack_blobstore_suite_test.go │ └── openstack_blobstore_test.go ├── s3 │ ├── s3_blobstore.go │ ├── signed_urls_test.go │ ├── signer │ │ ├── default.go │ │ ├── gcp.go │ │ └── v2.go │ └── util.go ├── validate │ └── validate.go └── webdav │ ├── http_client.go │ ├── webdav_blobstore.go │ ├── webdav_blobstore_test.go │ └── webdav_suite_test.go ├── body_size_limit.go ├── body_size_limit_test.go ├── ccupdater ├── ccupdater.go ├── ccupdater_suite_test.go ├── ccupdater_test.go ├── matchers │ ├── ptr_to_http_request.go │ └── ptr_to_http_response.go └── mock_httpclient_test.go ├── cmd ├── bitsgo │ ├── factory.go │ └── main.go └── dashboard │ └── main.go ├── config ├── config.go └── config_test.go ├── docs ├── README.markdown ├── baremetal-bosh-lite.markdown ├── bits_logo_horizontal.svg ├── cf-deployment-in-softlayer.md ├── create-app-with-bits-service.png ├── create-app-with-bits-service.txt ├── create-app-with-bits-service_proposal_for_cli.png ├── create-app-with-bits-service_proposal_for_cli.txt ├── create-app.png ├── create-app.txt ├── create-droplet-with-bits-service.png ├── create-droplet-with-bits-service.txt ├── create-droplet.png ├── create-droplet.txt ├── create-v3-app-with-bits-service-async-no-resource-match.png ├── create-v3-app-with-bits-service-async-no-resource-match.txt ├── create-v3-app-with-bits-service-async-with-resource-match.png ├── create-v3-app-with-bits-service-async-with-resource-match.txt ├── create-v3-app-with-bits-service-sync-no-resource-match.png ├── create-v3-app-with-bits-service-sync-no-resource-match.txt ├── create-v3-app.png ├── create-v3-app.txt ├── onboarding.markdown ├── status-quo.md └── websequencediagram ├── glide.lock ├── glide.yaml ├── httputil └── httputil.go ├── images ├── GET-request-speed-comparison-10-1.png ├── GET-request-speed-comparison-100-10.png ├── GET-request-speed-comparison-100-100.png ├── PUT-request-speed-comparison.png ├── bits-service-ruby-mem-consumption.png └── routes-and-stores.png ├── logger └── logger.go ├── matchers └── slice_of_byte.go ├── metrics_service.go ├── middlewares ├── basic_auth_middleware.go ├── basic_auth_middleware_test.go ├── body_size_limit_middleware.go ├── body_size_limit_middleware_test.go ├── logger_middleware.go ├── matchers │ └── time_duration.go ├── metrics_middleware.go ├── metrics_middleware_test.go ├── mock_handler_test.go ├── mock_metricsservice_test.go ├── multipart_middleware.go ├── negroni_gorilla_adapter.go ├── panic_middleware.go ├── panic_middleware_test.go ├── signature_verification_middleware.go └── signature_verification_middleware_test.go ├── mock_blobstore_test.go ├── mock_metricsservice_test.go ├── mock_readcloser_test.go ├── mock_resourcesigner_test.go ├── mock_updater_test.go ├── oci_registry ├── assets │ └── example_droplet ├── models │ └── docker │ │ ├── mediatype │ │ └── models.go │ │ └── models.go ├── oci_registry_suite_test.go ├── registry.go └── registry_test.go ├── package_bundle.go ├── package_bundle_test.go ├── pathsigner ├── pathsigner.go └── pathsigner_test.go ├── performance-comparison.md ├── resource_handler.go ├── resource_handler_test.go ├── routes ├── mock_metricsservice_test.go ├── resource_routes.go ├── resource_routes_test.go └── test_data │ └── test_archive.zip ├── scripts ├── build-and-run ├── install-git-hooks.sh ├── list-lastpass-passwords.sh └── run-unit-tests ├── sign_handler.go ├── sign_handler_test.go ├── statsd └── metrics_service.go ├── testutil ├── http.go └── zip.go └── util ├── context.go ├── panic.go ├── response.go └── safecloser.go /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | **/*integration_test_config*.yml 3 | vendor 4 | api-doc/build -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Bits-Service 2 | 3 | Copyright (c) 2013-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bits Service 2 | **Please note: the Bits-Service is not actively maintained anymore. [More information](https://lists.cloudfoundry.org/g/cf-dev/message/8660).** 3 | 4 | 5 | 6 | 7 | 8 | The bits-service is an extraction from existing functionality of the [cloud controller](https://github.com/cloudfoundry/cloud_controller_ng). It encapsulates all "bits operations" into its own, separately scalable service. All bits operations comprise buildpacks, droplets, app_stashes, packages and the buildpack_cache. 9 | 10 | [The API](http://cloudfoundry-incubator.github.io/bits-service/) is a work in progress and will most likely change. 11 | 12 | ## Supported Backends 13 | 14 | Bits currently supports [WebDAV](https://en.wikipedia.org/wiki/WebDAV) and the following [Fog](http://fog.io/) connectors: 15 | 16 | * AWS S3 17 | * Azure 18 | * Google 19 | * Local (NFS) 20 | * Openstack 21 | 22 | 23 | ## Development 24 | 25 | The CI config is in the [bits-service-ci](https://github.com/cloudfoundry-incubator/bits-service-ci) repo. 26 | 27 | 28 | ## Additional Notes 29 | 30 | It can be used standalone or through its [BOSH-release](https://github.com/cloudfoundry-incubator/bits-service-release). 31 | 32 | ## Getting Started 33 | 34 | Make sure you have a working [Go environment](https://golang.org/doc/install) and the Go vendoring tool [glide](https://github.com/Masterminds/glide#install) is properly installed. 35 | 36 | To install bitsgo: 37 | 38 | ```bash 39 | mkdir -p $GOPATH/src/github.com/cloudfoundry-incubator 40 | cd $GOPATH/src/github.com/cloudfoundry-incubator 41 | 42 | git clone https://github.com/cloudfoundry-incubator/bits-service.git 43 | cd bits-service 44 | 45 | glide install 46 | 47 | cd cmd/bitsgo 48 | go install 49 | ``` 50 | 51 | Then run it: 52 | 53 | ``` 54 | bitsgo --config my/path/to/config.yml 55 | ``` 56 | 57 | To run tests: 58 | 59 | 1. Install [ginkgo](https://onsi.github.io/ginkgo/#getting-ginkgo) 60 | 1. Configure `$PATH`: 61 | 62 | ```bash 63 | export PATH=$GOPATH/bin:$PATH 64 | ``` 65 | 66 | 1. Run tests with 67 | 68 | ```bash 69 | scripts/run-unit-tests 70 | ``` 71 | 72 | ## Contributing to Bits-Service 73 | The Bits-Service team is happy to receive feedback, suggestions, improvements and Pull Requests.

74 | 75 | **If you want to create a Pull Request against Bits-Service please make sure that the Unit Tests are passing successfully** (as described in the [Getting Started section](#Getting-started)) 76 | 77 | If you would like to discuss about possible changes or improvements feel free to reach out to us via [Bits-Service Cloud Foundry Slack](https://cloudfoundry.slack.com/messages/C0BNGJY0G) 78 | -------------------------------------------------------------------------------- /acceptance_test/ca_cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDEzCCAfugAwIBAgIQPcITKGgjCxXcIPp9mFiSJTANBgkqhkiG9w0BAQsFADAz 3 | MQwwCgYDVQQGEwNVU0ExFjAUBgNVBAoTDUNsb3VkIEZvdW5kcnkxCzAJBgNVBAMT 4 | AmNhMB4XDTE4MDgxNTE3MDMyNloXDTE5MDgxNTE3MDMyNlowMzEMMAoGA1UEBhMD 5 | VVNBMRYwFAYDVQQKEw1DbG91ZCBGb3VuZHJ5MQswCQYDVQQDEwJjYTCCASIwDQYJ 6 | KoZIhvcNAQEBBQADggEPADCCAQoCggEBANcTFkdZCiwmshhP0y5TtGIhBrVy4vE6 7 | /OvbX5du/o6vvzMXPiYFWDqTzwY3oTHWdHJXfROVxpmgrOquzjdQjV0lWThfAGB5 8 | diJhibqMjbMJg9vI4G6dh3Ev0Zxm4OzYNN5aEuw4sqsTQmdecY1+wLljyjqXLsa6 9 | vMNuKJRPBHvLvXywaETMR3iOIaxV/vsxRCA0FrbEhlaofiPoYQck38yEP7fMG/MX 10 | oe8LpntCXZMAShn4JpN4TTMBJ1637qmK2d7U10WFBWylrlH2xVpf1zjrnb5B8sZs 11 | ebYyoyJE1SYsaZt7p+O+iPVb+q77YmcQV0HWZYljiCRppjnTa0fXuD0CAwEAAaMj 12 | MCEwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL 13 | BQADggEBAAodsNnRnyxxZLgo7puKqUKOEiWm3YsO/bZjG7/8EMNo6bQ/H7lDxBkn 14 | pC9fS8pLOZaQvWF/95HchjUbV5WVhQzuRZ1wzzEb+WcT+ufgW/RMYo6qrNiQ7hbN 15 | 861hx0VKLGM4uAHRcGHCM1I8GvrHONCQWeFWa40sS7P8zAzzFpvUbFIcHvf8chEQ 16 | w1l0s7VWAMkuwqC/5xxQDu2pyKFcGkpjaxHQwvv9zCzIXcSYbT+5bV7NYrUpCkwf 17 | wdQKF71XCTJnKjcMCFpGAotKXblYvVjCYsPLm9YMIDROD05i1KPdgljM9aQnymsn 18 | fNphskva2Fa3YJX92iQW4UKum7JjMzk= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /acceptance_test/cert_file: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDZTCCAk2gAwIBAgIRAKnSWy5iYFKPf1EIci5sb2kwDQYJKoZIhvcNAQELBQAw 3 | MzEMMAoGA1UEBhMDVVNBMRYwFAYDVQQKEw1DbG91ZCBGb3VuZHJ5MQswCQYDVQQD 4 | EwJjYTAeFw0xODA4MTUxNzAzMjZaFw0xOTA4MTUxNzAzMjZaMEExDDAKBgNVBAYT 5 | A1VTQTEWMBQGA1UEChMNQ2xvdWQgRm91bmRyeTEZMBcGA1UEAxMQMTI3LjAuMC4x 6 | Lm5pcC5pbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANKuzYc/C/DN 7 | HYN2HC60iQVIOm9oi9fY+QvDL0lUhogw+zY7zv0sreWc+hyM7oJ2fhS3TSoAvrnh 8 | a/6UVbZe8U/aG/IeO2mlhV+JBZGcdsRhmKcxYBdtuCVwH67tUe2iAaTaI6EDXoOL 9 | ojfW8/sCn1I8pmhpZWQEuYxicLIpGmwF6AR8Ac2Va9aB7oqEqanQe4IG/NwsDsqz 10 | G9C4kuP5ePwpIupmnWyGPeax94v4fp2EbVqP6gNHdtiaAgEawhRnQMiuvoM3KDQe 11 | uz/qxHjfrz7AeJyJp7izV/w2625uYrWCRGkc6k8Jva7nRho2D7POcL+r33HWcQ63 12 | blSK/LVOVOMCAwEAAaNmMGQwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsG 13 | AQUFBwMBMAwGA1UdEwEB/wQCMAAwLwYDVR0RBCgwJoISKi4xMjcuMC4wLjEubmlw 14 | LmlvghAxMjcuMC4wLjEubmlwLmlvMA0GCSqGSIb3DQEBCwUAA4IBAQBkWsgOvXSI 15 | 7HGWyAtNSZ/YpZCo6YE906iLkbI6W/dYneJHhNJZ+wDgZIWsFXiXWP3z+AbN8mzB 16 | kOGHjIa86O3WZORVVjs+6WYtj8Kq4/ryI5POkPcldNRrH0gVgOKRD+yVIYWXdTuZ 17 | yJVrTVoIDHkjltEsmnWEZkhQseK0JRS3tKMXQqtHBejkMr3wjz2Fb2vXwVKKYDLB 18 | W3lY6bTAZkgez53qAXbCk4tY0djo0YQ9k+SwD+iviYGyPDl2fzk1m0unMX21tQAm 19 | TWeOmdCaNs0sZbnzfHZuGNzrec+HS80w4cXvwRTWKQ7/8nM5An93KDG1zroSb/Ak 20 | IVmFIxtfD1zs 21 | -----END CERTIFICATE----- 22 | -------------------------------------------------------------------------------- /acceptance_test/config.yml: -------------------------------------------------------------------------------- 1 | buildpacks: 2 | blobstore_type: local 3 | local_config: 4 | path_prefix: /tmp/buildpacks 5 | droplets: 6 | blobstore_type: local 7 | local_config: 8 | path_prefix: /tmp/droplets 9 | packages: 10 | blobstore_type: local 11 | local_config: 12 | path_prefix: /tmp/packages 13 | app_stash: 14 | blobstore_type: local 15 | local_config: 16 | path_prefix: /tmp/app_stash 17 | logging: 18 | file: /tmp/bits-service.log 19 | syslog: vcap.bits-service 20 | level: debug 21 | public_endpoint: https://public.127.0.0.1.nip.io 22 | private_endpoint: https://internal.127.0.0.1.nip.io 23 | secret: geheim 24 | signing_keys: 25 | - key_id: some_key_id 26 | secret: some_secret 27 | - key_id: some_other_key_id 28 | secret: some_other_secret 29 | active_key_id: some_other_key_id 30 | port: 4443 31 | cert_file: cert_file 32 | key_file: key_file 33 | signing_users: 34 | - username: the-username 35 | password: the-password 36 | metrics_log_destination: /tmp/bitsgo_metrics.log 37 | enable_registry: true 38 | registry_endpoint: https://registry.127.0.0.1.nip.io 39 | rootfs: 40 | blobstore_type: local 41 | local_config: 42 | path_prefix: /tmp/eirinifs 43 | -------------------------------------------------------------------------------- /acceptance_test/helpers.go: -------------------------------------------------------------------------------- 1 | package acceptance 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "os/exec" 10 | "time" 11 | 12 | "github.com/cloudfoundry-incubator/bits-service/util" 13 | 14 | "github.com/onsi/gomega/gexec" 15 | 16 | . "github.com/onsi/ginkgo" 17 | . "github.com/onsi/gomega" 18 | ) 19 | 20 | func StartServer(configYamlFile string) (session *gexec.Session) { 21 | pathToWebserver, err := gexec.Build("github.com/cloudfoundry-incubator/bits-service/cmd/bitsgo") 22 | Ω(err).ShouldNot(HaveOccurred()) 23 | 24 | os.Setenv("BITS_LISTEN_ADDR", "127.0.0.1") 25 | session, err = gexec.Start(exec.Command(pathToWebserver, "--config", configYamlFile), GinkgoWriter, GinkgoWriter) 26 | Ω(err).ShouldNot(HaveOccurred()) 27 | time.Sleep(200 * time.Millisecond) 28 | Expect(session.ExitCode()).To(Equal(-1), "Webserver error message: %s", string(session.Err.Contents())) 29 | return 30 | } 31 | 32 | func CreateTLSClient(caCertFile string) *http.Client { 33 | caCert, err := ioutil.ReadFile(caCertFile) 34 | util.PanicOnError(err) 35 | caCertPool := x509.NewCertPool() 36 | caCertPool.AppendCertsFromPEM(caCert) 37 | return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{ 38 | RootCAs: caCertPool, 39 | }}} 40 | } 41 | 42 | func SetUpAndTearDownServer() { 43 | var session *gexec.Session 44 | 45 | BeforeSuite(func() { 46 | session = StartServer("config.yml") 47 | }) 48 | 49 | AfterSuite(func() { 50 | if session != nil { 51 | session.Kill() 52 | } 53 | gexec.CleanupBuildArtifacts() 54 | os.Remove("/tmp/eirinifs/assets/eirinifs.tar") 55 | time.Sleep(2 * time.Second) 56 | }) 57 | } 58 | 59 | func CreateFakeEiriniFS() { 60 | err := os.MkdirAll("/tmp/eirinifs/assets", 0755) 61 | util.PanicOnError(err) 62 | file, err := os.Create("/tmp/eirinifs/assets/eirinifs.tar") 63 | util.PanicOnError(err) 64 | file.Close() 65 | } 66 | 67 | func GetStatusCode(response *http.Response) int { return response.StatusCode } 68 | -------------------------------------------------------------------------------- /acceptance_test/key_file: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEA0q7Nhz8L8M0dg3YcLrSJBUg6b2iL19j5C8MvSVSGiDD7NjvO 3 | /Syt5Zz6HIzugnZ+FLdNKgC+ueFr/pRVtl7xT9ob8h47aaWFX4kFkZx2xGGYpzFg 4 | F224JXAfru1R7aIBpNojoQNeg4uiN9bz+wKfUjymaGllZAS5jGJwsikabAXoBHwB 5 | zZVr1oHuioSpqdB7ggb83CwOyrMb0LiS4/l4/Cki6madbIY95rH3i/h+nYRtWo/q 6 | A0d22JoCARrCFGdAyK6+gzcoNB67P+rEeN+vPsB4nImnuLNX/Dbrbm5itYJEaRzq 7 | Twm9rudGGjYPs85wv6vfcdZxDrduVIr8tU5U4wIDAQABAoIBAQCqxNeBFaBXGdsf 8 | UJ/9V4ZsyqPa49HeLleHqjjs5l95isJErmvTmOCHahrtA2rcQrFll0IrlpkBz+8a 9 | Vh+3h/NvNypJsSmVM34M9uKVqU43VahHdOUfeGa3qc90mZY72IIenlYcz79PPuJJ 10 | AqmWPwTmwpRbs3pkfNPtL2mLb3ekMBfunpRtSUaToh+SRkn9uxAi1XMV4HJ68AJO 11 | Peg4KFJC285DfkapKr6PU/77+DIl1HxBtumF3lkh6bg2i6jrNNf9m4aQNZK+P+Tx 12 | l3waOARjE5/FcVUOVn3Z7fxJwVxDjuyQCJ/hmBYPMoJHpGVqBWpMCVc6hj/UKzjB 13 | MPDhbgn5AoGBANgaqTuzlgpDjmMyf5dxPi3xPKfrl6IGWI0y45kNtK8mgxP92S/F 14 | zMO2ssZzfi11cFTkVF53nS1/37sQ8Y+5kFbuX8L5sbRPCHzUWXOqZ207Ua3hNv6b 15 | TtK9USVD+bIcyKVv5a4S5o9w14WRSbZT5YI2AJBPCWDpXefRKeqscwo3AoGBAPmT 16 | 7Hb2Jiqnr1eaZmN3sWRnejuEWRHu4XWbytp5MXJu1LyBPu//nMo9F7wBBtqDTD5U 17 | fh764M+PRtevmVFRmi5f5hFsIhw4Y2XoAHULHSN6hQdD5LBukvPTIHr5K/Qpdq/N 18 | Y6FKzWfVzO6Ig+Ii5locBljbXSooaN6+5ckiCsS1AoGAGAxXetJQRxIffUB4XGT4 19 | s2oeAt2/wQMNxaC9HSIeUkNp4Mal7aAIWlsxZ84gY3SnLHtAPEb5Ub/iKNII36KZ 20 | wmLCe1MICHWnDyUeUzXKTqiEPWJLmWe1DNSOfCQlXEHBvk9GcumdiKbZBP8XAdgy 21 | ORxDUcvJ0mQF1C89h+Tq5F8CgYEA3+JIj3bESiNecbF6A+SNZ0pEJjvVQvcNnVkC 22 | IfXx23t8rxUqBlVAq1MehXJOWZrKvGdDNDtNjCQ1IqrNWFthehRg6GQePT6APBxg 23 | vJ4Zp4fy6c+HyJWIkd1lF6uKOF8xrwcKRtg5ZtouGhSwah1wkojtUKyH6JeTa63H 24 | qCQ3kLkCgYAJZ+ErOpH/Qxioc7ZXkK0jDCwavdFzZCeMIB6xszJd/wUGiXXNKPsF 25 | 16tzRHCby4Tk9e4GsLBDl5I6wOoyjCeTEueHtOzAERqXoOjmrWJeOruYW0NPmAyH 26 | lEaVoCW8/ZFkGUUMYfF4QxdM1QB5p8WxkFMl4KDAFoM0+VYPR/VsvA== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /acceptance_test/with-http/acceptance_test.go: -------------------------------------------------------------------------------- 1 | package acceptance_test 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "testing" 7 | 8 | . "github.com/cloudfoundry-incubator/bits-service/acceptance_test" 9 | acceptance "github.com/cloudfoundry-incubator/bits-service/acceptance_test" 10 | "github.com/cloudfoundry-incubator/bits-service/httputil" 11 | . "github.com/cloudfoundry-incubator/bits-service/testutil" 12 | "github.com/onsi/ginkgo" 13 | . "github.com/onsi/ginkgo" 14 | "github.com/onsi/gomega" 15 | . "github.com/onsi/gomega" 16 | ) 17 | 18 | func TestEndToEnd(t *testing.T) { 19 | gomega.RegisterFailHandler(ginkgo.Fail) 20 | acceptance.SetUpAndTearDownServer() 21 | ginkgo.RunSpecs(t, "EndToEnd HTTP") 22 | } 23 | 24 | var _ = Describe("Accessing the bits-service through HTTP", func() { 25 | Context("through private host", func() { 26 | Context("HTTP", func() { 27 | 28 | It("return http.StatusOK for a package that does exist", func() { 29 | client := &http.Client{} 30 | 31 | request, e := httputil.NewPutRequest("http://internal.127.0.0.1.nip.io:8888/packages/myguid", map[string]map[string]io.Reader{ 32 | "package": map[string]io.Reader{"somefilename": CreateZip(map[string]string{"somefile": "lalala\n\n"})}, 33 | }) 34 | Expect(e).NotTo(HaveOccurred()) 35 | 36 | Expect(client.Do(request)).To(WithTransform(GetStatusCode, Equal(201))) 37 | 38 | Expect(client.Get("http://internal.127.0.0.1.nip.io:8888/packages/myguid")). 39 | To(WithTransform(GetStatusCode, Equal(http.StatusOK))) 40 | }) 41 | }) 42 | 43 | Context("HTTPS", func() { 44 | It("return http.StatusOK for a package that does exist", func() { 45 | client := acceptance.CreateTLSClient("../ca_cert") 46 | 47 | request, e := httputil.NewPutRequest("https://internal.127.0.0.1.nip.io:4444/packages/myguid", map[string]map[string]io.Reader{ 48 | "package": map[string]io.Reader{"somefilename": CreateZip(map[string]string{"somefile": "lalala\n\n"})}, 49 | }) 50 | Expect(e).NotTo(HaveOccurred()) 51 | 52 | Expect(client.Do(request)).To(WithTransform(GetStatusCode, Equal(201))) 53 | 54 | Expect(client.Get("https://internal.127.0.0.1.nip.io:4444/packages/myguid")). 55 | To(WithTransform(GetStatusCode, Equal(http.StatusOK))) 56 | }) 57 | }) 58 | 59 | }) 60 | }) 61 | -------------------------------------------------------------------------------- /acceptance_test/with-http/config.yml: -------------------------------------------------------------------------------- 1 | buildpacks: 2 | blobstore_type: local 3 | local_config: 4 | path_prefix: /tmp/buildpacks 5 | droplets: 6 | blobstore_type: local 7 | local_config: 8 | path_prefix: /tmp/droplets 9 | packages: 10 | blobstore_type: local 11 | local_config: 12 | path_prefix: /tmp/packages 13 | app_stash: 14 | blobstore_type: local 15 | local_config: 16 | path_prefix: /tmp/app_stash 17 | logging: 18 | file: /tmp/bits-service.log 19 | syslog: vcap.bits-service 20 | level: debug 21 | public_endpoint: https://public.127.0.0.1.nip.io 22 | private_endpoint: https://internal.127.0.0.1.nip.io 23 | secret: geheim 24 | port: 4444 25 | cert_file: ../cert_file 26 | key_file: ../key_file 27 | signing_users: 28 | - username: the-username 29 | password: the-password 30 | metrics_log_destination: /tmp/bitsgo_metrics.log 31 | enable_http: true 32 | http_port: 8888 33 | -------------------------------------------------------------------------------- /acceptance_test/with-identical-registry-and-public/acceptance_test.go: -------------------------------------------------------------------------------- 1 | package acceptance_test 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | . "github.com/cloudfoundry-incubator/bits-service/acceptance_test" 8 | acceptance "github.com/cloudfoundry-incubator/bits-service/acceptance_test" 9 | "github.com/onsi/ginkgo" 10 | . "github.com/onsi/ginkgo" 11 | "github.com/onsi/gomega" 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | var client = &http.Client{} 16 | 17 | func TestEndToEnd(t *testing.T) { 18 | gomega.RegisterFailHandler(ginkgo.Fail) 19 | CreateFakeEiriniFS() 20 | 21 | acceptance.SetUpAndTearDownServer() 22 | ginkgo.RunSpecs(t, "EndToEnd Identical registry and public hostname") 23 | } 24 | 25 | var _ = Describe("Accessing the bits-service", func() { 26 | Context("when public and registry endpoint use the same hostname", func() { 27 | Context("accessing non-exiting package through public host", func() { 28 | It("gets a status forbidden from the signature verification middleware", func() { 29 | Expect(client.Get("http://public-and-registry.127.0.0.1.nip.io:8888/packages/notexistent")). 30 | To(WithTransform(GetStatusCode, Equal(http.StatusForbidden))) 31 | }) 32 | }) 33 | 34 | Context("accessing OCI /v2 endpoint through registry host", func() { 35 | It("gets an HTTP Status OK", func() { 36 | req, err := http.NewRequest("GET", "http://public-and-registry.127.0.0.1.nip.io:8888/v2/", nil) 37 | Expect(err).ToNot(HaveOccurred()) 38 | 39 | req.SetBasicAuth("the-username", "the-password") 40 | Expect(client.Do(req)).To(WithTransform(GetStatusCode, Equal(http.StatusOK))) 41 | }) 42 | }) 43 | }) 44 | }) 45 | -------------------------------------------------------------------------------- /acceptance_test/with-identical-registry-and-public/config.yml: -------------------------------------------------------------------------------- 1 | buildpacks: 2 | blobstore_type: local 3 | local_config: 4 | path_prefix: /tmp/buildpacks 5 | droplets: 6 | blobstore_type: local 7 | local_config: 8 | path_prefix: /tmp/droplets 9 | packages: 10 | blobstore_type: local 11 | local_config: 12 | path_prefix: /tmp/packages 13 | app_stash: 14 | blobstore_type: local 15 | local_config: 16 | path_prefix: /tmp/app_stash 17 | logging: 18 | file: /tmp/bits-service.log 19 | syslog: vcap.bits-service 20 | level: debug 21 | public_endpoint: https://public-and-registry.127.0.0.1.nip.io 22 | private_endpoint: https://internal.127.0.0.1.nip.io 23 | secret: geheim 24 | port: 4444 25 | cert_file: ../cert_file 26 | key_file: ../key_file 27 | signing_users: 28 | - username: the-username 29 | password: the-password 30 | metrics_log_destination: /tmp/bitsgo_metrics.log 31 | enable_http: true 32 | http_port: 8888 33 | enable_registry: true 34 | registry_endpoint: https://public-and-registry.127.0.0.1.nip.io 35 | rootfs: 36 | blobstore_type: local 37 | local_config: 38 | path_prefix: /tmp/eirinifs 39 | -------------------------------------------------------------------------------- /api-doc/Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | # Middleman 6 | gem 'middleman', '~>4.1.0' 7 | gem 'middleman-autoprefixer', '~> 2.7.0' 8 | gem 'middleman-sprockets', '~> 4.0.0' 9 | gem 'middleman-syntax', '~> 3.0.0' 10 | gem 'redcarpet', '~> 3.3.2' 11 | gem 'rouge', '~> 2.0.5' 12 | -------------------------------------------------------------------------------- /api-doc/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | activesupport (5.0.7) 5 | concurrent-ruby (~> 1.0, >= 1.0.2) 6 | i18n (>= 0.7, < 2) 7 | minitest (~> 5.1) 8 | tzinfo (~> 1.1) 9 | addressable (2.5.2) 10 | public_suffix (>= 2.0.2, < 4.0) 11 | autoprefixer-rails (6.7.7.2) 12 | execjs 13 | backports (3.11.3) 14 | coffee-script (2.4.1) 15 | coffee-script-source 16 | execjs 17 | coffee-script-source (1.12.2) 18 | compass-import-once (1.0.5) 19 | sass (>= 3.2, < 3.5) 20 | concurrent-ruby (1.0.5) 21 | contracts (0.13.0) 22 | dotenv (2.5.0) 23 | erubis (2.7.0) 24 | execjs (2.7.0) 25 | fast_blank (1.0.0) 26 | fastimage (2.1.3) 27 | ffi (1.9.25) 28 | haml (5.0.4) 29 | temple (>= 0.8.0) 30 | tilt 31 | hamster (3.0.0) 32 | concurrent-ruby (~> 1.0) 33 | hashie (3.5.7) 34 | i18n (0.7.0) 35 | kramdown (1.17.0) 36 | listen (3.0.8) 37 | rb-fsevent (~> 0.9, >= 0.9.4) 38 | rb-inotify (~> 0.9, >= 0.9.7) 39 | memoist (0.16.0) 40 | middleman (4.1.14) 41 | coffee-script (~> 2.2) 42 | compass-import-once (= 1.0.5) 43 | haml (>= 4.0.5) 44 | kramdown (~> 1.2) 45 | middleman-cli (= 4.1.14) 46 | middleman-core (= 4.1.14) 47 | sass (>= 3.4.0, < 4.0) 48 | middleman-autoprefixer (2.7.1) 49 | autoprefixer-rails (>= 6.5.2, < 7.0.0) 50 | middleman-core (>= 3.3.3) 51 | middleman-cli (4.1.14) 52 | thor (>= 0.17.0, < 2.0) 53 | middleman-core (4.1.14) 54 | activesupport (>= 4.2, < 5.1) 55 | addressable (~> 2.3) 56 | backports (~> 3.6) 57 | bundler (~> 1.1) 58 | contracts (~> 0.13.0) 59 | dotenv 60 | erubis 61 | execjs (~> 2.0) 62 | fast_blank 63 | fastimage (~> 2.0) 64 | hamster (~> 3.0) 65 | hashie (~> 3.4) 66 | i18n (~> 0.7.0) 67 | listen (~> 3.0.0) 68 | memoist (~> 0.14) 69 | padrino-helpers (~> 0.13.0) 70 | parallel 71 | rack (>= 1.4.5, < 3) 72 | sass (>= 3.4) 73 | servolux 74 | tilt (~> 2.0) 75 | uglifier (~> 3.0) 76 | middleman-sprockets (4.0.0) 77 | middleman-core (~> 4.0) 78 | sprockets (>= 3.0) 79 | middleman-syntax (3.0.0) 80 | middleman-core (>= 3.2) 81 | rouge (~> 2.0) 82 | minitest (5.11.3) 83 | padrino-helpers (0.13.3.4) 84 | i18n (~> 0.6, >= 0.6.7) 85 | padrino-support (= 0.13.3.4) 86 | tilt (>= 1.4.1, < 3) 87 | padrino-support (0.13.3.4) 88 | activesupport (>= 3.1) 89 | parallel (1.12.1) 90 | public_suffix (3.0.2) 91 | rack (2.0.6) 92 | rb-fsevent (0.10.3) 93 | rb-inotify (0.9.10) 94 | ffi (>= 0.5.0, < 2) 95 | redcarpet (3.3.4) 96 | rouge (2.0.7) 97 | sass (3.4.25) 98 | servolux (0.13.0) 99 | sprockets (3.7.2) 100 | concurrent-ruby (~> 1.0) 101 | rack (> 1, < 3) 102 | temple (0.8.0) 103 | thor (0.20.0) 104 | thread_safe (0.3.6) 105 | tilt (2.0.8) 106 | tzinfo (1.2.5) 107 | thread_safe (~> 0.1) 108 | uglifier (3.2.0) 109 | execjs (>= 0.3.0, < 3) 110 | 111 | PLATFORMS 112 | ruby 113 | 114 | DEPENDENCIES 115 | middleman (~> 4.1.0) 116 | middleman-autoprefixer (~> 2.7.0) 117 | middleman-sprockets (~> 4.0.0) 118 | middleman-syntax (~> 3.0.0) 119 | redcarpet (~> 3.3.2) 120 | rouge (~> 2.0.5) 121 | 122 | BUNDLED WITH 123 | 1.15.4 124 | -------------------------------------------------------------------------------- /api-doc/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2008-2013 Concur Technologies, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may 4 | not use this file except in compliance with the License. You may obtain 5 | a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | License for the specific language governing permissions and limitations 13 | under the License. -------------------------------------------------------------------------------- /api-doc/README.md: -------------------------------------------------------------------------------- 1 | # Bits-Service API docs 2 | 3 | This documentation is built using [slate](https://github.com/lord/slate). 4 | 5 | Changes are made in the `source` directory. 6 | 7 | To see changes locally, run: 8 | 9 | ``` 10 | bundle exec middleman server 11 | ``` 12 | 13 | To publish changes to the documentation, run the `deploy.sh` script. 14 | 15 | Documentation is published on https://cloudfoundry-incubator.github.io/bits-service. 16 | -------------------------------------------------------------------------------- /api-doc/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Markdown 4 | set :markdown_engine, :redcarpet 5 | set :markdown, 6 | fenced_code_blocks: true, 7 | smartypants: true, 8 | disable_indented_code_blocks: true, 9 | prettify: true, 10 | tables: true, 11 | with_toc_data: true, 12 | no_intra_emphasis: true 13 | 14 | # Assets 15 | set :css_dir, 'stylesheets' 16 | set :js_dir, 'javascripts' 17 | set :images_dir, 'images' 18 | set :fonts_dir, 'fonts' 19 | 20 | # Activate the syntax highlighter 21 | activate :syntax 22 | 23 | activate :sprockets 24 | 25 | activate :autoprefixer do |config| 26 | config.browsers = ['last 2 version', 'Firefox ESR'] 27 | config.cascade = false 28 | config.inline = true 29 | end 30 | 31 | # Github pages require relative links 32 | activate :relative_assets 33 | set :relative_links, true 34 | 35 | # Build Configuration 36 | configure :build do 37 | # If you're having trouble with Middleman hanging, commenting 38 | # out the following two lines has been known to help 39 | activate :minify_css 40 | activate :minify_javascript 41 | # activate :relative_assets 42 | # activate :asset_hash 43 | # activate :gzip 44 | end 45 | 46 | # Deploy Configuration 47 | # If you want Middleman to listen on a different port, you can set that below 48 | set :port, 4567 49 | -------------------------------------------------------------------------------- /api-doc/font-selection.json: -------------------------------------------------------------------------------- 1 | { 2 | "IcoMoonType": "selection", 3 | "icons": [ 4 | { 5 | "icon": { 6 | "paths": [ 7 | "M438.857 73.143q119.429 0 220.286 58.857t159.714 159.714 58.857 220.286-58.857 220.286-159.714 159.714-220.286 58.857-220.286-58.857-159.714-159.714-58.857-220.286 58.857-220.286 159.714-159.714 220.286-58.857zM512 785.714v-108.571q0-8-5.143-13.429t-12.571-5.429h-109.714q-7.429 0-13.143 5.714t-5.714 13.143v108.571q0 7.429 5.714 13.143t13.143 5.714h109.714q7.429 0 12.571-5.429t5.143-13.429zM510.857 589.143l10.286-354.857q0-6.857-5.714-10.286-5.714-4.571-13.714-4.571h-125.714q-8 0-13.714 4.571-5.714 3.429-5.714 10.286l9.714 354.857q0 5.714 5.714 10t13.714 4.286h105.714q8 0 13.429-4.286t6-10z" 8 | ], 9 | "attrs": [], 10 | "isMulticolor": false, 11 | "tags": [ 12 | "exclamation-circle" 13 | ], 14 | "defaultCode": 61546, 15 | "grid": 14 16 | }, 17 | "attrs": [], 18 | "properties": { 19 | "id": 100, 20 | "order": 4, 21 | "prevSize": 28, 22 | "code": 58880, 23 | "name": "exclamation-sign", 24 | "ligatures": "" 25 | }, 26 | "setIdx": 0, 27 | "iconIdx": 0 28 | }, 29 | { 30 | "icon": { 31 | "paths": [ 32 | "M585.143 786.286v-91.429q0-8-5.143-13.143t-13.143-5.143h-54.857v-292.571q0-8-5.143-13.143t-13.143-5.143h-182.857q-8 0-13.143 5.143t-5.143 13.143v91.429q0 8 5.143 13.143t13.143 5.143h54.857v182.857h-54.857q-8 0-13.143 5.143t-5.143 13.143v91.429q0 8 5.143 13.143t13.143 5.143h256q8 0 13.143-5.143t5.143-13.143zM512 274.286v-91.429q0-8-5.143-13.143t-13.143-5.143h-109.714q-8 0-13.143 5.143t-5.143 13.143v91.429q0 8 5.143 13.143t13.143 5.143h109.714q8 0 13.143-5.143t5.143-13.143zM877.714 512q0 119.429-58.857 220.286t-159.714 159.714-220.286 58.857-220.286-58.857-159.714-159.714-58.857-220.286 58.857-220.286 159.714-159.714 220.286-58.857 220.286 58.857 159.714 159.714 58.857 220.286z" 33 | ], 34 | "attrs": [], 35 | "isMulticolor": false, 36 | "tags": [ 37 | "info-circle" 38 | ], 39 | "defaultCode": 61530, 40 | "grid": 14 41 | }, 42 | "attrs": [], 43 | "properties": { 44 | "id": 85, 45 | "order": 3, 46 | "name": "info-sign", 47 | "prevSize": 28, 48 | "code": 58882 49 | }, 50 | "setIdx": 0, 51 | "iconIdx": 2 52 | }, 53 | { 54 | "icon": { 55 | "paths": [ 56 | "M733.714 419.429q0-16-10.286-26.286l-52-51.429q-10.857-10.857-25.714-10.857t-25.714 10.857l-233.143 232.571-129.143-129.143q-10.857-10.857-25.714-10.857t-25.714 10.857l-52 51.429q-10.286 10.286-10.286 26.286 0 15.429 10.286 25.714l206.857 206.857q10.857 10.857 25.714 10.857 15.429 0 26.286-10.857l310.286-310.286q10.286-10.286 10.286-25.714zM877.714 512q0 119.429-58.857 220.286t-159.714 159.714-220.286 58.857-220.286-58.857-159.714-159.714-58.857-220.286 58.857-220.286 159.714-159.714 220.286-58.857 220.286 58.857 159.714 159.714 58.857 220.286z" 57 | ], 58 | "attrs": [], 59 | "isMulticolor": false, 60 | "tags": [ 61 | "check-circle" 62 | ], 63 | "defaultCode": 61528, 64 | "grid": 14 65 | }, 66 | "attrs": [], 67 | "properties": { 68 | "id": 83, 69 | "order": 9, 70 | "prevSize": 28, 71 | "code": 58886, 72 | "name": "ok-sign" 73 | }, 74 | "setIdx": 0, 75 | "iconIdx": 6 76 | }, 77 | { 78 | "icon": { 79 | "paths": [ 80 | "M658.286 475.429q0-105.714-75.143-180.857t-180.857-75.143-180.857 75.143-75.143 180.857 75.143 180.857 180.857 75.143 180.857-75.143 75.143-180.857zM950.857 950.857q0 29.714-21.714 51.429t-51.429 21.714q-30.857 0-51.429-21.714l-196-195.429q-102.286 70.857-228 70.857-81.714 0-156.286-31.714t-128.571-85.714-85.714-128.571-31.714-156.286 31.714-156.286 85.714-128.571 128.571-85.714 156.286-31.714 156.286 31.714 128.571 85.714 85.714 128.571 31.714 156.286q0 125.714-70.857 228l196 196q21.143 21.143 21.143 51.429z" 81 | ], 82 | "width": 951, 83 | "attrs": [], 84 | "isMulticolor": false, 85 | "tags": [ 86 | "search" 87 | ], 88 | "defaultCode": 61442, 89 | "grid": 14 90 | }, 91 | "attrs": [], 92 | "properties": { 93 | "id": 2, 94 | "order": 1, 95 | "prevSize": 28, 96 | "code": 58887, 97 | "name": "icon-search" 98 | }, 99 | "setIdx": 0, 100 | "iconIdx": 7 101 | } 102 | ], 103 | "height": 1024, 104 | "metadata": { 105 | "name": "slate", 106 | "license": "SIL OFL 1.1" 107 | }, 108 | "preferences": { 109 | "showGlyphs": true, 110 | "showQuickUse": true, 111 | "showQuickUse2": true, 112 | "showSVGs": true, 113 | "fontPref": { 114 | "prefix": "icon-", 115 | "metadata": { 116 | "fontFamily": "slate", 117 | "majorVersion": 1, 118 | "minorVersion": 0, 119 | "description": "Based on FontAwesome", 120 | "license": "SIL OFL 1.1" 121 | }, 122 | "metrics": { 123 | "emSize": 1024, 124 | "baseline": 6.25, 125 | "whitespace": 50 126 | }, 127 | "resetPoint": 58880, 128 | "showSelector": false, 129 | "selector": "class", 130 | "classSelector": ".icon", 131 | "showMetrics": false, 132 | "showMetadata": true, 133 | "showVersion": true, 134 | "ie7": false 135 | }, 136 | "imagePref": { 137 | "prefix": "icon-", 138 | "png": true, 139 | "useClassSelector": true, 140 | "color": 4473924, 141 | "bgColor": 16777215 142 | }, 143 | "historySize": 100, 144 | "showCodes": true, 145 | "gridSize": 16, 146 | "showLiga": false 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /api-doc/source/fonts/slate.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/api-doc/source/fonts/slate.eot -------------------------------------------------------------------------------- /api-doc/source/fonts/slate.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Generated by IcoMoon 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /api-doc/source/fonts/slate.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/api-doc/source/fonts/slate.ttf -------------------------------------------------------------------------------- /api-doc/source/fonts/slate.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/api-doc/source/fonts/slate.woff -------------------------------------------------------------------------------- /api-doc/source/fonts/slate.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/api-doc/source/fonts/slate.woff2 -------------------------------------------------------------------------------- /api-doc/source/images/api_doc_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/api-doc/source/images/api_doc_logo.png -------------------------------------------------------------------------------- /api-doc/source/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/api-doc/source/images/logo.png -------------------------------------------------------------------------------- /api-doc/source/images/navbar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/api-doc/source/images/navbar.png -------------------------------------------------------------------------------- /api-doc/source/javascripts/all.js: -------------------------------------------------------------------------------- 1 | //= require ./lib/_energize 2 | //= require ./app/_lang 3 | //= require ./app/_search 4 | //= require ./app/_toc 5 | -------------------------------------------------------------------------------- /api-doc/source/javascripts/all_nosearch.js: -------------------------------------------------------------------------------- 1 | //= require ./lib/_energize 2 | //= require ./app/_lang 3 | //= require ./app/_toc 4 | -------------------------------------------------------------------------------- /api-doc/source/javascripts/app/_lang.js: -------------------------------------------------------------------------------- 1 | //= require ../lib/_jquery 2 | 3 | /* 4 | Copyright 2008-2013 Concur Technologies, Inc. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | not use this file except in compliance with the License. You may obtain 8 | a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | License for the specific language governing permissions and limitations 16 | under the License. 17 | */ 18 | (function (global) { 19 | 'use strict'; 20 | 21 | var languages = []; 22 | 23 | global.setupLanguages = setupLanguages; 24 | global.activateLanguage = activateLanguage; 25 | 26 | function activateLanguage(language) { 27 | if (!language) return; 28 | if (language === "") return; 29 | 30 | $(".lang-selector a").removeClass('active'); 31 | $(".lang-selector a[data-language-name='" + language + "']").addClass('active'); 32 | for (var i=0; i < languages.length; i++) { 33 | $(".highlight." + languages[i]).hide(); 34 | $(".lang-specific." + languages[i]).hide(); 35 | } 36 | $(".highlight." + language).show(); 37 | $(".lang-specific." + language).show(); 38 | 39 | global.toc.calculateHeights(); 40 | 41 | // scroll to the new location of the position 42 | if ($(window.location.hash).get(0)) { 43 | $(window.location.hash).get(0).scrollIntoView(true); 44 | } 45 | } 46 | 47 | // parseURL and stringifyURL are from https://github.com/sindresorhus/query-string 48 | // MIT licensed 49 | // https://github.com/sindresorhus/query-string/blob/7bee64c16f2da1a326579e96977b9227bf6da9e6/license 50 | function parseURL(str) { 51 | if (typeof str !== 'string') { 52 | return {}; 53 | } 54 | 55 | str = str.trim().replace(/^(\?|#|&)/, ''); 56 | 57 | if (!str) { 58 | return {}; 59 | } 60 | 61 | return str.split('&').reduce(function (ret, param) { 62 | var parts = param.replace(/\+/g, ' ').split('='); 63 | var key = parts[0]; 64 | var val = parts[1]; 65 | 66 | key = decodeURIComponent(key); 67 | // missing `=` should be `null`: 68 | // http://w3.org/TR/2012/WD-url-20120524/#collect-url-parameters 69 | val = val === undefined ? null : decodeURIComponent(val); 70 | 71 | if (!ret.hasOwnProperty(key)) { 72 | ret[key] = val; 73 | } else if (Array.isArray(ret[key])) { 74 | ret[key].push(val); 75 | } else { 76 | ret[key] = [ret[key], val]; 77 | } 78 | 79 | return ret; 80 | }, {}); 81 | }; 82 | 83 | function stringifyURL(obj) { 84 | return obj ? Object.keys(obj).sort().map(function (key) { 85 | var val = obj[key]; 86 | 87 | if (Array.isArray(val)) { 88 | return val.sort().map(function (val2) { 89 | return encodeURIComponent(key) + '=' + encodeURIComponent(val2); 90 | }).join('&'); 91 | } 92 | 93 | return encodeURIComponent(key) + '=' + encodeURIComponent(val); 94 | }).join('&') : ''; 95 | }; 96 | 97 | // gets the language set in the query string 98 | function getLanguageFromQueryString() { 99 | if (location.search.length >= 1) { 100 | var language = parseURL(location.search).language 101 | if (language) { 102 | return language; 103 | } else if (jQuery.inArray(location.search.substr(1), languages) != -1) { 104 | return location.search.substr(1); 105 | } 106 | } 107 | 108 | return false; 109 | } 110 | 111 | // returns a new query string with the new language in it 112 | function generateNewQueryString(language) { 113 | var url = parseURL(location.search); 114 | if (url.language) { 115 | url.language = language; 116 | return stringifyURL(url); 117 | } 118 | return language; 119 | } 120 | 121 | // if a button is clicked, add the state to the history 122 | function pushURL(language) { 123 | if (!history) { return; } 124 | var hash = window.location.hash; 125 | if (hash) { 126 | hash = hash.replace(/^#+/, ''); 127 | } 128 | history.pushState({}, '', '?' + generateNewQueryString(language) + '#' + hash); 129 | 130 | // save language as next default 131 | localStorage.setItem("language", language); 132 | } 133 | 134 | function setupLanguages(l) { 135 | var defaultLanguage = localStorage.getItem("language"); 136 | 137 | languages = l; 138 | 139 | var presetLanguage = getLanguageFromQueryString(); 140 | if (presetLanguage) { 141 | // the language is in the URL, so use that language! 142 | activateLanguage(presetLanguage); 143 | 144 | localStorage.setItem("language", presetLanguage); 145 | } else if ((defaultLanguage !== null) && (jQuery.inArray(defaultLanguage, languages) != -1)) { 146 | // the language was the last selected one saved in localstorage, so use that language! 147 | activateLanguage(defaultLanguage); 148 | } else { 149 | // no language selected, so use the default 150 | activateLanguage(languages[0]); 151 | } 152 | } 153 | 154 | // if we click on a language tab, activate that language 155 | $(function() { 156 | $(".lang-selector a").on("click", function() { 157 | var language = $(this).data("language-name"); 158 | pushURL(language); 159 | activateLanguage(language); 160 | return false; 161 | }); 162 | window.onpopstate = function() { 163 | activateLanguage(getLanguageFromQueryString()); 164 | }; 165 | }); 166 | })(window); 167 | -------------------------------------------------------------------------------- /api-doc/source/javascripts/app/_search.js: -------------------------------------------------------------------------------- 1 | //= require ../lib/_lunr 2 | //= require ../lib/_jquery 3 | //= require ../lib/_jquery.highlight 4 | (function () { 5 | 'use strict'; 6 | 7 | var content, searchResults; 8 | var highlightOpts = { element: 'span', className: 'search-highlight' }; 9 | 10 | var index = new lunr.Index(); 11 | 12 | index.ref('id'); 13 | index.field('title', { boost: 10 }); 14 | index.field('body'); 15 | index.pipeline.add(lunr.trimmer, lunr.stopWordFilter); 16 | 17 | $(populate); 18 | $(bind); 19 | 20 | function populate() { 21 | $('h1, h2').each(function() { 22 | var title = $(this); 23 | var body = title.nextUntil('h1, h2'); 24 | index.add({ 25 | id: title.prop('id'), 26 | title: title.text(), 27 | body: body.text() 28 | }); 29 | }); 30 | } 31 | 32 | function bind() { 33 | content = $('.content'); 34 | searchResults = $('.search-results'); 35 | 36 | $('#input-search').on('keyup', search); 37 | } 38 | 39 | function search(event) { 40 | unhighlight(); 41 | searchResults.addClass('visible'); 42 | 43 | // ESC clears the field 44 | if (event.keyCode === 27) this.value = ''; 45 | 46 | if (this.value) { 47 | var results = index.search(this.value).filter(function(r) { 48 | return r.score > 0.0001; 49 | }); 50 | 51 | if (results.length) { 52 | searchResults.empty(); 53 | $.each(results, function (index, result) { 54 | var elem = document.getElementById(result.ref); 55 | searchResults.append("

  • " + $(elem).text() + "
    • "); 56 | }); 57 | highlight.call(this); 58 | } else { 59 | searchResults.html('
  • '); 60 | $('.search-results li').text('No Results Found for "' + this.value + '"'); 61 | } 62 | } else { 63 | unhighlight(); 64 | searchResults.removeClass('visible'); 65 | } 66 | } 67 | 68 | function highlight() { 69 | if (this.value) content.highlight(this.value, highlightOpts); 70 | } 71 | 72 | function unhighlight() { 73 | content.unhighlight(highlightOpts); 74 | } 75 | })(); 76 | -------------------------------------------------------------------------------- /api-doc/source/javascripts/app/_toc.js: -------------------------------------------------------------------------------- 1 | //= require ../lib/_jquery 2 | //= require ../lib/_jquery_ui 3 | //= require ../lib/_jquery.tocify 4 | //= require ../lib/_imagesloaded.min 5 | (function (global) { 6 | 'use strict'; 7 | 8 | var closeToc = function() { 9 | $(".tocify-wrapper").removeClass('open'); 10 | $("#nav-button").removeClass('open'); 11 | }; 12 | 13 | var makeToc = function() { 14 | global.toc = $("#toc").tocify({ 15 | selectors: 'h1, h2', 16 | extendPage: false, 17 | theme: 'none', 18 | smoothScroll: false, 19 | showEffectSpeed: 0, 20 | hideEffectSpeed: 180, 21 | ignoreSelector: '.toc-ignore', 22 | highlightOffset: 60, 23 | scrollTo: -1, 24 | scrollHistory: true, 25 | hashGenerator: function (text, element) { 26 | return element.prop('id'); 27 | } 28 | }).data('toc-tocify'); 29 | 30 | $("#nav-button").click(function() { 31 | $(".tocify-wrapper").toggleClass('open'); 32 | $("#nav-button").toggleClass('open'); 33 | return false; 34 | }); 35 | 36 | $(".page-wrapper").click(closeToc); 37 | $(".tocify-item").click(closeToc); 38 | }; 39 | 40 | // Hack to make already open sections to start opened, 41 | // instead of displaying an ugly animation 42 | function animate() { 43 | setTimeout(function() { 44 | toc.setOption('showEffectSpeed', 180); 45 | }, 50); 46 | } 47 | 48 | $(function() { 49 | makeToc(); 50 | animate(); 51 | setupLanguages($('body').data('languages')); 52 | $('.content').imagesLoaded( function() { 53 | global.toc.calculateHeights(); 54 | }); 55 | }); 56 | })(window); 57 | 58 | -------------------------------------------------------------------------------- /api-doc/source/javascripts/lib/_jquery.highlight.js: -------------------------------------------------------------------------------- 1 | /* 2 | * jQuery Highlight plugin 3 | * 4 | * Based on highlight v3 by Johann Burkard 5 | * http://johannburkard.de/blog/programming/javascript/highlight-javascript-text-higlighting-jquery-plugin.html 6 | * 7 | * Code a little bit refactored and cleaned (in my humble opinion). 8 | * Most important changes: 9 | * - has an option to highlight only entire words (wordsOnly - false by default), 10 | * - has an option to be case sensitive (caseSensitive - false by default) 11 | * - highlight element tag and class names can be specified in options 12 | * 13 | * Usage: 14 | * // wrap every occurrance of text 'lorem' in content 15 | * // with (default options) 16 | * $('#content').highlight('lorem'); 17 | * 18 | * // search for and highlight more terms at once 19 | * // so you can save some time on traversing DOM 20 | * $('#content').highlight(['lorem', 'ipsum']); 21 | * $('#content').highlight('lorem ipsum'); 22 | * 23 | * // search only for entire word 'lorem' 24 | * $('#content').highlight('lorem', { wordsOnly: true }); 25 | * 26 | * // don't ignore case during search of term 'lorem' 27 | * $('#content').highlight('lorem', { caseSensitive: true }); 28 | * 29 | * // wrap every occurrance of term 'ipsum' in content 30 | * // with 31 | * $('#content').highlight('ipsum', { element: 'em', className: 'important' }); 32 | * 33 | * // remove default highlight 34 | * $('#content').unhighlight(); 35 | * 36 | * // remove custom highlight 37 | * $('#content').unhighlight({ element: 'em', className: 'important' }); 38 | * 39 | * 40 | * Copyright (c) 2009 Bartek Szopka 41 | * 42 | * Licensed under MIT license. 43 | * 44 | */ 45 | 46 | jQuery.extend({ 47 | highlight: function (node, re, nodeName, className) { 48 | if (node.nodeType === 3) { 49 | var match = node.data.match(re); 50 | if (match) { 51 | var highlight = document.createElement(nodeName || 'span'); 52 | highlight.className = className || 'highlight'; 53 | var wordNode = node.splitText(match.index); 54 | wordNode.splitText(match[0].length); 55 | var wordClone = wordNode.cloneNode(true); 56 | highlight.appendChild(wordClone); 57 | wordNode.parentNode.replaceChild(highlight, wordNode); 58 | return 1; //skip added node in parent 59 | } 60 | } else if ((node.nodeType === 1 && node.childNodes) && // only element nodes that have children 61 | !/(script|style)/i.test(node.tagName) && // ignore script and style nodes 62 | !(node.tagName === nodeName.toUpperCase() && node.className === className)) { // skip if already highlighted 63 | for (var i = 0; i < node.childNodes.length; i++) { 64 | i += jQuery.highlight(node.childNodes[i], re, nodeName, className); 65 | } 66 | } 67 | return 0; 68 | } 69 | }); 70 | 71 | jQuery.fn.unhighlight = function (options) { 72 | var settings = { className: 'highlight', element: 'span' }; 73 | jQuery.extend(settings, options); 74 | 75 | return this.find(settings.element + "." + settings.className).each(function () { 76 | var parent = this.parentNode; 77 | parent.replaceChild(this.firstChild, this); 78 | parent.normalize(); 79 | }).end(); 80 | }; 81 | 82 | jQuery.fn.highlight = function (words, options) { 83 | var settings = { className: 'highlight', element: 'span', caseSensitive: false, wordsOnly: false }; 84 | jQuery.extend(settings, options); 85 | 86 | if (words.constructor === String) { 87 | words = [words]; 88 | } 89 | words = jQuery.grep(words, function(word, i){ 90 | return word != ''; 91 | }); 92 | words = jQuery.map(words, function(word, i) { 93 | return word.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); 94 | }); 95 | if (words.length == 0) { return this; }; 96 | 97 | var flag = settings.caseSensitive ? "" : "i"; 98 | var pattern = "(" + words.join("|") + ")"; 99 | if (settings.wordsOnly) { 100 | pattern = "\\b" + pattern + "\\b"; 101 | } 102 | var re = new RegExp(pattern, flag); 103 | 104 | return this.each(function () { 105 | jQuery.highlight(this, re, settings.element, settings.className); 106 | }); 107 | }; 108 | 109 | -------------------------------------------------------------------------------- /api-doc/source/layouts/layout.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | Copyright 2008-2013 Concur Technologies, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | not use this file except in compliance with the License. You may obtain 6 | a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | License for the specific language governing permissions and limitations 14 | under the License. 15 | %> 16 | <% language_tabs = current_page.data.language_tabs || [] %> 17 | 18 | 19 | 20 | 21 | 22 | 23 | <%= current_page.data.title || "API Documentation" %> 24 | 25 | 28 | <%= stylesheet_link_tag :screen, media: :screen %> 29 | <%= stylesheet_link_tag :print, media: :print %> 30 | <% if current_page.data.search %> 31 | <%= javascript_include_tag "all" %> 32 | <% else %> 33 | <%= javascript_include_tag "all_nosearch" %> 34 | <% end %> 35 | 36 | 37 | 38 | 39 | 40 | NAV 41 | <%= image_tag('navbar.png') %> 42 | 43 | 44 |
    45 | <%= image_tag "api_doc_logo.png" %> 46 | <% if language_tabs.any? %> 47 |
    48 | <% language_tabs.each do |lang| %> 49 | <% if lang.is_a? Hash %> 50 | <%= lang.values.first %> 51 | <% else %> 52 | <%= lang %> 53 | <% end %> 54 | <% end %> 55 |
    56 | <% end %> 57 | <% if current_page.data.search %> 58 |
    59 | 60 |
    61 |
      62 | <% end %> 63 |
      64 |
      65 | <% if current_page.data.toc_footers %> 66 |
        67 | <% current_page.data.toc_footers.each do |footer| %> 68 |
      • <%= footer %>
        • 69 | <% end %> 70 |
      71 | <% end %> 72 |
      73 |
      74 |
      75 |
      76 | <%= yield %> 77 | <% current_page.data.includes && current_page.data.includes.each do |include| %> 78 | <%= partial "includes/#{include}" %> 79 | <% end %> 80 |
      81 |
      82 | <% if language_tabs.any? %> 83 |
      84 | <% language_tabs.each do |lang| %> 85 | <% if lang.is_a? Hash %> 86 | <%= lang.values.first %> 87 | <% else %> 88 | <%= lang %> 89 | <% end %> 90 | <% end %> 91 |
      92 | <% end %> 93 |
      94 |
      95 | 96 | 97 | -------------------------------------------------------------------------------- /api-doc/source/stylesheets/_icon-font.scss: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'slate'; 3 | src:font-url('slate.eot?-syv14m'); 4 | src:font-url('slate.eot?#iefix-syv14m') format('embedded-opentype'), 5 | font-url('slate.woff2?-syv14m') format('woff2'), 6 | font-url('slate.woff?-syv14m') format('woff'), 7 | font-url('slate.ttf?-syv14m') format('truetype'), 8 | font-url('slate.svg?-syv14m#slate') format('svg'); 9 | font-weight: normal; 10 | font-style: normal; 11 | } 12 | 13 | %icon { 14 | font-family: 'slate'; 15 | speak: none; 16 | font-style: normal; 17 | font-weight: normal; 18 | font-variant: normal; 19 | text-transform: none; 20 | line-height: 1; 21 | } 22 | 23 | %icon-exclamation-sign { 24 | @extend %icon; 25 | content: "\e600"; 26 | } 27 | %icon-info-sign { 28 | @extend %icon; 29 | content: "\e602"; 30 | } 31 | %icon-ok-sign { 32 | @extend %icon; 33 | content: "\e606"; 34 | } 35 | %icon-search { 36 | @extend %icon; 37 | content: "\e607"; 38 | } 39 | -------------------------------------------------------------------------------- /api-doc/source/stylesheets/_variables.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2008-2013 Concur Technologies, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | not use this file except in compliance with the License. You may obtain 6 | a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | License for the specific language governing permissions and limitations 14 | under the License. 15 | */ 16 | 17 | 18 | //////////////////////////////////////////////////////////////////////////////// 19 | // CUSTOMIZE SLATE 20 | //////////////////////////////////////////////////////////////////////////////// 21 | // Use these settings to help adjust the appearance of Slate 22 | 23 | 24 | // BACKGROUND COLORS 25 | //////////////////// 26 | $nav-bg: #393939 !default; 27 | $examples-bg: #393939 !default; 28 | $code-bg: #292929 !default; 29 | $code-annotation-bg: #1c1c1c !default; 30 | $nav-subitem-bg: #262626 !default; 31 | $nav-active-bg: #2467af !default; 32 | $lang-select-border: #000 !default; 33 | $lang-select-bg: #222 !default; 34 | $lang-select-active-bg: $examples-bg !default; // feel free to change this to blue or something 35 | $lang-select-pressed-bg: #111 !default; // color of language tab bg when mouse is pressed 36 | $main-bg: #eaf2f6 !default; 37 | $aside-notice-bg: #8fbcd4 !default; 38 | $aside-warning-bg: #c97a7e !default; 39 | $aside-success-bg: #6ac174 !default; 40 | $search-notice-bg: #c97a7e !default; 41 | 42 | 43 | // TEXT COLORS 44 | //////////////////// 45 | $main-text: #333 !default; // main content text color 46 | $nav-text: #fff !default; 47 | $nav-active-text: #fff !default; 48 | $lang-select-text: #fff !default; // color of unselected language tab text 49 | $lang-select-active-text: #fff !default; // color of selected language tab text 50 | $lang-select-pressed-text: #fff !default; // color of language tab text when mouse is pressed 51 | 52 | 53 | // SIZES 54 | //////////////////// 55 | $nav-width: 230px !default; // width of the navbar 56 | $examples-width: 50% !default; // portion of the screen taken up by code examples 57 | $logo-margin: 20px !default; // margin between nav items and logo, ignored if search is active 58 | $main-padding: 28px !default; // padding to left and right of content & examples 59 | $nav-padding: 15px !default; // padding to left and right of navbar 60 | $nav-v-padding: 10px !default; // padding used vertically around search boxes and results 61 | $nav-indent: 10px !default; // extra padding for ToC subitems 62 | $code-annotation-padding: 13px !default; // padding inside code annotations 63 | $h1-margin-bottom: 21px !default; // padding under the largest header tags 64 | $tablet-width: 930px !default; // min width before reverting to tablet size 65 | $phone-width: $tablet-width - $nav-width !default; // min width before reverting to mobile size 66 | 67 | 68 | // FONTS 69 | //////////////////// 70 | %default-font { 71 | font-family: "Helvetica Neue", Helvetica, Arial, "Microsoft Yahei","微软雅黑", STXihei, "华文细黑", sans-serif; 72 | font-size: 13px; 73 | } 74 | 75 | %header-font { 76 | @extend %default-font; 77 | font-weight: bold; 78 | } 79 | 80 | %code-font { 81 | font-family: Consolas, Menlo, Monaco, "Lucida Console", "Liberation Mono", "DejaVu Sans Mono", "Bitstream Vera Sans Mono", "Courier New", monospace, serif; 82 | font-size: 12px; 83 | line-height: 1.5; 84 | } 85 | 86 | 87 | // OTHER 88 | //////////////////// 89 | $nav-active-shadow: #000 !default; 90 | $nav-footer-border-color: #666 !default; 91 | $nav-embossed-border-top: #000 !default; 92 | $nav-embossed-border-bottom: #939393 !default; 93 | $main-embossed-text-shadow: 0px 1px 0px #fff !default; 94 | $search-box-border-color: #666 !default; 95 | 96 | 97 | //////////////////////////////////////////////////////////////////////////////// 98 | // INTERNAL 99 | //////////////////////////////////////////////////////////////////////////////// 100 | // These settings are probably best left alone. 101 | 102 | %break-words { 103 | word-break: break-all; 104 | hyphens: auto; 105 | } 106 | -------------------------------------------------------------------------------- /api-doc/source/stylesheets/print.css.scss: -------------------------------------------------------------------------------- 1 | @charset "utf-8"; 2 | @import 'normalize'; 3 | @import 'variables'; 4 | @import 'icon-font'; 5 | 6 | /* 7 | Copyright 2008-2013 Concur Technologies, Inc. 8 | 9 | Licensed under the Apache License, Version 2.0 (the "License"); you may 10 | not use this file except in compliance with the License. You may obtain 11 | a copy of the License at 12 | 13 | http://www.apache.org/licenses/LICENSE-2.0 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 18 | License for the specific language governing permissions and limitations 19 | under the License. 20 | */ 21 | 22 | $print-color: #999; 23 | $print-color-light: #ccc; 24 | $print-font-size: 12px; 25 | 26 | body { 27 | @extend %default-font; 28 | } 29 | 30 | .tocify, .toc-footer, .lang-selector, .search, #nav-button { 31 | display: none; 32 | } 33 | 34 | .tocify-wrapper>img { 35 | margin: 0 auto; 36 | display: block; 37 | } 38 | 39 | .content { 40 | font-size: 12px; 41 | 42 | pre, code { 43 | @extend %code-font; 44 | @extend %break-words; 45 | border: 1px solid $print-color; 46 | border-radius: 5px; 47 | font-size: 0.8em; 48 | } 49 | 50 | pre { 51 | code { 52 | border: 0; 53 | } 54 | } 55 | 56 | pre { 57 | padding: 1.3em; 58 | } 59 | 60 | code { 61 | padding: 0.2em; 62 | } 63 | 64 | table { 65 | border: 1px solid $print-color; 66 | tr { 67 | border-bottom: 1px solid $print-color; 68 | } 69 | td,th { 70 | padding: 0.7em; 71 | } 72 | } 73 | 74 | p { 75 | line-height: 1.5; 76 | } 77 | 78 | a { 79 | text-decoration: none; 80 | color: #000; 81 | } 82 | 83 | h1 { 84 | @extend %header-font; 85 | font-size: 2.5em; 86 | padding-top: 0.5em; 87 | padding-bottom: 0.5em; 88 | margin-top: 1em; 89 | margin-bottom: $h1-margin-bottom; 90 | border: 2px solid $print-color-light; 91 | border-width: 2px 0; 92 | text-align: center; 93 | } 94 | 95 | h2 { 96 | @extend %header-font; 97 | font-size: 1.8em; 98 | margin-top: 2em; 99 | border-top: 2px solid $print-color-light; 100 | padding-top: 0.8em; 101 | } 102 | 103 | h1+h2, h1+div+h2 { 104 | border-top: none; 105 | padding-top: 0; 106 | margin-top: 0; 107 | } 108 | 109 | h3, h4 { 110 | @extend %header-font; 111 | font-size: 0.8em; 112 | margin-top: 1.5em; 113 | margin-bottom: 0.8em; 114 | text-transform: uppercase; 115 | } 116 | 117 | h5, h6 { 118 | text-transform: uppercase; 119 | } 120 | 121 | aside { 122 | padding: 1em; 123 | border: 1px solid $print-color-light; 124 | border-radius: 5px; 125 | margin-top: 1.5em; 126 | margin-bottom: 1.5em; 127 | line-height: 1.6; 128 | } 129 | 130 | aside:before { 131 | vertical-align: middle; 132 | padding-right: 0.5em; 133 | font-size: 14px; 134 | } 135 | 136 | aside.notice:before { 137 | @extend %icon-info-sign; 138 | } 139 | 140 | aside.warning:before { 141 | @extend %icon-exclamation-sign; 142 | } 143 | 144 | aside.success:before { 145 | @extend %icon-ok-sign; 146 | } 147 | } -------------------------------------------------------------------------------- /assets/lots-of-files.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/assets/lots-of-files.zip -------------------------------------------------------------------------------- /assets/test-file.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/assets/test-file.zip -------------------------------------------------------------------------------- /bitsgo_suite_test.go: -------------------------------------------------------------------------------- 1 | package bitsgo_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/onsi/ginkgo" 7 | "github.com/onsi/gomega" 8 | "github.com/petergtz/pegomock" 9 | ) 10 | 11 | func TestBitsgo(t *testing.T) { 12 | gomega.RegisterFailHandler(ginkgo.Fail) 13 | pegomock.RegisterMockFailHandler(ginkgo.Fail) 14 | ginkgo.RunSpecs(t, "bitsgo") 15 | } 16 | 17 | // Declarations for Ginkgo DSL 18 | type Done ginkgo.Done 19 | type Benchmarker ginkgo.Benchmarker 20 | 21 | var GinkgoWriter = ginkgo.GinkgoWriter 22 | var GinkgoRandomSeed = ginkgo.GinkgoRandomSeed 23 | var GinkgoParallelNode = ginkgo.GinkgoParallelNode 24 | var GinkgoT = ginkgo.GinkgoT 25 | var CurrentGinkgoTestDescription = ginkgo.CurrentGinkgoTestDescription 26 | var RunSpecs = ginkgo.RunSpecs 27 | var RunSpecsWithDefaultAndCustomReporters = ginkgo.RunSpecsWithDefaultAndCustomReporters 28 | var RunSpecsWithCustomReporters = ginkgo.RunSpecsWithCustomReporters 29 | var Skip = ginkgo.Skip 30 | var Fail = ginkgo.Fail 31 | var GinkgoRecover = ginkgo.GinkgoRecover 32 | var Describe = ginkgo.Describe 33 | var FDescribe = ginkgo.FDescribe 34 | var PDescribe = ginkgo.PDescribe 35 | var XDescribe = ginkgo.XDescribe 36 | var Context = ginkgo.Context 37 | var FContext = ginkgo.FContext 38 | var PContext = ginkgo.PContext 39 | var XContext = ginkgo.XContext 40 | var It = ginkgo.It 41 | var FIt = ginkgo.FIt 42 | var PIt = ginkgo.PIt 43 | var XIt = ginkgo.XIt 44 | var Specify = ginkgo.Specify 45 | var FSpecify = ginkgo.FSpecify 46 | var PSpecify = ginkgo.PSpecify 47 | var XSpecify = ginkgo.XSpecify 48 | var By = ginkgo.By 49 | var Measure = ginkgo.Measure 50 | var FMeasure = ginkgo.FMeasure 51 | var PMeasure = ginkgo.PMeasure 52 | var XMeasure = ginkgo.XMeasure 53 | var BeforeSuite = ginkgo.BeforeSuite 54 | var AfterSuite = ginkgo.AfterSuite 55 | var SynchronizedBeforeSuite = ginkgo.SynchronizedBeforeSuite 56 | var SynchronizedAfterSuite = ginkgo.SynchronizedAfterSuite 57 | var BeforeEach = ginkgo.BeforeEach 58 | var JustBeforeEach = ginkgo.JustBeforeEach 59 | var AfterEach = ginkgo.AfterEach 60 | 61 | // Declarations for Gomega DSL 62 | var RegisterFailHandler = gomega.RegisterFailHandler 63 | var RegisterTestingT = gomega.RegisterTestingT 64 | var InterceptGomegaFailures = gomega.InterceptGomegaFailures 65 | var Ω = gomega.Ω 66 | var Expect = gomega.Expect 67 | var ExpectWithOffset = gomega.ExpectWithOffset 68 | var Eventually = gomega.Eventually 69 | var EventuallyWithOffset = gomega.EventuallyWithOffset 70 | var Consistently = gomega.Consistently 71 | var ConsistentlyWithOffset = gomega.ConsistentlyWithOffset 72 | var SetDefaultEventuallyTimeout = gomega.SetDefaultEventuallyTimeout 73 | var SetDefaultEventuallyPollingInterval = gomega.SetDefaultEventuallyPollingInterval 74 | var SetDefaultConsistentlyDuration = gomega.SetDefaultConsistentlyDuration 75 | var SetDefaultConsistentlyPollingInterval = gomega.SetDefaultConsistentlyPollingInterval 76 | var NewGomegaWithT = gomega.NewGomegaWithT 77 | 78 | // Declarations for Gomega Matchers 79 | var Equal = gomega.Equal 80 | var BeEquivalentTo = gomega.BeEquivalentTo 81 | var BeIdenticalTo = gomega.BeIdenticalTo 82 | var BeNil = gomega.BeNil 83 | var BeTrue = gomega.BeTrue 84 | var BeFalse = gomega.BeFalse 85 | var HaveOccurred = gomega.HaveOccurred 86 | var Succeed = gomega.Succeed 87 | var MatchError = gomega.MatchError 88 | var BeClosed = gomega.BeClosed 89 | var Receive = gomega.Receive 90 | var BeSent = gomega.BeSent 91 | var MatchRegexp = gomega.MatchRegexp 92 | var ContainSubstring = gomega.ContainSubstring 93 | var HavePrefix = gomega.HavePrefix 94 | var HaveSuffix = gomega.HaveSuffix 95 | var MatchJSON = gomega.MatchJSON 96 | var MatchXML = gomega.MatchXML 97 | var MatchYAML = gomega.MatchYAML 98 | var BeEmpty = gomega.BeEmpty 99 | var HaveLen = gomega.HaveLen 100 | var HaveCap = gomega.HaveCap 101 | var BeZero = gomega.BeZero 102 | var ContainElement = gomega.ContainElement 103 | var ConsistOf = gomega.ConsistOf 104 | var HaveKey = gomega.HaveKey 105 | var HaveKeyWithValue = gomega.HaveKeyWithValue 106 | var BeNumerically = gomega.BeNumerically 107 | var BeTemporally = gomega.BeTemporally 108 | var BeAssignableToTypeOf = gomega.BeAssignableToTypeOf 109 | var Panic = gomega.Panic 110 | var BeAnExistingFile = gomega.BeAnExistingFile 111 | var BeARegularFile = gomega.BeARegularFile 112 | var BeADirectory = gomega.BeADirectory 113 | var And = gomega.And 114 | var SatisfyAll = gomega.SatisfyAll 115 | var Or = gomega.Or 116 | var SatisfyAny = gomega.SatisfyAny 117 | var Not = gomega.Not 118 | var WithTransform = gomega.WithTransform 119 | -------------------------------------------------------------------------------- /blobstore.go: -------------------------------------------------------------------------------- 1 | package bitsgo 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | type NotFoundError struct { 9 | error 10 | MissingKey string 11 | } 12 | 13 | // Deprecated. Use NewNotFoundErrorWithKey 14 | func NewNotFoundError() *NotFoundError { 15 | return &NotFoundError{error: fmt.Errorf("NotFoundError")} 16 | } 17 | 18 | func NewNotFoundErrorWithKey(key string) *NotFoundError { 19 | return &NotFoundError{error: fmt.Errorf("Not found: " + key), MissingKey: key} 20 | } 21 | 22 | // Deprecated. Use NewNotFoundErrorWithKey 23 | func NewNotFoundErrorWithMessage(message string) *NotFoundError { 24 | return &NotFoundError{error: fmt.Errorf(message)} 25 | } 26 | 27 | func IsNotFoundError(e error) bool { 28 | _, notFound := e.(*NotFoundError) 29 | return notFound 30 | } 31 | 32 | type NoSpaceLeftError struct { 33 | error 34 | } 35 | 36 | func NewNoSpaceLeftError() *NoSpaceLeftError { 37 | return &NoSpaceLeftError{fmt.Errorf("NoSpaceLeftError")} 38 | } 39 | 40 | //go:generate pegomock generate --use-experimental-model-gen --package bitsgo_test Blobstore 41 | type Blobstore interface { 42 | Exists(path string) (bool, error) 43 | 44 | // Implementers must return *NotFoundError when the resource cannot be found 45 | GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) 46 | // Implementers must return *NotFoundError when the resource cannot be found 47 | Get(path string) (body io.ReadCloser, err error) 48 | 49 | // Implementers must return *NoSpaceLeftError when there's no space left on device. 50 | Put(path string, src io.ReadSeeker) error 51 | Copy(src, dest string) error 52 | Delete(path string) error 53 | DeleteDir(prefix string) error 54 | } 55 | -------------------------------------------------------------------------------- /blobstores/alibaba/alibaba_blobstore.go: -------------------------------------------------------------------------------- 1 | package alibaba 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | "time" 8 | 9 | "github.com/cloudfoundry-incubator/bits-service" 10 | "github.com/pkg/errors" 11 | 12 | "github.com/aliyun/aliyun-oss-go-sdk/oss" 13 | "github.com/cloudfoundry-incubator/bits-service/blobstores/validate" 14 | "github.com/cloudfoundry-incubator/bits-service/config" 15 | "github.com/cloudfoundry-incubator/bits-service/logger" 16 | ) 17 | 18 | type Blobstore struct { 19 | Client *oss.Client 20 | bucket *oss.Bucket 21 | } 22 | 23 | func NewBlobstore(config config.AlibabaBlobstoreConfig) *Blobstore { 24 | validate.NotEmpty(config.BucketName) 25 | validate.NotEmpty(config.ApiKey) 26 | validate.NotEmpty(config.ApiSecret) 27 | validate.NotEmpty(config.Endpoint) 28 | 29 | client, err := oss.New(config.Endpoint, config.ApiKey, config.ApiSecret) 30 | if err != nil { 31 | panic(err) 32 | } 33 | bucket, e := client.Bucket(config.BucketName) 34 | if e != nil { 35 | panic(fmt.Errorf("could not get bucket")) 36 | } 37 | return &Blobstore{ 38 | bucket: bucket, 39 | Client: client, 40 | } 41 | } 42 | 43 | func (blobstore *Blobstore) Copy(src string, dest string) error { 44 | logger.Log.Debugw("Copy in Alibaba", "bucket", blobstore.bucket.BucketName, "src", src, "dest", dest) 45 | _, e := blobstore.bucket.CopyObject(src, dest) 46 | if e != nil { 47 | return errors.Wrapf(e, "Error while trying to copy src %v to dest %v in bucket %v", src, dest, blobstore.bucket.BucketName) 48 | } 49 | return nil 50 | } 51 | 52 | func (blobstore *Blobstore) Delete(path string) error { 53 | e := blobstore.bucket.DeleteObject(path) 54 | if e != nil { 55 | return errors.Wrapf(e, "Path %v", path) 56 | } 57 | return nil 58 | } 59 | 60 | func (blobstore *Blobstore) DeleteDir(prefix string) error { 61 | deletionErrs := []error{} 62 | marker := oss.Marker("") 63 | 64 | for { 65 | objList, e := blobstore.bucket.ListObjects(oss.MaxKeys(20), marker, oss.Prefix(prefix)) 66 | if e != nil { 67 | return errors.Wrapf(e, "Prefix %v", prefix) 68 | } 69 | deletionErrs = append(deletionErrs, blobstore.deleteObjects(objList)...) 70 | marker = oss.Marker(objList.NextMarker) 71 | if !objList.IsTruncated { 72 | break 73 | } 74 | } 75 | 76 | if len(deletionErrs) > 0 { 77 | return errors.Errorf("Prefix %v, errors from deleting: %v", prefix, deletionErrs) 78 | } 79 | return nil 80 | } 81 | 82 | func (blobstore *Blobstore) deleteObjects(objListResult oss.ListObjectsResult) []error { 83 | deletionErrs := []error{} 84 | for _, obj := range objListResult.Objects { 85 | e := blobstore.bucket.DeleteObject(obj.Key) 86 | if e != nil { 87 | deletionErrs = append(deletionErrs, e) 88 | } 89 | } 90 | return deletionErrs 91 | } 92 | 93 | func (blobstore *Blobstore) Exists(path string) (bool, error) { 94 | return blobstore.bucket.IsObjectExist(path) 95 | } 96 | 97 | func (blobstore *Blobstore) Get(path string) (io.ReadCloser, error) { 98 | logger.Log.Debugw("GET", "bucket", blobstore.bucket.BucketName, "path", path) 99 | exists, _ := blobstore.Client.IsBucketExist(blobstore.bucket.BucketName) 100 | if !exists { 101 | return nil, errors.Errorf("Bucket not found: '%v'", blobstore.bucket.BucketName) 102 | } 103 | obj, err := blobstore.bucket.GetObject(path) 104 | if err != nil { 105 | return nil, bitsgo.NewNotFoundErrorWithMessage("Could not find object: " + path) 106 | } 107 | return obj, nil 108 | } 109 | 110 | func (blobstore *Blobstore) GetOrRedirect(path string) (io.ReadCloser, string, error) { 111 | signedURL, err := blobstore.bucket.SignURL(path, oss.HTTPGet, getValidityPeriod(time.Now().Add(1*time.Hour))) 112 | return nil, signedURL, err 113 | } 114 | 115 | func (blobstore *Blobstore) Put(path string, rs io.ReadSeeker) error { 116 | logger.Log.Debugw("Put", "bucket", blobstore.bucket.BucketName, "path", path) 117 | exists, _ := blobstore.Client.IsBucketExist(blobstore.bucket.BucketName) 118 | if !exists { 119 | return errors.Errorf("Bucket not found: '%v'", blobstore.bucket.BucketName) 120 | } 121 | return blobstore.bucket.PutObject(path, rs) 122 | } 123 | 124 | func (blobstore *Blobstore) Sign(path string, method string, timestamp time.Time) string { 125 | var ossMethod oss.HTTPMethod 126 | switch strings.ToLower(method) { 127 | case "put": 128 | ossMethod = oss.HTTPPut 129 | case "get": 130 | ossMethod = oss.HTTPGet 131 | 132 | default: 133 | panic("Supported methods are 'put' and 'get'.Got'" + method + "'") 134 | } 135 | signedURL, e := blobstore.bucket.SignURL(path, ossMethod, getValidityPeriod(timestamp)) 136 | if e != nil { 137 | panic(e) 138 | } 139 | return signedURL 140 | } 141 | 142 | func getValidityPeriod(timestamp time.Time) int64 { 143 | duration := timestamp.Sub(time.Now()).Seconds() 144 | return int64(duration) 145 | } 146 | -------------------------------------------------------------------------------- /blobstores/blobstore_test.go: -------------------------------------------------------------------------------- 1 | package blobstores_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/onsi/ginkgo" 9 | 10 | . "github.com/onsi/ginkgo" 11 | 12 | "github.com/onsi/gomega" 13 | 14 | "os" 15 | 16 | "github.com/cloudfoundry-incubator/bits-service" 17 | inmemory "github.com/cloudfoundry-incubator/bits-service/blobstores/inmemory" 18 | "github.com/cloudfoundry-incubator/bits-service/blobstores/local" 19 | "github.com/cloudfoundry-incubator/bits-service/config" 20 | . "github.com/onsi/gomega" 21 | ) 22 | 23 | func TestInMemoryBlobstore(t *testing.T) { 24 | gomega.RegisterFailHandler(ginkgo.Fail) 25 | ginkgo.RunSpecs(t, "InMemory Blobstore") 26 | } 27 | 28 | var _ = Describe("Blobstore", func() { 29 | var blobstore bitsgo.Blobstore 30 | 31 | itCanBeModifiedByItsMethods := func() { 32 | It("can be modified by its methods", func() { 33 | Expect(blobstore.Exists("/some/path")).To(BeFalse()) 34 | 35 | Expect(blobstore.Put("/some/path", strings.NewReader("some string"))).To(Succeed()) 36 | 37 | Expect(blobstore.Exists("/some/path")).To(BeTrue()) 38 | 39 | body, redirectLocation, e := blobstore.GetOrRedirect("/some/path") 40 | Expect(redirectLocation, e).To(BeEmpty()) 41 | Expect(ioutil.ReadAll(body)).To(MatchRegexp("some string")) 42 | 43 | Expect(blobstore.Copy("/some/path", "/some/other/path")).To(Succeed()) 44 | Expect(blobstore.Copy("/some/other/path", "/some/yet/other/path")).To(Succeed()) 45 | Expect(blobstore.Copy("/some/other/path", "/yet/some/other/path")).To(Succeed()) 46 | Expect(blobstore.Copy("/yet/some/other/path", "/yet/some/other/path")).To(Succeed()) 47 | 48 | body, redirectLocation, e = blobstore.GetOrRedirect("/some/other/path") 49 | Expect(redirectLocation, e).To(BeEmpty()) 50 | Expect(ioutil.ReadAll(body)).To(MatchRegexp("some string")) 51 | 52 | Expect(blobstore.Delete("/some/path")).To(Succeed()) 53 | 54 | Expect(blobstore.Exists("/some/path")).To(BeFalse()) 55 | 56 | Expect(blobstore.Exists("/some/other/path")).To(BeTrue()) 57 | 58 | Expect(blobstore.DeleteDir("/some")).To(Succeed()) 59 | Expect(blobstore.Exists("/some/other/path")).To(BeFalse()) 60 | Expect(blobstore.Exists("/some/yet/other/path")).To(BeFalse()) 61 | Expect(blobstore.Exists("/yet/some/other/path")).To(BeTrue()) 62 | 63 | Expect(blobstore.DeleteDir("")).To(Succeed()) 64 | Expect(blobstore.Exists("/yet/some/other/path")).To(BeFalse()) 65 | }) 66 | } 67 | 68 | Describe("Local", func() { 69 | var tempDirname string 70 | 71 | BeforeEach(func() { 72 | var e error 73 | tempDirname, e = ioutil.TempDir("", "bitsgo") 74 | Expect(e).NotTo(HaveOccurred()) 75 | 76 | blobstore = local.NewBlobstore(config.LocalBlobstoreConfig{PathPrefix: tempDirname}) 77 | }) 78 | AfterEach(func() { os.RemoveAll(tempDirname) }) 79 | 80 | itCanBeModifiedByItsMethods() 81 | }) 82 | 83 | Describe("In-memory", func() { 84 | BeforeEach(func() { blobstore = inmemory.NewBlobstore() }) 85 | 86 | itCanBeModifiedByItsMethods() 87 | }) 88 | }) 89 | -------------------------------------------------------------------------------- /blobstores/contract_integ_test/contract_integ_test_suite_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "github.com/cloudfoundry-incubator/bits-service" 5 | "github.com/onsi/ginkgo" 6 | "github.com/onsi/gomega" 7 | 8 | "testing" 9 | ) 10 | 11 | func TestBlobstores(t *testing.T) { 12 | gomega.RegisterFailHandler(ginkgo.Fail) 13 | ginkgo.RunSpecs(t, "Blobstores Contract Integration") 14 | } 15 | 16 | type blobstore interface { 17 | bitsgo.Blobstore 18 | bitsgo.ResourceSigner 19 | } 20 | -------------------------------------------------------------------------------- /blobstores/contract_integ_test/run-contract-integ-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | BLOBSTORE_TYPE="${1:?Missing parameter indicating blobstore type 4 | USAGE: run-contract-integ-tests.sh }" 5 | 6 | ginkgo -r --focus=$BLOBSTORE_TYPE -skip='SLOW TESTS' 7 | -------------------------------------------------------------------------------- /blobstores/decorator/metrics_emitting_blobstore_decorator.go: -------------------------------------------------------------------------------- 1 | package decorator 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | "github.com/cloudfoundry-incubator/bits-service" 8 | ) 9 | 10 | type MetricsEmittingBlobstoreDecorator struct { 11 | delegate bitsgo.Blobstore 12 | metricsService bitsgo.MetricsService 13 | resourceType string 14 | } 15 | 16 | func ForBlobstoreWithMetricsEmitter(delegate bitsgo.Blobstore, metricsService bitsgo.MetricsService, resourceType string) *MetricsEmittingBlobstoreDecorator { 17 | return &MetricsEmittingBlobstoreDecorator{delegate, metricsService, resourceType} 18 | } 19 | 20 | func (decorator *MetricsEmittingBlobstoreDecorator) Exists(path string) (bool, error) { 21 | startTime := time.Now() 22 | exists, e := decorator.delegate.Exists(path) 23 | decorator.metricsService.SendTimingMetric(decorator.resourceType+"-exists_in_blobstore-time", time.Since(startTime)) 24 | return exists, e 25 | } 26 | 27 | func (decorator *MetricsEmittingBlobstoreDecorator) Get(path string) (body io.ReadCloser, err error) { 28 | return decorator.delegate.Get(path) 29 | } 30 | 31 | func (decorator *MetricsEmittingBlobstoreDecorator) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) { 32 | return decorator.delegate.GetOrRedirect(path) 33 | } 34 | 35 | func (decorator *MetricsEmittingBlobstoreDecorator) Put(path string, src io.ReadSeeker) error { 36 | startTime := time.Now() 37 | e := decorator.delegate.Put(path, src) 38 | decorator.metricsService.SendTimingMetric(decorator.resourceType+"-cp_to_blobstore-time", time.Since(startTime)) 39 | return e 40 | } 41 | 42 | func (decorator *MetricsEmittingBlobstoreDecorator) Copy(src, dest string) error { 43 | startTime := time.Now() 44 | e := decorator.delegate.Copy(src, dest) 45 | decorator.metricsService.SendTimingMetric(decorator.resourceType+"-copy_in_blobstore-time", time.Since(startTime)) 46 | return e 47 | } 48 | 49 | func (decorator *MetricsEmittingBlobstoreDecorator) Delete(path string) error { 50 | startTime := time.Now() 51 | e := decorator.delegate.Delete(path) 52 | decorator.metricsService.SendTimingMetric(decorator.resourceType+"-delete_from_blobstore-time", time.Since(startTime)) 53 | return e 54 | } 55 | 56 | func (decorator *MetricsEmittingBlobstoreDecorator) DeleteDir(prefix string) error { 57 | startTime := time.Now() 58 | e := decorator.delegate.DeleteDir(prefix) 59 | decorator.metricsService.SendTimingMetric(decorator.resourceType+"-delete_dir_from_blobstore-time", time.Since(startTime)) 60 | return e 61 | } 62 | -------------------------------------------------------------------------------- /blobstores/decorator/partitioning_path_blobstore_decorator.go: -------------------------------------------------------------------------------- 1 | package decorator 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "time" 8 | 9 | "github.com/cloudfoundry-incubator/bits-service" 10 | ) 11 | 12 | func ForBlobstoreWithPathPartitioning(delegate bitsgo.Blobstore) *PartitioningPathBlobstoreDecorator { 13 | return &PartitioningPathBlobstoreDecorator{delegate} 14 | } 15 | 16 | type PartitioningPathBlobstoreDecorator struct { 17 | delegate bitsgo.Blobstore 18 | } 19 | 20 | func (decorator *PartitioningPathBlobstoreDecorator) Exists(path string) (bool, error) { 21 | return decorator.delegate.Exists(pathFor(path)) 22 | } 23 | 24 | func (decorator *PartitioningPathBlobstoreDecorator) Get(path string) (body io.ReadCloser, err error) { 25 | return decorator.delegate.Get(pathFor(path)) 26 | } 27 | 28 | func (decorator *PartitioningPathBlobstoreDecorator) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) { 29 | return decorator.delegate.GetOrRedirect(pathFor(path)) 30 | } 31 | 32 | func (decorator *PartitioningPathBlobstoreDecorator) Put(path string, src io.ReadSeeker) error { 33 | return decorator.delegate.Put(pathFor(path), src) 34 | } 35 | 36 | func (decorator *PartitioningPathBlobstoreDecorator) Copy(src, dest string) error { 37 | return decorator.delegate.Copy(pathFor(src), pathFor(dest)) 38 | } 39 | 40 | func (decorator *PartitioningPathBlobstoreDecorator) Delete(path string) error { 41 | return decorator.delegate.Delete(pathFor(path)) 42 | } 43 | 44 | func (decorator *PartitioningPathBlobstoreDecorator) DeleteDir(prefix string) error { 45 | if prefix == "" { 46 | return decorator.delegate.DeleteDir(prefix) 47 | } else { 48 | return decorator.delegate.DeleteDir(pathFor(prefix)) 49 | } 50 | } 51 | 52 | func pathFor(identifier string) string { 53 | if len(identifier) >= 4 { 54 | return fmt.Sprintf("%s/%s/%s", identifier[0:2], identifier[2:4], identifier) 55 | } else if len(identifier) == 3 { 56 | return fmt.Sprintf("%s/%s/%s", identifier[0:2], identifier[2:3], identifier) 57 | } else if len(identifier) == 2 { 58 | return fmt.Sprintf("%s/%s", identifier[0:2], identifier) 59 | } else if len(identifier) == 1 { 60 | return fmt.Sprintf("%s/%s", identifier[0:1], identifier) 61 | } 62 | return "" 63 | } 64 | 65 | func ForResourceSignerWithPathPartitioning(delegate bitsgo.ResourceSigner) *PartitioningPathResourceSigner { 66 | return &PartitioningPathResourceSigner{delegate} 67 | } 68 | 69 | type PartitioningPathResourceSigner struct { 70 | delegate bitsgo.ResourceSigner 71 | } 72 | 73 | func (signer *PartitioningPathResourceSigner) Sign(resource string, method string, expirationTime time.Time) (signedURL string) { 74 | return signer.delegate.Sign(pathFor(resource), method, expirationTime) 75 | } 76 | -------------------------------------------------------------------------------- /blobstores/decorator/prefixing_path_blobstore_decorator.go: -------------------------------------------------------------------------------- 1 | package decorator 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | "github.com/cloudfoundry-incubator/bits-service" 8 | ) 9 | 10 | type PrefixingPathBlobstoreDecorator struct { 11 | delegate bitsgo.Blobstore 12 | prefix string 13 | } 14 | 15 | func ForBlobstoreWithPathPrefixing(delegate bitsgo.Blobstore, prefix string) *PrefixingPathBlobstoreDecorator { 16 | return &PrefixingPathBlobstoreDecorator{delegate, prefix} 17 | } 18 | 19 | func (decorator *PrefixingPathBlobstoreDecorator) Exists(path string) (bool, error) { 20 | return decorator.delegate.Exists(decorator.prefix + path) 21 | } 22 | 23 | func (decorator *PrefixingPathBlobstoreDecorator) Get(path string) (body io.ReadCloser, err error) { 24 | return decorator.delegate.Get(decorator.prefix + path) 25 | } 26 | 27 | func (decorator *PrefixingPathBlobstoreDecorator) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) { 28 | return decorator.delegate.GetOrRedirect(decorator.prefix + path) 29 | } 30 | 31 | func (decorator *PrefixingPathBlobstoreDecorator) Put(path string, src io.ReadSeeker) error { 32 | return decorator.delegate.Put(decorator.prefix+path, src) 33 | } 34 | 35 | func (decorator *PrefixingPathBlobstoreDecorator) Copy(src, dest string) error { 36 | return decorator.delegate.Copy(decorator.prefix+src, decorator.prefix+dest) 37 | } 38 | 39 | func (decorator *PrefixingPathBlobstoreDecorator) Delete(path string) error { 40 | return decorator.delegate.Delete(decorator.prefix + path) 41 | } 42 | 43 | func (decorator *PrefixingPathBlobstoreDecorator) DeleteDir(prefix string) error { 44 | return decorator.delegate.DeleteDir(decorator.prefix + prefix) 45 | } 46 | 47 | type PrefixingPathResourceSigner struct { 48 | delegate bitsgo.ResourceSigner 49 | prefix string 50 | } 51 | 52 | func ForResourceSignerWithPathPrefixing(delegate bitsgo.ResourceSigner, prefix string) *PrefixingPathResourceSigner { 53 | return &PrefixingPathResourceSigner{delegate, prefix} 54 | } 55 | 56 | func (signer *PrefixingPathResourceSigner) Sign(resource string, method string, expirationTime time.Time) (signedURL string) { 57 | return signer.delegate.Sign(signer.prefix+resource, method, expirationTime) 58 | } 59 | -------------------------------------------------------------------------------- /blobstores/gcp/timeout_retry_test.go: -------------------------------------------------------------------------------- 1 | package gcp_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "testing" 8 | 9 | "github.com/cloudfoundry-incubator/bits-service/blobstores/gcp" 10 | "github.com/onsi/ginkgo" 11 | . "github.com/onsi/ginkgo" 12 | "github.com/onsi/gomega" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | func TestTimeoutRetry(t *testing.T) { 17 | gomega.RegisterFailHandler(ginkgo.Fail) 18 | ginkgo.RunSpecs(t, "Timeout retry test") 19 | } 20 | 21 | var _ = Describe("Timeout retry", func() { 22 | Context("error is context.DeadlineExceeded", func() { 23 | It("retries", func() { 24 | numRetries := 0 25 | e := gcp.WithRetries(2, func() error { 26 | e := context.DeadlineExceeded 27 | numRetries++ 28 | By(fmt.Sprintf("Try #%v", numRetries)) 29 | return gcp.TimeoutOrPermanent(e) 30 | }) 31 | Expect(numRetries).To(Equal(3)) 32 | Expect(e).To(MatchError(context.DeadlineExceeded)) 33 | }) 34 | }) 35 | 36 | Context("error is any other error", func() { 37 | It("stops after first attempt", func() { 38 | numRetries := 0 39 | e := gcp.WithRetries(3, func() error { 40 | e := errors.New("Some error") 41 | numRetries++ 42 | By(fmt.Sprintf("Try #%v", numRetries)) 43 | return gcp.TimeoutOrPermanent(e) 44 | }) 45 | Expect(numRetries).To(Equal(1)) 46 | Expect(e).To(MatchError("Some error")) 47 | }) 48 | }) 49 | }) 50 | -------------------------------------------------------------------------------- /blobstores/inmemory/inmemory_blobstore.go: -------------------------------------------------------------------------------- 1 | package inmemory_blobstore 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | 8 | "github.com/cloudfoundry-incubator/bits-service" 9 | 10 | "bytes" 11 | 12 | "io/ioutil" 13 | ) 14 | 15 | type Blobstore struct { 16 | Entries map[string][]byte 17 | } 18 | 19 | func NewBlobstore() *Blobstore { 20 | return &Blobstore{Entries: make(map[string][]byte)} 21 | } 22 | 23 | func NewBlobstoreWithEntries(entries map[string][]byte) *Blobstore { 24 | return &Blobstore{Entries: entries} 25 | } 26 | 27 | func (blobstore *Blobstore) Exists(path string) (bool, error) { 28 | _, hasKey := blobstore.Entries[path] 29 | return hasKey, nil 30 | } 31 | 32 | func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) { 33 | entry, hasKey := blobstore.Entries[path] 34 | if !hasKey { 35 | return nil, bitsgo.NewNotFoundError() 36 | } 37 | return ioutil.NopCloser(bytes.NewBuffer(entry)), nil 38 | } 39 | 40 | func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) { 41 | body, e := blobstore.Get(path) 42 | return body, "", e 43 | } 44 | 45 | func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error { 46 | b, e := ioutil.ReadAll(src) 47 | if e != nil { 48 | return fmt.Errorf("Error while reading from src %v. Caused by: %v", path, e) 49 | } 50 | blobstore.Entries[path] = b 51 | return nil 52 | } 53 | 54 | func (blobstore *Blobstore) Copy(src, dest string) error { 55 | blobstore.Entries[dest] = blobstore.Entries[src] 56 | return nil 57 | } 58 | 59 | func (blobstore *Blobstore) Delete(path string) error { 60 | _, hasKey := blobstore.Entries[path] 61 | if !hasKey { 62 | return bitsgo.NewNotFoundError() 63 | } 64 | delete(blobstore.Entries, path) 65 | return nil 66 | } 67 | 68 | func (blobstore *Blobstore) DeleteDir(prefix string) error { 69 | for key := range blobstore.Entries { 70 | if strings.HasPrefix(key, prefix) { 71 | delete(blobstore.Entries, key) 72 | } 73 | 74 | } 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /blobstores/local/local_blobstore.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/cloudfoundry-incubator/bits-service/config" 10 | 11 | "syscall" 12 | 13 | "github.com/cloudfoundry-incubator/bits-service" 14 | "github.com/cloudfoundry-incubator/bits-service/logger" 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | type Blobstore struct { 19 | pathPrefix string 20 | } 21 | 22 | func NewBlobstore(localConfig config.LocalBlobstoreConfig) *Blobstore { 23 | return &Blobstore{pathPrefix: localConfig.PathPrefix} 24 | } 25 | 26 | func (blobstore *Blobstore) Exists(path string) (bool, error) { 27 | _, err := os.Stat(filepath.Join(blobstore.pathPrefix, path)) 28 | if os.IsNotExist(err) { 29 | return false, nil 30 | } 31 | if err != nil { 32 | return false, fmt.Errorf("Could not stat on %v. Caused by: %v", filepath.Join(blobstore.pathPrefix, path), err) 33 | } 34 | return true, nil 35 | } 36 | 37 | func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) { 38 | logger.Log.Debugw("GetNoRedirect", "local-path", filepath.Join(blobstore.pathPrefix, path)) 39 | file, e := os.Open(filepath.Join(blobstore.pathPrefix, path)) 40 | 41 | if os.IsNotExist(e) { 42 | return nil, bitsgo.NewNotFoundError() 43 | } 44 | if e != nil { 45 | return nil, fmt.Errorf("Error while opening file %v. Caused by: %v", path, e) 46 | } 47 | return file, nil 48 | } 49 | 50 | func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) { 51 | body, e := blobstore.Get(path) 52 | return body, "", e 53 | } 54 | 55 | func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error { 56 | e := os.MkdirAll(filepath.Dir(filepath.Join(blobstore.pathPrefix, path)), os.ModeDir|0755) 57 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 58 | return bitsgo.NewNoSpaceLeftError() 59 | } 60 | if e != nil { 61 | return fmt.Errorf("Error while creating directories for %v. Caused by: %v", path, e) 62 | } 63 | file, e := os.Create(filepath.Join(blobstore.pathPrefix, path)) 64 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 65 | return bitsgo.NewNoSpaceLeftError() 66 | } 67 | if e != nil { 68 | return fmt.Errorf("Error while creating file %v. Caused by: %v", path, e) 69 | } 70 | defer file.Close() 71 | _, e = io.Copy(file, src) 72 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 73 | return bitsgo.NewNoSpaceLeftError() 74 | } 75 | if e != nil { 76 | return fmt.Errorf("Error while writing file %v. Caused by: %v", path, e) 77 | } 78 | return nil 79 | } 80 | 81 | func (blobstore *Blobstore) Copy(src, dest string) error { 82 | srcFull := filepath.Join(blobstore.pathPrefix, src) 83 | destFull := filepath.Join(blobstore.pathPrefix, dest) 84 | 85 | srcFile, e := os.Open(srcFull) 86 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 87 | return bitsgo.NewNoSpaceLeftError() 88 | } 89 | if os.IsNotExist(e) { 90 | return bitsgo.NewNotFoundError() 91 | } 92 | if e != nil { 93 | return errors.Wrapf(e, "Opening src failed. (src=%v, dest=%v)", srcFull, destFull) 94 | } 95 | defer srcFile.Close() 96 | 97 | e = os.MkdirAll(filepath.Dir(destFull), 0755) 98 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 99 | return bitsgo.NewNoSpaceLeftError() 100 | } 101 | if e != nil { 102 | return errors.Wrapf(e, "Make dir failed. (src=%v, dest=%v)", srcFull, destFull) 103 | } 104 | 105 | destFile, e := os.Create(destFull) 106 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 107 | return bitsgo.NewNoSpaceLeftError() 108 | } 109 | if e != nil { 110 | return errors.Wrapf(e, "Creating dest failed. (src=%v, dest=%v)", srcFull, destFull) 111 | } 112 | defer destFile.Close() 113 | 114 | _, e = io.Copy(destFile, srcFile) 115 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 116 | return bitsgo.NewNoSpaceLeftError() 117 | } 118 | if e != nil { 119 | return errors.Wrapf(e, "Copying failed. (src=%v, dest=%v)", srcFull, destFull) 120 | } 121 | 122 | return nil 123 | } 124 | 125 | func (blobstore *Blobstore) Delete(path string) error { 126 | _, e := os.Stat(filepath.Join(blobstore.pathPrefix, path)) 127 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 128 | return bitsgo.NewNoSpaceLeftError() 129 | } 130 | if os.IsNotExist(e) { 131 | return bitsgo.NewNotFoundError() 132 | } 133 | e = os.RemoveAll(filepath.Join(blobstore.pathPrefix, path)) 134 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 135 | return bitsgo.NewNoSpaceLeftError() 136 | } 137 | if e != nil { 138 | return fmt.Errorf("Error while deleting file %v. Caused by: %v", path, e) 139 | } 140 | return nil 141 | } 142 | 143 | func (blobstore *Blobstore) DeleteDir(prefix string) error { 144 | e := os.RemoveAll(filepath.Join(blobstore.pathPrefix, prefix)) 145 | if e, isPathError := e.(*os.PathError); isPathError && e.Err == syscall.ENOSPC { 146 | return bitsgo.NewNoSpaceLeftError() 147 | } 148 | if e != nil { 149 | return errors.Wrapf(e, "Failed to delete path %v", filepath.Join(blobstore.pathPrefix, prefix)) 150 | } 151 | return nil 152 | } 153 | -------------------------------------------------------------------------------- /blobstores/local/signed_urls.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "fmt" 5 | 6 | "time" 7 | 8 | "github.com/cloudfoundry-incubator/bits-service/pathsigner" 9 | ) 10 | 11 | type LocalResourceSigner struct { 12 | Signer pathsigner.PathSigner 13 | ResourcePathPrefix string 14 | DelegateEndpoint string 15 | } 16 | 17 | func (signer *LocalResourceSigner) Sign(resource string, method string, expirationTime time.Time) (signedURL string) { 18 | return fmt.Sprintf("%s%s", signer.DelegateEndpoint, signer.Signer.Sign(method, signer.ResourcePathPrefix+resource, expirationTime)) 19 | } 20 | -------------------------------------------------------------------------------- /blobstores/openstack/openstack_blobstore_suite_test.go: -------------------------------------------------------------------------------- 1 | package openstack_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestOpenstack(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Openstack Suite") 13 | } 14 | -------------------------------------------------------------------------------- /blobstores/openstack/openstack_blobstore_test.go: -------------------------------------------------------------------------------- 1 | package openstack_test 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | . "github.com/cloudfoundry-incubator/bits-service/blobstores/openstack" 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | const numWorkers = 1000 15 | 16 | var _ = Describe("DeleteInParallel", func() { 17 | Context("names is empty", func() { 18 | It("doesn't call the deletionFunc", func() { 19 | errs := DeleteInParallel([]string{}, numWorkers, func(string) error { 20 | defer GinkgoRecover() 21 | 22 | Fail("This function should not be called for an empty names slice") 23 | return nil 24 | }) 25 | Expect(errs).To(BeEmpty()) 26 | }) 27 | }) 28 | 29 | Context("names contains one element", func() { 30 | It("calls the deletionFunc once and returns no error", func() { 31 | var m sync.Mutex 32 | namesDeleted := make(map[string]bool) 33 | errs := DeleteInParallel([]string{"foo"}, numWorkers, func(name string) error { 34 | m.Lock() 35 | defer m.Unlock() 36 | namesDeleted[name] = true 37 | return nil 38 | }) 39 | Expect(errs).To(BeEmpty()) 40 | Expect(namesDeleted).To(SatisfyAll( 41 | HaveLen(1), 42 | HaveKey("foo"))) 43 | }) 44 | 45 | Context("deletionFunc returns an error", func() { 46 | It("returns the error as a result", func() { 47 | errs := DeleteInParallel([]string{"foo"}, numWorkers, func(string) error { 48 | time.Sleep(10 * time.Millisecond) 49 | return errors.New("some error") 50 | }) 51 | Expect(errs).To(ConsistOf(MatchError("some error"))) 52 | }) 53 | }) 54 | }) 55 | 56 | Context("names contains many elements", func() { 57 | const numNames = 10000 58 | var names []string 59 | 60 | BeforeEach(func() { 61 | names = make([]string, numNames) 62 | for i := 0; i < numNames; i++ { 63 | names[i] = fmt.Sprintf("%v", i) 64 | } 65 | }) 66 | 67 | Context("names contains many elements where each item takes 10ms to delete ", func() { 68 | It("calls the deletionFunc for every item and finishes within 2s", func(done Done) { 69 | var m sync.Mutex 70 | namesDeleted := make(map[string]bool) 71 | 72 | errs := DeleteInParallel(names, numWorkers, func(name string) error { 73 | time.Sleep(10 * time.Millisecond) 74 | m.Lock() 75 | defer m.Unlock() 76 | namesDeleted[name] = true 77 | return nil 78 | }) 79 | 80 | Expect(errs).To(BeEmpty()) 81 | Expect(namesDeleted).To(HaveLen(numNames)) 82 | for _, name := range names { 83 | if _, exists := namesDeleted[name]; !exists { 84 | Fail("Name not deleted :" + name) 85 | } 86 | } 87 | 88 | close(done) 89 | }, 2.0) 90 | }) 91 | 92 | Context("names contains many elements where all items return an erro after 10ms", func() { 93 | It("calls the deletionFunc for every item and finishes within 2s", func(done Done) { 94 | errs := DeleteInParallel(names, numWorkers, func(name string) error { 95 | time.Sleep(10 * time.Millisecond) 96 | return errors.New("some error") 97 | }) 98 | Expect(len(errs)).To(Equal(numNames)) 99 | 100 | close(done) 101 | }, 2.0) 102 | 103 | }) 104 | }) 105 | 106 | }) 107 | -------------------------------------------------------------------------------- /blobstores/s3/signed_urls_test.go: -------------------------------------------------------------------------------- 1 | package s3_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/cloudfoundry-incubator/bits-service/blobstores/decorator" 8 | . "github.com/cloudfoundry-incubator/bits-service/blobstores/s3" 9 | "github.com/cloudfoundry-incubator/bits-service/config" 10 | "github.com/onsi/ginkgo" 11 | . "github.com/onsi/ginkgo" 12 | "github.com/onsi/gomega" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | func TestS3Blobstore(t *testing.T) { 17 | gomega.RegisterFailHandler(ginkgo.Fail) 18 | ginkgo.RunSpecs(t, "S3Blobstore") 19 | } 20 | 21 | var _ = Describe("Signing URLs", func() { 22 | It("Can create pre-signed URLs for S3", func() { 23 | signer := decorator.ForResourceSignerWithPathPartitioning(NewBlobstore( 24 | config.S3BlobstoreConfig{ 25 | Bucket: "mybucket", 26 | AccessKeyID: "MY-Key_ID", 27 | SecretAccessKey: "dummy", 28 | Region: "us-east-1", 29 | })) 30 | 31 | signedURL := signer.Sign("myresource", "get", time.Now().Add(time.Hour)) 32 | 33 | Expect(signedURL).To(SatisfyAll( 34 | ContainSubstring("https://mybucket.s3.amazonaws.com/my/re/myresource"), 35 | ContainSubstring("X-Amz-Algorithm="), 36 | ContainSubstring("X-Amz-Credential=MY-Key_ID"), 37 | ContainSubstring("X-Amz-Date="), 38 | ContainSubstring("X-Amz-Expires="), 39 | ContainSubstring("X-Amz-Signature="), 40 | )) 41 | }) 42 | }) 43 | -------------------------------------------------------------------------------- /blobstores/s3/signer/default.go: -------------------------------------------------------------------------------- 1 | package signer 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/aws/aws-sdk-go/aws/request" 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | type Default struct{} 11 | 12 | func (d *Default) Sign(req *request.Request, bucket, path string, expires time.Time) (string, error) { 13 | signedURL, e := req.Presign(expires.Sub(time.Now())) 14 | if e != nil { 15 | return "", errors.Wrapf(e, "Bucket/Path %v/%v", bucket, path) 16 | } 17 | return signedURL, nil 18 | } 19 | -------------------------------------------------------------------------------- /blobstores/s3/signer/gcp.go: -------------------------------------------------------------------------------- 1 | package signer 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha1" 6 | "strings" 7 | "time" 8 | 9 | "cloud.google.com/go/storage" 10 | "github.com/aws/aws-sdk-go/aws/request" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type Google struct { 15 | AccessID string 16 | SecretAccessKey string 17 | } 18 | 19 | func (g *Google) Sign(req *request.Request, bucket, path string, expires time.Time) (string, error) { 20 | signedURL, e := storage.SignedURL(bucket, path, &storage.SignedURLOptions{ 21 | GoogleAccessID: g.AccessID, 22 | SignBytes: func(b []byte) ([]byte, error) { 23 | hash := hmac.New(sha1.New, []byte(g.SecretAccessKey)) 24 | hash.Write(b) 25 | return hash.Sum(nil), nil 26 | }, 27 | Method: strings.ToUpper(req.HTTPRequest.Method), 28 | Expires: expires, 29 | }) 30 | if e != nil { 31 | return "", errors.Wrapf(e, "Bucket/Path %v/%v", bucket, path) 32 | } 33 | return signedURL, nil 34 | } 35 | -------------------------------------------------------------------------------- /blobstores/s3/signer/v2.go: -------------------------------------------------------------------------------- 1 | package signer 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha1" 6 | "encoding/base64" 7 | "fmt" 8 | "net/http" 9 | "net/url" 10 | "sort" 11 | "strings" 12 | "time" 13 | 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/aws/credentials" 16 | "github.com/aws/aws-sdk-go/aws/request" 17 | ) 18 | 19 | type V2Signer struct { 20 | Request *http.Request 21 | ExpireTime time.Duration 22 | Credentials *credentials.Credentials 23 | Debug aws.LogLevelType 24 | Logger aws.Logger 25 | Bucket string 26 | } 27 | 28 | func (v2 V2Signer) Sign(req *request.Request) { 29 | if req.Config.Credentials == credentials.AnonymousCredentials { 30 | return 31 | } 32 | v2.Request = req.HTTPRequest 33 | v2.ExpireTime = req.ExpireTime 34 | 35 | req.Error = v2.sign() 36 | } 37 | 38 | func (v2 *V2Signer) sign() error { 39 | credValue, err := v2.Credentials.Get() 40 | if err != nil { 41 | return err 42 | } 43 | 44 | expireTime := time.Now().Add(v2.ExpireTime) 45 | 46 | var keys []string 47 | for k := range v2.Request.Header { 48 | keys = append(keys, k) 49 | } 50 | sort.StringSlice(keys).Sort() 51 | 52 | var sarray []string 53 | for _, k := range keys { 54 | if strings.HasPrefix(strings.ToLower(k), "x-amz-") && strings.ToLower(k) != "x-amz-date" { 55 | sarray = append(sarray, strings.ToLower(k)+":"+strings.Join(v2.Request.Header[k], ",")) 56 | } 57 | } 58 | canonicalizedAmzHeaders := "" 59 | if len(sarray) > 0 { 60 | canonicalizedAmzHeaders = strings.Join(sarray, "\n") + "\n" 61 | } 62 | 63 | stringToSign := "" 64 | if v2.ExpireTime > 0 { 65 | stringToSign = strings.Join([]string{ 66 | v2.Request.Method, 67 | v2.Request.Header.Get("Content-MD5"), 68 | v2.Request.Header.Get("Content-Type"), 69 | fmt.Sprintf("%v", expireTime.Unix()), 70 | canonicalizedAmzHeaders + 71 | "/" + v2.Bucket + strings.Replace(v2.Request.URL.EscapedPath(), "%2F", "/", -1), 72 | }, "\n") 73 | } else { 74 | stringToSign = strings.Join([]string{ 75 | v2.Request.Method, 76 | v2.Request.Header.Get("Content-MD5"), 77 | v2.Request.Header.Get("Content-Type"), 78 | expireTime.UTC().Format(http.TimeFormat), 79 | canonicalizedAmzHeaders + 80 | "/" + v2.Bucket + strings.Replace(v2.Request.URL.EscapedPath(), "%2F", "/", -1), 81 | }, "\n") 82 | } 83 | 84 | hash := hmac.New(sha1.New, []byte(credValue.SecretAccessKey)) 85 | hash.Write([]byte(stringToSign)) 86 | signature := string(base64.StdEncoding.EncodeToString(hash.Sum(nil))) 87 | 88 | if v2.ExpireTime > 0 { 89 | params := v2.Request.URL.Query() 90 | params["Expires"] = []string{fmt.Sprintf("%v", expireTime.Unix())} 91 | params["AWSAccessKeyId"] = []string{credValue.AccessKeyID} 92 | params["Signature"] = []string{signature} 93 | v2.Request.URL.RawQuery = url.Values(params).Encode() 94 | } else { 95 | v2.Request.Header.Set("Authorization", "AWS "+credValue.AccessKeyID+":"+signature) 96 | v2.Request.Header.Set("Date", expireTime.UTC().Format(http.TimeFormat)) 97 | } 98 | 99 | if v2.Debug.Matches(aws.LogDebugWithSigning) { 100 | v2.Logger.Log(fmt.Sprintf(logSignInfoMsg, stringToSign, v2.Request.URL.String())) 101 | } 102 | return nil 103 | } 104 | 105 | const logSignInfoMsg = `DEBUG: Request Signature: 106 | ---[ STRING TO SIGN ]-------------------------------- 107 | %s 108 | ---[ SIGNED URL ]------------------------------------ 109 | %s` 110 | -------------------------------------------------------------------------------- /blobstores/s3/util.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/aws/awserr" 6 | "github.com/aws/aws-sdk-go/aws/credentials" 7 | "github.com/aws/aws-sdk-go/aws/request" 8 | "github.com/aws/aws-sdk-go/aws/session" 9 | "github.com/aws/aws-sdk-go/aws/signer/v4" 10 | "github.com/aws/aws-sdk-go/service/s3" 11 | "github.com/cloudfoundry-incubator/bits-service/blobstores/s3/signer" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | var loglevelTypes = map[string]aws.LogLevelType{ 16 | "LogDebug": aws.LogDebug, 17 | "LogDebugWithSigning": aws.LogDebugWithSigning, 18 | "LogDebugWithHTTPBody": aws.LogDebugWithHTTPBody, 19 | "LogDebugWithRequestRetries": aws.LogDebugWithRequestRetries, 20 | "LogDebugWithRequestErrors": aws.LogDebugWithRequestErrors, 21 | "LogDebugWithEventStreamBody": aws.LogDebugWithEventStreamBody, 22 | } 23 | 24 | func newS3Client(region string, 25 | useIAMProfile bool, 26 | accessKeyID string, 27 | secretAccessKey string, 28 | host string, 29 | logger *zap.SugaredLogger, 30 | loglevelString string, 31 | bucket string, 32 | signatureVersion int, 33 | ) *s3.S3 { 34 | c := &aws.Config{ 35 | Region: aws.String(region), 36 | Endpoint: aws.String(host), 37 | } 38 | if !useIAMProfile { 39 | c.Credentials = credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "") 40 | } 41 | if loglevelString != "" { 42 | c.Logger = aws.LoggerFunc(func(args ...interface{}) { logger.Debug(args...) }) 43 | 44 | if loglevel, exist := loglevelTypes[loglevelString]; exist { 45 | c.LogLevel = aws.LogLevel(loglevel) 46 | logger.Infow("Enabled S3 debug log", "log-level", loglevelString) 47 | } else { 48 | c.LogLevel = aws.LogLevel(aws.LogDebug) 49 | logger.Errorw("Invalid S3 debug loglevel. Using default S3 log-level", "log-level", loglevelString, "default-log-level", "LogDebug") 50 | } 51 | } 52 | ses := session.Must(session.NewSession(c)) 53 | s3Client := s3.New(ses) 54 | 55 | if signatureVersion == 2 { 56 | s3Client.Handlers.Sign.Swap(v4.SignRequestHandler.Name, request.NamedHandler{ 57 | Name: "v2.SignHandler", 58 | Fn: (&signer.V2Signer{ 59 | Credentials: c.Credentials, 60 | Debug: *ses.Config.LogLevel, 61 | Logger: ses.Config.Logger, 62 | Bucket: bucket, 63 | }).Sign, 64 | }) 65 | logger.Infow("Using signature version 2 signing.") 66 | } 67 | 68 | // This priming is only done to make the service fail fast in case it was misconfigured instead of making it fail on the first request served. 69 | _, e := s3Client.GetObject(&s3.GetObjectInput{ 70 | Bucket: aws.String("dummy"), 71 | Key: aws.String("dummy"), 72 | }) 73 | if awsErr, isAwsErr := e.(awserr.Error); isAwsErr && awsErr.Code() == "NoCredentialProviders" && useIAMProfile { 74 | logger.Fatalw("Blobstore is configured to use EC2 instance roles (use-iam-profiles), but no EC2 instance role could be found. "+ 75 | "If you want to use EC2 instance roles, please make sure that an EC2 instance role is attached to the EC2 instance this service is running on. "+ 76 | "No access-key-id and no secret-access-key is needed in that case. See also: https://docs.cloudfoundry.org/deploying/common/cc-blobstore-config.html#fog-aws-iam.", 77 | "use-iam-profiles", useIAMProfile, 78 | "access-key-id", accessKeyID, 79 | "secret-access-key-is-set", secretAccessKey != "", 80 | "region", region, 81 | "host", host) 82 | } 83 | return s3Client 84 | } 85 | 86 | func isS3NotFoundError(e error) bool { 87 | if ae, isAwsErr := e.(awserr.Error); isAwsErr { 88 | if ae.Code() == "NoSuchKey" || ae.Code() == "NotFound" { 89 | return true 90 | } 91 | } 92 | return false 93 | } 94 | 95 | func isS3NoSuchBucketError(e error) bool { 96 | if ae, isAwsErr := e.(awserr.Error); isAwsErr { 97 | if ae.Code() == "NoSuchBucket" { 98 | return true 99 | } 100 | } 101 | return false 102 | } 103 | -------------------------------------------------------------------------------- /blobstores/validate/validate.go: -------------------------------------------------------------------------------- 1 | package validate 2 | 3 | func NotEmptyMessage(s string, panicMessage string) { 4 | if s == "" { 5 | panic(panicMessage) 6 | } 7 | } 8 | 9 | func NotEmpty(s string) { 10 | if s == "" { 11 | panic("String must not be empty") 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /blobstores/webdav/http_client.go: -------------------------------------------------------------------------------- 1 | package webdav 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "net/http" 7 | "time" 8 | 9 | log "github.com/cloudfoundry-incubator/bits-service/logger" 10 | ) 11 | 12 | func NewHttpClient(pemCerts string, insecureSkipVerify bool) *http.Client { 13 | log.Log.Infow("Creating Http Client", "insecure-skip-verify", insecureSkipVerify, "ca-cert-path", pemCerts) 14 | 15 | caCertPool := x509.NewCertPool() 16 | ok := caCertPool.AppendCertsFromPEM([]byte(pemCerts)) 17 | if !ok { 18 | panic("Could not append pemCerts. pemCerts content:\n\n```\n" + pemCerts + "\n```") 19 | } 20 | 21 | return &http.Client{ 22 | Transport: &http.Transport{ 23 | TLSClientConfig: &tls.Config{ 24 | InsecureSkipVerify: insecureSkipVerify, 25 | RootCAs: caCertPool, 26 | }, 27 | }, 28 | Timeout: 15 * time.Minute, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /blobstores/webdav/webdav_blobstore_test.go: -------------------------------------------------------------------------------- 1 | package webdav_test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | 10 | . "github.com/cloudfoundry-incubator/bits-service/blobstores/webdav" 11 | "github.com/cloudfoundry-incubator/bits-service/config" 12 | ) 13 | 14 | var _ = Describe("WebdavBlobstore", func() { 15 | Describe("DeleteDir", func() { 16 | var ( 17 | webdavBlobstore *Blobstore 18 | testServer *httptest.Server 19 | ) 20 | 21 | BeforeEach(func() { 22 | testServer = httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { 23 | Expect(req.URL).ToNot(HaveSuffix("//")) 24 | Expect(req.URL).To(HaveSuffix("/")) 25 | })) 26 | webdavBlobstore = NewBlobstoreWithHttpClient(config.WebdavBlobstoreConfig{ 27 | PrivateEndpoint: testServer.URL, 28 | PublicEndpoint: testServer.URL, 29 | }, &http.Client{}) 30 | }) 31 | 32 | AfterEach(func() { testServer.Close() }) 33 | 34 | It("appends slash to url if needed", func() { 35 | webdavBlobstore.DeleteDir("path/without/slash/suffix") 36 | }) 37 | 38 | It("does not append a slash when there is already a slash at the end", func() { 39 | webdavBlobstore.DeleteDir("path/with/single/slash/suffix/") 40 | }) 41 | }) 42 | }) 43 | -------------------------------------------------------------------------------- /blobstores/webdav/webdav_suite_test.go: -------------------------------------------------------------------------------- 1 | package webdav_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestWebdav(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Webdav Suite") 13 | } 14 | -------------------------------------------------------------------------------- /body_size_limit.go: -------------------------------------------------------------------------------- 1 | package bitsgo 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "net/http" 7 | 8 | "github.com/cloudfoundry-incubator/bits-service/logger" 9 | ) 10 | 11 | // Note: this changes the request under certain conditions 12 | func HandleBodySizeLimits(responseWriter http.ResponseWriter, request *http.Request, maxBodySizeLimit uint64) (shouldContinue bool) { 13 | if maxBodySizeLimit != 0 { 14 | logger.From(request).Debugw("max-body-size is enabled", "max-body-size", maxBodySizeLimit) 15 | if request.ContentLength == -1 { 16 | badRequest(responseWriter, request, "HTTP header does not contain Content-Length") 17 | return 18 | } 19 | if uint64(request.ContentLength) > maxBodySizeLimit { 20 | defer request.Body.Close() 21 | 22 | // Reading the body here is really just to make Ruby's RestClient happy. 23 | // For some reason it crashes if we don't read the body. 24 | io.Copy(ioutil.Discard, request.Body) 25 | responseWriter.WriteHeader(http.StatusRequestEntityTooLarge) 26 | return 27 | } 28 | request.Body = &limitedReader{request.Body, request.ContentLength} 29 | } 30 | shouldContinue = true 31 | return 32 | } 33 | 34 | // Copied more or less from io.LimitedReader 35 | type limitedReader struct { 36 | delegate io.Reader 37 | maxBytesRemaining int64 38 | } 39 | 40 | func (l *limitedReader) Read(p []byte) (n int, err error) { 41 | if l.maxBytesRemaining <= 0 { 42 | return 0, io.EOF 43 | } 44 | if int64(len(p)) > l.maxBytesRemaining { 45 | p = p[0:l.maxBytesRemaining] 46 | } 47 | n, err = l.delegate.Read(p) 48 | l.maxBytesRemaining -= int64(n) 49 | return 50 | } 51 | 52 | func (l *limitedReader) Close() error { 53 | // Reading the body here is really just to make Ruby's RestClient happy. 54 | // For some reason it crashes if we don't read the body. 55 | io.Copy(ioutil.Discard, l.delegate) 56 | // TODO Should we return errors? 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /body_size_limit_test.go: -------------------------------------------------------------------------------- 1 | package bitsgo_test 2 | 3 | import ( 4 | . "github.com/cloudfoundry-incubator/bits-service" 5 | "github.com/cloudfoundry-incubator/bits-service/httputil" 6 | 7 | "net/http" 8 | "net/http/httptest" 9 | 10 | "strings" 11 | 12 | "io/ioutil" 13 | 14 | "io" 15 | ) 16 | 17 | type testHandler struct { 18 | maxBodySize uint64 19 | manipulatedContentLength int64 20 | } 21 | 22 | func (handler *testHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { 23 | defer GinkgoRecover() 24 | if handler.manipulatedContentLength != 0 { 25 | manipulateContentLength(request, handler.manipulatedContentLength) 26 | } 27 | if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySize) { 28 | return 29 | } 30 | _, e := io.Copy(responseWriter, request.Body) 31 | Expect(e).NotTo(HaveOccurred()) 32 | } 33 | 34 | func manipulateContentLength(request *http.Request, newContentLength int64) { 35 | request.ContentLength = newContentLength 36 | } 37 | 38 | var _ = Describe("HandleBodySizeLimits", func() { 39 | 40 | It("returns StatusRequestEntityTooLarge when body size exceeds maxBodySize", func() { 41 | server := httptest.NewServer(&testHandler{maxBodySize: 5}) 42 | 43 | response, e := http.DefaultClient.Do(httputil.NewRequest("GET", server.URL, strings.NewReader("Hello world!")).Build()) 44 | Expect(e).NotTo(HaveOccurred()) 45 | Expect(response.StatusCode).To(Equal(http.StatusRequestEntityTooLarge)) 46 | 47 | server.Close() 48 | }) 49 | 50 | It("does not check body sizes at all when maxBodySize is 0", func() { 51 | server := httptest.NewServer(&testHandler{maxBodySize: 0}) 52 | 53 | response, e := http.DefaultClient.Do(httputil.NewRequest("GET", server.URL, strings.NewReader("Hello world!")).Build()) 54 | Expect(e).NotTo(HaveOccurred()) 55 | Expect(ioutil.ReadAll(response.Body)).To(MatchRegexp("^Hello world!$")) 56 | 57 | server.Close() 58 | }) 59 | 60 | It("uses the full body when the body size does not exceed maxBodySize", func() { 61 | server := httptest.NewServer(&testHandler{maxBodySize: 20}) 62 | 63 | response, e := http.DefaultClient.Do(httputil.NewRequest("GET", server.URL, strings.NewReader("Hello world!")).Build()) 64 | Expect(e).NotTo(HaveOccurred()) 65 | Expect(ioutil.ReadAll(response.Body)).To(MatchRegexp("^Hello world!$")) 66 | 67 | server.Close() 68 | }) 69 | 70 | It("uses ContentLength as authoritative source for body sizes when ContentLength and body size differ", func() { 71 | server := httptest.NewServer(&testHandler{maxBodySize: 20, manipulatedContentLength: 5}) 72 | 73 | response, e := http.DefaultClient.Do(httputil.NewRequest("GET", server.URL, strings.NewReader("Hello world!")).Build()) 74 | Expect(e).NotTo(HaveOccurred()) 75 | Expect(ioutil.ReadAll(response.Body)).To(MatchRegexp("^Hello$")) 76 | 77 | server.Close() 78 | }) 79 | 80 | }) 81 | -------------------------------------------------------------------------------- /ccupdater/ccupdater.go: -------------------------------------------------------------------------------- 1 | package ccupdater 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "encoding/json" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "strings" 12 | 13 | "github.com/cloudfoundry-incubator/bits-service/logger" 14 | 15 | "github.com/cloudfoundry-incubator/bits-service" 16 | 17 | "github.com/pkg/errors" 18 | ) 19 | 20 | type CCUpdater struct { 21 | httpClient HttpClient 22 | endpoint string 23 | method string 24 | } 25 | 26 | type processingUploadPayload struct { 27 | State string `json:"state"` 28 | } 29 | 30 | type checksum struct { 31 | Type string `json:"type"` 32 | Value string `json:"value"` 33 | } 34 | 35 | type successPayload struct { 36 | State string `json:"state"` 37 | Checksums []checksum `json:"checksums"` 38 | } 39 | 40 | type failurePayload struct { 41 | State string `json:"state"` 42 | Error string `json:"error"` 43 | } 44 | 45 | type HttpClient interface { 46 | Do(*http.Request) (*http.Response, error) 47 | } 48 | 49 | func NewCCUpdater(endpoint string, method string, clientCertFile string, clientKeyFile string, caCertFile string) *CCUpdater { 50 | u, e := url.Parse(endpoint) 51 | if e != nil { 52 | logger.Log.Fatalw("Could not parse endpoint", "endpoint", endpoint, "error", e) 53 | } 54 | 55 | var tlsConfig *tls.Config 56 | if u.Scheme == "https" { 57 | tlsConfig = loadTLSConfig(clientCertFile, clientKeyFile, caCertFile) 58 | } 59 | return NewCCUpdaterWithHttpClient(endpoint, method, &http.Client{ 60 | Transport: &http.Transport{TLSClientConfig: tlsConfig}, 61 | }) 62 | } 63 | 64 | func NewCCUpdaterWithHttpClient(endpoint string, method string, httpClient HttpClient) *CCUpdater { 65 | return &CCUpdater{ 66 | httpClient: httpClient, 67 | endpoint: endpoint, 68 | method: method, 69 | } 70 | } 71 | 72 | func loadTLSConfig(clientCertFile string, clientKeyFile string, caCertFile string) *tls.Config { 73 | cert, e := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) 74 | if e != nil { 75 | logger.Log.Fatalw("Could not load X509 key pair", "error", e, "client-cert-file", clientCertFile, "client-key-file", clientKeyFile) 76 | } 77 | 78 | caCert, e := ioutil.ReadFile(caCertFile) 79 | if e != nil { 80 | logger.Log.Fatalw("Could not read CA Cert file", "error", e, "ca-cert-file", caCertFile) 81 | } 82 | caCertPool := x509.NewCertPool() 83 | caCertPool.AppendCertsFromPEM(caCert) 84 | 85 | tlsConfig := &tls.Config{ 86 | Certificates: []tls.Certificate{cert}, 87 | RootCAs: caCertPool, 88 | } 89 | tlsConfig.BuildNameToCertificate() 90 | return tlsConfig 91 | } 92 | 93 | func (updater *CCUpdater) NotifyProcessingUpload(guid string) error { 94 | return updater.update(guid, processingUploadPayload{"PROCESSING_UPLOAD"}) 95 | } 96 | 97 | func (updater *CCUpdater) NotifyUploadSucceeded(guid string, sha1 string, sha256 string) error { 98 | return updater.update(guid, successPayload{ 99 | "READY", 100 | []checksum{ 101 | checksum{Type: "sha1", Value: sha1}, 102 | checksum{Type: "sha256", Value: sha256}, 103 | }, 104 | }) 105 | } 106 | 107 | func (updater *CCUpdater) NotifyUploadFailed(guid string, e error) error { 108 | return updater.update(guid, failurePayload{"FAILED", e.Error()}) 109 | } 110 | 111 | func (updater *CCUpdater) update(guid string, p interface{}) error { 112 | payload, e := json.Marshal(p) 113 | if e != nil { 114 | logger.Log.Fatalw("Unexpected error in CC Updater update when marshalling payload", 115 | "error", e, "guid", guid, "payload", p) 116 | } 117 | 118 | r, e := http.NewRequest(updater.method, strings.TrimRight(updater.endpoint, "/")+"/"+guid, bytes.NewReader(payload)) 119 | if e != nil { 120 | logger.Log.Fatalw("Unexpected error in CC Updater update when creating new request", 121 | "error", e, "guid", guid, "payload", p) 122 | } 123 | resp, e := updater.httpClient.Do(r) 124 | if e != nil { 125 | return errors.Wrapf(e, "Could not make request against CC (GUID: \"%v\")", guid) 126 | } 127 | if resp.StatusCode == http.StatusNotFound { 128 | return bitsgo.NewNotFoundError() 129 | } 130 | if resp.StatusCode == http.StatusUnprocessableEntity { 131 | return bitsgo.NewStateForbiddenError() 132 | } 133 | return nil 134 | } 135 | -------------------------------------------------------------------------------- /ccupdater/ccupdater_suite_test.go: -------------------------------------------------------------------------------- 1 | package ccupdater_test 2 | 3 | import ( 4 | "github.com/onsi/ginkgo" 5 | "github.com/onsi/gomega" 6 | "github.com/petergtz/pegomock" 7 | 8 | "testing" 9 | ) 10 | 11 | func TestCcUpdater(t *testing.T) { 12 | RegisterFailHandler(Fail) 13 | pegomock.RegisterMockFailHandler(ginkgo.Fail) 14 | RunSpecs(t, "CcUpdater Suite") 15 | } 16 | 17 | // Declarations for Ginkgo DSL 18 | type Done ginkgo.Done 19 | type Benchmarker ginkgo.Benchmarker 20 | 21 | var GinkgoWriter = ginkgo.GinkgoWriter 22 | var GinkgoRandomSeed = ginkgo.GinkgoRandomSeed 23 | var GinkgoParallelNode = ginkgo.GinkgoParallelNode 24 | var GinkgoT = ginkgo.GinkgoT 25 | var CurrentGinkgoTestDescription = ginkgo.CurrentGinkgoTestDescription 26 | var RunSpecs = ginkgo.RunSpecs 27 | var RunSpecsWithDefaultAndCustomReporters = ginkgo.RunSpecsWithDefaultAndCustomReporters 28 | var RunSpecsWithCustomReporters = ginkgo.RunSpecsWithCustomReporters 29 | var Skip = ginkgo.Skip 30 | var Fail = ginkgo.Fail 31 | var GinkgoRecover = ginkgo.GinkgoRecover 32 | var Describe = ginkgo.Describe 33 | var FDescribe = ginkgo.FDescribe 34 | var PDescribe = ginkgo.PDescribe 35 | var XDescribe = ginkgo.XDescribe 36 | var Context = ginkgo.Context 37 | var FContext = ginkgo.FContext 38 | var PContext = ginkgo.PContext 39 | var XContext = ginkgo.XContext 40 | var It = ginkgo.It 41 | var FIt = ginkgo.FIt 42 | var PIt = ginkgo.PIt 43 | var XIt = ginkgo.XIt 44 | var Specify = ginkgo.Specify 45 | var FSpecify = ginkgo.FSpecify 46 | var PSpecify = ginkgo.PSpecify 47 | var XSpecify = ginkgo.XSpecify 48 | var By = ginkgo.By 49 | var Measure = ginkgo.Measure 50 | var FMeasure = ginkgo.FMeasure 51 | var PMeasure = ginkgo.PMeasure 52 | var XMeasure = ginkgo.XMeasure 53 | var BeforeSuite = ginkgo.BeforeSuite 54 | var AfterSuite = ginkgo.AfterSuite 55 | var SynchronizedBeforeSuite = ginkgo.SynchronizedBeforeSuite 56 | var SynchronizedAfterSuite = ginkgo.SynchronizedAfterSuite 57 | var BeforeEach = ginkgo.BeforeEach 58 | var JustBeforeEach = ginkgo.JustBeforeEach 59 | var AfterEach = ginkgo.AfterEach 60 | 61 | // Declarations for Gomega DSL 62 | var RegisterFailHandler = gomega.RegisterFailHandler 63 | var RegisterTestingT = gomega.RegisterTestingT 64 | var InterceptGomegaFailures = gomega.InterceptGomegaFailures 65 | var Ω = gomega.Ω 66 | var Expect = gomega.Expect 67 | var ExpectWithOffset = gomega.ExpectWithOffset 68 | var Eventually = gomega.Eventually 69 | var EventuallyWithOffset = gomega.EventuallyWithOffset 70 | var Consistently = gomega.Consistently 71 | var ConsistentlyWithOffset = gomega.ConsistentlyWithOffset 72 | var SetDefaultEventuallyTimeout = gomega.SetDefaultEventuallyTimeout 73 | var SetDefaultEventuallyPollingInterval = gomega.SetDefaultEventuallyPollingInterval 74 | var SetDefaultConsistentlyDuration = gomega.SetDefaultConsistentlyDuration 75 | var SetDefaultConsistentlyPollingInterval = gomega.SetDefaultConsistentlyPollingInterval 76 | var NewGomegaWithT = gomega.NewGomegaWithT 77 | 78 | // Declarations for Gomega Matchers 79 | var Equal = gomega.Equal 80 | var BeEquivalentTo = gomega.BeEquivalentTo 81 | var BeIdenticalTo = gomega.BeIdenticalTo 82 | var BeNil = gomega.BeNil 83 | var BeTrue = gomega.BeTrue 84 | var BeFalse = gomega.BeFalse 85 | var HaveOccurred = gomega.HaveOccurred 86 | var Succeed = gomega.Succeed 87 | var MatchError = gomega.MatchError 88 | var BeClosed = gomega.BeClosed 89 | var Receive = gomega.Receive 90 | var BeSent = gomega.BeSent 91 | var MatchRegexp = gomega.MatchRegexp 92 | var ContainSubstring = gomega.ContainSubstring 93 | var HavePrefix = gomega.HavePrefix 94 | var HaveSuffix = gomega.HaveSuffix 95 | var MatchJSON = gomega.MatchJSON 96 | var MatchXML = gomega.MatchXML 97 | var MatchYAML = gomega.MatchYAML 98 | var BeEmpty = gomega.BeEmpty 99 | var HaveLen = gomega.HaveLen 100 | var HaveCap = gomega.HaveCap 101 | var BeZero = gomega.BeZero 102 | var ContainElement = gomega.ContainElement 103 | var ConsistOf = gomega.ConsistOf 104 | var HaveKey = gomega.HaveKey 105 | var HaveKeyWithValue = gomega.HaveKeyWithValue 106 | var BeNumerically = gomega.BeNumerically 107 | var BeTemporally = gomega.BeTemporally 108 | var BeAssignableToTypeOf = gomega.BeAssignableToTypeOf 109 | var Panic = gomega.Panic 110 | var BeAnExistingFile = gomega.BeAnExistingFile 111 | var BeARegularFile = gomega.BeARegularFile 112 | var BeADirectory = gomega.BeADirectory 113 | var And = gomega.And 114 | var SatisfyAll = gomega.SatisfyAll 115 | var Or = gomega.Or 116 | var SatisfyAny = gomega.SatisfyAny 117 | var Not = gomega.Not 118 | var WithTransform = gomega.WithTransform 119 | -------------------------------------------------------------------------------- /ccupdater/ccupdater_test.go: -------------------------------------------------------------------------------- 1 | package ccupdater_test 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | 8 | "github.com/cloudfoundry-incubator/bits-service" 9 | 10 | . "github.com/cloudfoundry-incubator/bits-service/ccupdater" 11 | . "github.com/cloudfoundry-incubator/bits-service/ccupdater/matchers" 12 | . "github.com/petergtz/pegomock" 13 | ) 14 | 15 | var _ = Describe("CCUpdater", func() { 16 | var ( 17 | httpClient *MockHttpClient 18 | updater *CCUpdater 19 | ) 20 | 21 | BeforeEach(func() { 22 | httpClient = NewMockHttpClient() 23 | updater = NewCCUpdaterWithHttpClient("http://example.com/some/endpoint", "PATCH", httpClient) 24 | }) 25 | 26 | Describe("NotifyProcessingUpload", func() { 27 | It("works", func() { 28 | When(httpClient.Do(AnyPtrToHttpRequest())).ThenReturn(&http.Response{}, nil) 29 | 30 | e := updater.NotifyProcessingUpload("abc") 31 | 32 | Expect(e).NotTo(HaveOccurred()) 33 | 34 | request := httpClient.VerifyWasCalledOnce().Do(AnyPtrToHttpRequest()).GetCapturedArguments() 35 | Expect(request.Method).To(Equal("PATCH")) 36 | Expect(request.URL.String()).To(Equal("http://example.com/some/endpoint/abc")) 37 | Expect(ioutil.ReadAll(request.Body)).To(MatchJSON(`{"state":"PROCESSING_UPLOAD"}`)) 38 | }) 39 | 40 | Context("http client returns some generic error", func() { 41 | It("fails with a generic error", func() { 42 | When(httpClient.Do(AnyPtrToHttpRequest())).ThenReturn(nil, fmt.Errorf("Some network error")) 43 | 44 | e := updater.NotifyProcessingUpload("abc") 45 | 46 | Expect(e).To(MatchError(SatisfyAll( 47 | ContainSubstring("Could not make request against CC"), 48 | ContainSubstring("abc"), 49 | ContainSubstring("Some network error"), 50 | ))) 51 | }) 52 | }) 53 | 54 | Context("http client returns NotFound", func() { 55 | It("fails with a generic error", func() { 56 | When(httpClient.Do(AnyPtrToHttpRequest())).ThenReturn(&http.Response{StatusCode: http.StatusNotFound}, nil) 57 | 58 | e := updater.NotifyProcessingUpload("abc") 59 | 60 | Expect(e).To(Equal(bitsgo.NewNotFoundError())) 61 | }) 62 | }) 63 | }) 64 | 65 | Describe("NotifyUploadSucceeded", func() { 66 | It("works", func() { 67 | When(httpClient.Do(AnyPtrToHttpRequest())).ThenReturn(&http.Response{}, nil) 68 | 69 | e := updater.NotifyUploadSucceeded("abc", "sha1", "sha256") 70 | 71 | Expect(e).NotTo(HaveOccurred()) 72 | 73 | request := httpClient.VerifyWasCalledOnce().Do(AnyPtrToHttpRequest()).GetCapturedArguments() 74 | Expect(request.Method).To(Equal("PATCH")) 75 | Expect(request.URL.String()).To(Equal("http://example.com/some/endpoint/abc")) 76 | Expect(ioutil.ReadAll(request.Body)).To(MatchJSON(`{ 77 | "state": "READY", 78 | "checksums": [ 79 | { 80 | "type": "sha1", 81 | "value": "sha1" 82 | }, 83 | { 84 | "type": "sha256", 85 | "value": "sha256" 86 | } 87 | ] 88 | }`)) 89 | }) 90 | }) 91 | 92 | Describe("NotifyUploadFailed", func() { 93 | It("works", func() { 94 | When(httpClient.Do(AnyPtrToHttpRequest())).ThenReturn(&http.Response{}, nil) 95 | 96 | e := updater.NotifyUploadFailed("abc", fmt.Errorf("some error")) 97 | 98 | Expect(e).NotTo(HaveOccurred()) 99 | 100 | request := httpClient.VerifyWasCalledOnce().Do(AnyPtrToHttpRequest()).GetCapturedArguments() 101 | Expect(request.Method).To(Equal("PATCH")) 102 | Expect(request.URL.String()).To(Equal("http://example.com/some/endpoint/abc")) 103 | Expect(ioutil.ReadAll(request.Body)).To(MatchJSON(`{ 104 | "state": "FAILED", 105 | "error": "some error" 106 | }`)) 107 | }) 108 | }) 109 | }) 110 | -------------------------------------------------------------------------------- /ccupdater/matchers/ptr_to_http_request.go: -------------------------------------------------------------------------------- 1 | // Code generated by pegomock. DO NOT EDIT. 2 | package matchers 3 | 4 | import ( 5 | "reflect" 6 | "github.com/petergtz/pegomock" 7 | http "net/http" 8 | ) 9 | 10 | func AnyPtrToHttpRequest() *http.Request { 11 | pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(*http.Request))(nil)).Elem())) 12 | var nullValue *http.Request 13 | return nullValue 14 | } 15 | 16 | func EqPtrToHttpRequest(value *http.Request) *http.Request { 17 | pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) 18 | var nullValue *http.Request 19 | return nullValue 20 | } 21 | -------------------------------------------------------------------------------- /ccupdater/matchers/ptr_to_http_response.go: -------------------------------------------------------------------------------- 1 | // Code generated by pegomock. DO NOT EDIT. 2 | package matchers 3 | 4 | import ( 5 | "reflect" 6 | "github.com/petergtz/pegomock" 7 | http "net/http" 8 | ) 9 | 10 | func AnyPtrToHttpResponse() *http.Response { 11 | pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(*http.Response))(nil)).Elem())) 12 | var nullValue *http.Response 13 | return nullValue 14 | } 15 | 16 | func EqPtrToHttpResponse(value *http.Response) *http.Response { 17 | pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) 18 | var nullValue *http.Response 19 | return nullValue 20 | } 21 | -------------------------------------------------------------------------------- /ccupdater/mock_httpclient_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by pegomock. DO NOT EDIT. 2 | // Source: github.com/cloudfoundry-incubator/bits-service/ccupdater (interfaces: HttpClient) 3 | 4 | package ccupdater_test 5 | 6 | import ( 7 | pegomock "github.com/petergtz/pegomock" 8 | http "net/http" 9 | "reflect" 10 | ) 11 | 12 | type MockHttpClient struct { 13 | fail func(message string, callerSkip ...int) 14 | } 15 | 16 | func NewMockHttpClient() *MockHttpClient { 17 | return &MockHttpClient{fail: pegomock.GlobalFailHandler} 18 | } 19 | 20 | func (mock *MockHttpClient) Do(_param0 *http.Request) (*http.Response, error) { 21 | params := []pegomock.Param{_param0} 22 | result := pegomock.GetGenericMockFrom(mock).Invoke("Do", params, []reflect.Type{reflect.TypeOf((**http.Response)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) 23 | var ret0 *http.Response 24 | var ret1 error 25 | if len(result) != 0 { 26 | if result[0] != nil { 27 | ret0 = result[0].(*http.Response) 28 | } 29 | if result[1] != nil { 30 | ret1 = result[1].(error) 31 | } 32 | } 33 | return ret0, ret1 34 | } 35 | 36 | func (mock *MockHttpClient) VerifyWasCalledOnce() *VerifierHttpClient { 37 | return &VerifierHttpClient{mock, pegomock.Times(1), nil} 38 | } 39 | 40 | func (mock *MockHttpClient) VerifyWasCalled(invocationCountMatcher pegomock.Matcher) *VerifierHttpClient { 41 | return &VerifierHttpClient{mock, invocationCountMatcher, nil} 42 | } 43 | 44 | func (mock *MockHttpClient) VerifyWasCalledInOrder(invocationCountMatcher pegomock.Matcher, inOrderContext *pegomock.InOrderContext) *VerifierHttpClient { 45 | return &VerifierHttpClient{mock, invocationCountMatcher, inOrderContext} 46 | } 47 | 48 | type VerifierHttpClient struct { 49 | mock *MockHttpClient 50 | invocationCountMatcher pegomock.Matcher 51 | inOrderContext *pegomock.InOrderContext 52 | } 53 | 54 | func (verifier *VerifierHttpClient) Do(_param0 *http.Request) *HttpClient_Do_OngoingVerification { 55 | params := []pegomock.Param{_param0} 56 | methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "Do", params) 57 | return &HttpClient_Do_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} 58 | } 59 | 60 | type HttpClient_Do_OngoingVerification struct { 61 | mock *MockHttpClient 62 | methodInvocations []pegomock.MethodInvocation 63 | } 64 | 65 | func (c *HttpClient_Do_OngoingVerification) GetCapturedArguments() *http.Request { 66 | _param0 := c.GetAllCapturedArguments() 67 | return _param0[len(_param0)-1] 68 | } 69 | 70 | func (c *HttpClient_Do_OngoingVerification) GetAllCapturedArguments() (_param0 []*http.Request) { 71 | params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) 72 | if len(params) > 0 { 73 | _param0 = make([]*http.Request, len(params[0])) 74 | for u, param := range params[0] { 75 | _param0[u] = param.(*http.Request) 76 | } 77 | } 78 | return 79 | } 80 | -------------------------------------------------------------------------------- /cmd/dashboard/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "sort" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | type LogEntry struct { 14 | Timestamp float64 `json:"ts"` 15 | Message string `json:"msg"` 16 | VcapRequestID string `json:"vcap-request-id"` 17 | RequestID int64 `json:"request-id"` 18 | } 19 | 20 | type TimestampAndEntry struct { 21 | timestamp float64 22 | value LogEntry 23 | } 24 | 25 | func main() { 26 | filename := os.Args[1] 27 | file, e := os.Open(filename) 28 | if e != nil { 29 | panic(e) 30 | } 31 | defer file.Close() 32 | 33 | scanner := bufio.NewScanner(file) 34 | buf := make([]byte, 100*1024*1024) 35 | scanner.Buffer(buf, 100*1024*1024) 36 | logEntries := make(map[int64]LogEntry) 37 | for scanner.Scan() { 38 | var entry LogEntry 39 | e := json.Unmarshal(scanner.Bytes(), &entry) 40 | if e != nil { 41 | continue 42 | } 43 | if strings.Contains(entry.Message, "HTTP Request started") { 44 | logEntries[entry.RequestID] = entry 45 | } 46 | if strings.Contains(entry.Message, "HTTP Request completed") { 47 | delete(logEntries, entry.RequestID) 48 | } 49 | 50 | if _, exists := logEntries[entry.RequestID]; exists { 51 | logEntries[entry.RequestID] = entry 52 | } 53 | } 54 | sortedEntries := make([]TimestampAndEntry, 0) 55 | for _, request := range logEntries { 56 | sortedEntries = append(sortedEntries, TimestampAndEntry{request.Timestamp, request}) 57 | } 58 | sort.Slice(sortedEntries, func(i int, j int) bool { return sortedEntries[i].timestamp < sortedEntries[j].timestamp }) 59 | 60 | for _, entry := range sortedEntries { 61 | fmt.Printf("%v: (id: %v, vcap-request-id: %v): %v\n", 62 | time.Unix(int64(entry.value.Timestamp), 0), 63 | entry.value.RequestID, 64 | entry.value.VcapRequestID, 65 | entry.value.Message) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /docs/README.markdown: -------------------------------------------------------------------------------- 1 | The sequence charts were generated with [websequencediagrams](https://www.websequencediagrams.com/) and can be regenerated with `rake docs/create-app.png` etc. 2 | -------------------------------------------------------------------------------- /docs/create-app-with-bits-service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-app-with-bits-service.png -------------------------------------------------------------------------------- /docs/create-app-with-bits-service.txt: -------------------------------------------------------------------------------- 1 | title Create App with Bits-Service 2 | 3 | cf->CC: PUT /v2/apps/:guid 4 | CC-->cf: 201 5 | cf->CC: PUT /v2/resource_match [JSON of known file SHAs] 6 | 7 | CC->Bits-Service: POST /app_stash/matches [JSON of known file SHAs] 8 | loop each SHA 9 | Bits-Service->Blobstore: HEAD file 10 | Blobstore-->Bits-Service: 11 | end 12 | Bits-Service-->CC: 13 | 14 | CC-->cf: files found 15 | cf->cf: create ZIP with missing bits 16 | cf->CC: POST /v2/apps/:guid/bits [zip file + SHAs known to CC] 17 | 18 | CC->Bits-Service: POST /app_stash/entries [zip file] 19 | Bits-Service->Bits-Service: unzip 20 | loop unzipped files 21 | Bits-Service->Blobstore: store 22 | Blobstore-->Bits-Service: 23 | end 24 | Bits-Service-->CC: [SHAs of files from zip] 25 | 26 | CC->CC: Collect list of _all_ SHAs 27 | 28 | CC->Bits-Service: POST /app_stash/bundle [all SHAs] 29 | 30 | loop existing files 31 | Bits-Service->Blobstore: fetch file 32 | Blobstore-->Bits-Service: file 33 | end 34 | 35 | Bits-Service->Bits-Service: assemble package 36 | Bits-Service-->CC: [package] 37 | -------------------------------------------------------------------------------- /docs/create-app-with-bits-service_proposal_for_cli.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-app-with-bits-service_proposal_for_cli.png -------------------------------------------------------------------------------- /docs/create-app-with-bits-service_proposal_for_cli.txt: -------------------------------------------------------------------------------- 1 | # TODO 2 | # * async: Background / Polling? 3 | # * GET statt POST /signed/app_stash/matches 4 | # * CAPI 5 | # - return signed URLs to CLI for /app_stash/matches and /app_stash/entries 6 | # - who gets info about package ready? 7 | 8 | title Create v2 App with Bits-Service (Direct CLI Upload) 9 | cf->CC: GET /v2/info 10 | CC-->cf: $BITS_ENDPOINT 11 | 12 | cf->CC: PUT /v2/apps/:guid 13 | CC->Bits-Service: GET /sign/app_stash/matches 14 | Bits-Service-->CC: 201 $SIGNED_ENTRIES_URL 15 | 16 | CC->Bits-Service: GET /sign/app_stash/entries 17 | Bits-Service-->CC: 201 $SIGNED_BUNDLE_URL 18 | 19 | CC-->cf: 201 $APP_GUID, $SIGNED_ENTRIES_URL, $SIGNED_BUNDLE_URL 20 | cf->Bits-Service: POST /signed/app_stash/matches [JSON of known file SHAs] 21 | loop each SHA 22 | Bits-Service->Blobstore: HEAD file 23 | Blobstore-->Bits-Service: 24 | end 25 | Bits-Service-->cf: files found 26 | 27 | cf->cf: create ZIP with missing bits 28 | 29 | # start upload zip archive from cli 30 | # [Not implemented yet in CC] 31 | #what is with the sha? POST /v2/apps/:guid/bits [zip file + SHAs known to CC] 32 | 33 | cf->Bits-Service: POST /signed/app_stash/entries [zip file] 34 | Bits-Service->Bits-Service: unzip 35 | loop unzipped files 36 | Bits-Service->Blobstore: store 37 | Blobstore-->Bits-Service: 38 | end 39 | 40 | #who bundles? this is open question 41 | Bits-Service-->cf: [SHAs of files from zip] 42 | 43 | cf->cf: Collect list of all SHAs 44 | cf->Bits-Service: POST /signed/app_stash/bundle [all SHAs] 45 | 46 | loop existing files 47 | Bits-Service->Blobstore: fetch file 48 | Blobstore-->Bits-Service: file 49 | end 50 | 51 | Bits-Service->Bits-Service: assemble package 52 | Bits-Service-->cf: 201 53 | 54 | cf->CC: Ready to start 55 | -------------------------------------------------------------------------------- /docs/create-app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-app.png -------------------------------------------------------------------------------- /docs/create-app.txt: -------------------------------------------------------------------------------- 1 | title cf push --no-start 2 | 3 | cf->CC: POST /v2/apps 4 | CC-->cf: 201 5 | 6 | cf->CC: PUT /v2/resource_match 7 | loop fingerprints 8 | CC->Blobstore: HEAD file 9 | Blobstore-->CC: [200, 404] 10 | end 11 | CC-->cf: found fingerprints 12 | 13 | cf->cf: ZIP missing files 14 | cf->CC: PUT /v2/apps/:app-guid/bits 15 | 16 | loop existing files 17 | CC->Blobstore: fetch file 18 | Blobstore-->CC: file 19 | end 20 | 21 | loop new files 22 | CC->Blobstore: store file 23 | Blobstore-->CC: 201 24 | end 25 | CC-->cf: 201 26 | 27 | CC->>CC: assemble package 28 | 29 | CC->>Blobstore: store package 30 | Blobstore-->CC: 201 31 | -------------------------------------------------------------------------------- /docs/create-droplet-with-bits-service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-droplet-with-bits-service.png -------------------------------------------------------------------------------- /docs/create-droplet-with-bits-service.txt: -------------------------------------------------------------------------------- 1 | title Create Droplet with Bits-Service 2 | 3 | note over DEA,CC,Bits-Service,Blobstore: stage app 4 | DEA->CC: POST /staging/droplets//upload 5 | CC->Bits-Service: upload file 6 | Bits-Service->Bits-Service: calculate digest (SHA) 7 | Bits-Service->Blobstore: upload file 8 | Bits-Service-->CC: {:guid, :digest} 9 | 10 | note over DEA,CC,Bits-Service,Blobstore: run app 11 | CC-->>DEA: NATs message to start app {:guid, :digest} 12 | DEA->CC: GET /staging/droplets//download 13 | CC-->DEA: download url 14 | DEA->Bits-Service: fetch file 15 | Bits-Service->Blobstore: download file 16 | Bits-Service-->DEA: bits 17 | -------------------------------------------------------------------------------- /docs/create-droplet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-droplet.png -------------------------------------------------------------------------------- /docs/create-droplet.txt: -------------------------------------------------------------------------------- 1 | title Create Droplet Today 2 | 3 | note over DEA,CC,Blobstore: stage app 4 | DEA->CC: POST /staging/droplets//upload 5 | CC->Blobstore: upload file 6 | note over DEA,CC,Blobstore: run app 7 | DEA->CC: GET /staging/droplets//download 8 | CC-->DEA: download url 9 | DEA->Blobstore: fetch file 10 | -------------------------------------------------------------------------------- /docs/create-v3-app-with-bits-service-async-no-resource-match.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-v3-app-with-bits-service-async-no-resource-match.png -------------------------------------------------------------------------------- /docs/create-v3-app-with-bits-service-async-no-resource-match.txt: -------------------------------------------------------------------------------- 1 | title Create App (V3) with bits-service (async, no resource matching) 2 | 3 | cf->CC: POST /v3/apps 4 | CC-->cf: APP_GUID 5 | note over Bits-Service,Bits-Service-Upload-Queue: This is one process 6 | cf->CC: POST /v3/apps/$APP_GUID/packages 7 | CC-->CC: Generate\nPACKAGE_GUID 8 | CC->Bits-Service: GET /sign/packages/$PACKAGE_GUID 9 | activate Bits-Service 10 | Bits-Service-->CC: UPLOAD_URL (signed) 11 | deactivate Bits-Service 12 | CC-->cf: 201\n$UPLOAD_URL\n$PACKAGE_SELF_URL (for package state polling) 13 | 14 | cf->cf: create package.zip 15 | cf->Bits-Service: PUT $UPLOAD_URL package.zip 16 | activate Bits-Service 17 | Bits-Service->Bits-Service: extract PACKAGE_GUID\n from $UPLOAD_URL 18 | Bits-Service->Bits-Service-Upload-Queue: enqueue package upload job 19 | Bits-Service->CC: package PROCESSING $PACKAGE_GUID 20 | Bits-Service-->cf: 201 21 | deactivate Bits-Service 22 | cf->CC: GET $PACKAGE_SELF_URL 23 | CC-->cf: 200 package PROCESSING_UPLOAD 24 | note over cf,CC: loop until\npackage is READY 25 | activate Bits-Service-Upload-Queue 26 | Bits-Service-Upload-Queue->Bits-Service-Upload-Queue: dequeue package upload job 27 | Bits-Service-Upload-Queue->Bits-Service-Upload-Queue: calc SHA1_PACKAGE_HASH,\nSHA256_PACKAGE_HASH 28 | Bits-Service-Upload-Queue->Blobstore: PUT package.zip 29 | Blobstore-->Bits-Service-Upload-Queue: 201 30 | Bits-Service-Upload-Queue->CC: package $PACKAGE_GUID READY\n$SHA1_PACKAGE_HASH, $SHA256_PACKAGE_HASH 31 | deactivate Bits-Service-Upload-Queue 32 | cf->CC: GET $PACKAGE_SELF_URL 33 | CC-->cf: 200 package READY 34 | -------------------------------------------------------------------------------- /docs/create-v3-app-with-bits-service-async-with-resource-match.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-v3-app-with-bits-service-async-with-resource-match.png -------------------------------------------------------------------------------- /docs/create-v3-app-with-bits-service-async-with-resource-match.txt: -------------------------------------------------------------------------------- 1 | title Create App (V3) with bits-service (async, with resource matching) 2 | 3 | cf->CC: POST /v3/apps 4 | CC-->cf: APP_GUID 5 | note over Bits-Service,Bits-Service-Upload-Queue: This is one process 6 | cf->CC: POST /v3/apps/$APP_GUID/packages 7 | CC-->CC: Generate\nPACKAGE_GUID 8 | CC->Bits-Service: GET /sign/packages/$PACKAGE_GUID 9 | activate Bits-Service 10 | Bits-Service-->CC: Signed URLs:\n$UPLOAD_URL\n$RESOURCE_MATCHING_URLs 11 | deactivate Bits-Service 12 | CC-->cf: 201\n$UPLOAD_URL\n$PACKAGE_SELF_URL (for package state polling)\nRESOURCE_MATCHING_URLS 13 | 14 | cf->Bits-Service: POST $RESOURCE_MATCH_URL [JSON of known file SHAs] 15 | Bits-Service-->cf: 200 files found 16 | cf->Bits-Service: POST $RESOURCE_ENTRIES_URL [missing files as zip] 17 | Bits-Service-->cf: 200 18 | 19 | cf->Bits-Service: POST $RESOURCE_BUNDLES_URL [list of fingerprints to create package from +\nname of package to create] 20 | activate Bits-Service 21 | Bits-Service->Bits-Service: extract PACKAGE_GUID\n from $UPLOAD_URL 22 | Bits-Service->Bits-Service-Upload-Queue: enqueue package creation job\n[list of fingerprints] 23 | Bits-Service->CC: package PROCESSING $PACKAGE_GUID 24 | Bits-Service-->cf: 201 25 | deactivate Bits-Service 26 | cf->CC: GET $PACKAGE_SELF_URL 27 | CC-->cf: 200 package PROCESSING_UPLOAD 28 | note over cf,CC: loop until\npackage is READY 29 | Bits-Service-Upload-Queue->Bits-Service-Upload-Queue: dequeue package upload job 30 | 31 | activate Bits-Service-Upload-Queue 32 | Bits-Service-Upload-Queue->Bits-Service-Upload-Queue: assemble package.zip 33 | Bits-Service-Upload-Queue->Blobstore: PUT package.zip 34 | Blobstore-->Bits-Service-Upload-Queue: 201 35 | Bits-Service-Upload-Queue->CC: package $PACKAGE_GUID READY\n$SHA1_PACKAGE_HASH, $SHA256_PACKAGE_HASH 36 | deactivate Bits-Service-Upload-Queue 37 | cf->CC: GET $PACKAGE_SELF_URL 38 | CC-->cf: 200 package READY 39 | -------------------------------------------------------------------------------- /docs/create-v3-app-with-bits-service-sync-no-resource-match.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-v3-app-with-bits-service-sync-no-resource-match.png -------------------------------------------------------------------------------- /docs/create-v3-app-with-bits-service-sync-no-resource-match.txt: -------------------------------------------------------------------------------- 1 | title Create App (V3) with bits-service 2 | 3 | cf->CC: POST /v3/apps 4 | CC-->cf: APP_GUID 5 | cf->CC: POST /v3/apps/$APP_GUID/packages 6 | CC-->CC: Generate\nPACKAGE_GUID 7 | CC->Bits-Service: GET /sign/packages/$PACKAGE_GUID 8 | activate Bits-Service 9 | Bits-Service-->CC: UPLOAD_URL (signed) 10 | deactivate Bits-Service 11 | CC-->cf: 201 $UPLOAD_URL 12 | 13 | cf->cf: create package.zip 14 | cf->Bits-Service: POST $UPLOAD_URL package.zip 15 | activate Bits-Service 16 | Bits-Service->Bits-Service: extract PACKAGE_GUID\n from $UPLOAD_URL 17 | Bits-Service->CC: package PROCESSING $PACKAGE_GUID 18 | Bits-Service->Bits-Service: calc SHA1_PACKAGE_HASH,\nSHA256_PACKAGE_HASH 19 | Bits-Service->Blobstore: PUT package.zip 20 | Bits-Service->CC: package $PACKAGE_GUID READY\n$SHA1_PACKAGE_HASH, $SHA256_PACKAGE_HASH 21 | deactivate Bits-Service 22 | Bits-Service-->cf: 201 23 | CC-->cf: ready 24 | -------------------------------------------------------------------------------- /docs/create-v3-app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/docs/create-v3-app.png -------------------------------------------------------------------------------- /docs/create-v3-app.txt: -------------------------------------------------------------------------------- 1 | # Source: github.com/cloudfoundry/cloud_controller_ng/wiki/How-to-Create-an-App-Using-V3-of-the-CC-API 2 | 3 | title Create V3 App (without Bits-Service) 4 | 5 | cf->CC: POST /v3/apps 6 | CC-->cf: app_guid 7 | cf->CC: POST /v3/packages 8 | CC-->cf: package_guid 9 | cf->cf: create package.zip 10 | cf->CC: POST /v3/packages/:package_guid/upload \n-F bits=@"package.zip" 11 | CC->Blobstore: [package] 12 | cf->CC: POST /v3/builds :package_guid 13 | CC-->cf: build_guid 14 | CC->stager: stage 15 | loop until STAGED 16 | cf->CC: GET /v3/builds/:build_guid 17 | end 18 | cf->CC: PATCH /v3/apps/:app-guid/relationships/current_droplet\n :droplet_guid 19 | cf->CC: POST /v2/routes 20 | CC-->cf: route_guid 21 | cf->CC: PUT /v2/routes/:route_guid/apps/:app_guid 22 | cf->CC: POST /v3/apps/:app_guid/actions/start 23 | -------------------------------------------------------------------------------- /docs/onboarding.markdown: -------------------------------------------------------------------------------- 1 | # Onboarding a new Team Member 2 | 3 | * Copy this section into a new tracker chore 4 | * Get access to [cloudfoundry.slack.com](https://slack.cloudfoundry.org/) 5 | * Create an image for the tracker board (ask Steffen) 6 | * Get access to the [tracker](https://www.pivotaltracker.com/n/projects/1406862) (ask the PM) 7 | * Add team member to a group with access to the [github repo](https://github.com/cloudfoundry-incubator/bits-service) (ask the PM) 8 | * Add public SSH key to [github.com](https://help.github.com/articles/connecting-to-github-with-ssh/) and verify 9 | * Create a [SL](https://control.softlayer.com) account (ask the PM) 10 | * Create a [new VPN password](https://control.softlayer.com/account/user/profile) (In the user settings of your Softlayer Account you can set up the VPN password in the 'Log In Settings' section) 11 | * Set up the VPN client - Install "Motion Pro Plus" from AppStore directly and follow the [instructions](https://console.bluemix.net/docs/infrastructure/iaas-vpn/standalone-vpn-clients.html) 12 | * Get access to shared Lastpass folder (ask the PM or anchor) 13 | * Invite team member to Bluemix Flintstone Account, Cloud Foundry Flintstone Org, performance tests Space 14 | * [Install git hooks](#install-git-hooks) 15 | * Update [the onboarding document](https://github.com/cloudfoundry-incubator/bits-service/blob/master/docs/onboarding.markdown) if necessary 16 | 17 | # Team Communication 18 | 19 | * [#bits-service](https://cloudfoundry.slack.com/messages/bits-service/) 20 | 21 | # BOSH 22 | 23 | * Bring up the VPN 24 | * Point BOSH cli at the director: 25 | 26 | ``` 27 | bosh target https://10.155.248.165:25555 28 | ``` 29 | 30 | * If the IP address doesn't match, check the [device list](https://control.softlayer.com/devices) 31 | 32 | * To access the SL bosh-lite director from your working station: 33 | 34 | ``` 35 | ssh -L 25555:192.168.50.4:25555 root@10.155.248.181 36 | # And in another terminal : 37 | bosh target https://localhost:25555 38 | ``` 39 | 40 | # Concourse 41 | 42 | Our pipeline is public at [flintstone.ci.cf-app.com](https://ci.flintstone.cf.cloud.ibm.com). 43 | 44 | ``` 45 | # name the target 'flintstone' and login with password from the Lastpass CLI. 46 | fly --target flintstone login --concourse-url http://10.155.248.166:8080 --user admin --password $(lpass show concourse --password) 47 | 48 | # if the auth expired, re-login using the previously named target 49 | fly -t flintstone login 50 | 51 | # create or update a pipeline from yaml file 52 | fly -t flintstone set-pipeline -p test-exists -c test-exists.yml 53 | 54 | # destroy a pipeline 55 | fly -t flintstone destroy-pipeline -p test-exists 56 | 57 | # hijack into a job 58 | fly intercept -t flintstone --job bits-service/run-tests 59 | 60 | # let fly offer which container to hijack. Also, use sh instead of bash for busybox-based containers. 61 | fly intercept -t flintstone sh 62 | 63 | # run a single task with local changes without having to commit to git before 64 | fly execute -t flintstone --config ci/tasks/run-tests.yml --input=git-bits-service=. 65 | 66 | # same, but with two inputs 67 | fly execute -t flintstone --config ci/tasks/upload-to-object-storage.yml --input=git-bits-service-release=. --input=releases=dev_releases/bits-service 68 | ``` 69 | 70 | # Install git hooks 71 | 72 | We use this hook to prevent accidential commits of secrets: 73 | 74 | ```bash 75 | cd path/to/repo 76 | ~/workspace/bits-service/scripts/install-git-hooks.sh 77 | ``` 78 | 79 | This needs to be done for each repo where the hook should run, e.g. with 80 | 81 | ```bash 82 | for d in ~/workspace/bits-service*; do 83 | ( 84 | cd "$d" 85 | ~/workspace/bits-service/scripts/install-git-hooks.sh 86 | ) 87 | done 88 | ``` 89 | -------------------------------------------------------------------------------- /docs/websequencediagram: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # Render textual input as web sequence diagram 6 | # 7 | # Reads the text from the file given by the first argument and writes the result to the file given by the second argument. 8 | # 9 | # Example: 10 | # 11 | # websequencediagram create-v3-package-with-bits-service.txt create-v3-package-with-bits-service.png 12 | # 13 | # adapted from https://www.websequencediagrams.com/embedding.html#ruby 14 | # 15 | src = File.read(ARGV[0]) 16 | target = ARGV[1] 17 | 18 | require 'net/http' 19 | require 'yaml' 20 | require 'uri' 21 | require 'open-uri' 22 | 23 | response = Net::HTTP.post_form( 24 | URI.parse('http://www.websequencediagrams.com/index.php'), 25 | 'style' => 'default', 26 | 'message' => src 27 | ) 28 | 29 | appendix = YAML.safe_load(response.body)['img'] 30 | File.write(target, open("http://www.websequencediagrams.com/#{appendix}").read) 31 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/cloudfoundry-incubator/bits-service 2 | import: 3 | - package: github.com/aws/aws-sdk-go 4 | subpackages: 5 | - aws 6 | - aws/awserr 7 | - aws/credentials 8 | - aws/request 9 | - aws/session 10 | - service/s3 11 | - package: github.com/gorilla/mux 12 | - package: github.com/onsi/gomega 13 | subpackages: 14 | - gstruct 15 | - types 16 | - package: github.com/pkg/errors 17 | - package: github.com/urfave/negroni 18 | - package: gopkg.in/alecthomas/kingpin.v2 19 | - package: gopkg.in/yaml.v2 20 | - package: go.uber.org/zap 21 | - package: code.cloudfoundry.org/bytefmt 22 | - package: github.com/benbjohnson/clock 23 | - package: github.com/tecnickcom/statsd 24 | - package: cloud.google.com/go 25 | subpackages: 26 | - storage 27 | - package: github.com/Azure/azure-sdk-for-go 28 | - package: github.com/ncw/swift 29 | - package: github.com/cenkalti/backoff 30 | - package: github.com/aliyun/aliyun-oss-go-sdk 31 | subpackages: 32 | - oss 33 | - package: golang.org/x/sync 34 | subpackages: 35 | - semaphore 36 | - package: github.com/satori/go.uuid 37 | version: ^1.2.0 38 | testImport: 39 | - package: github.com/onsi/ginkgo 40 | - package: github.com/petergtz/pegomock 41 | -------------------------------------------------------------------------------- /httputil/httputil.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "mime/multipart" 8 | "net/http" 9 | "net/url" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type Request struct { 15 | http.Request 16 | } 17 | 18 | func (request Request) Build() *http.Request { 19 | return &request.Request 20 | } 21 | 22 | func NewRequest(method, urlStr string, body io.Reader) *Request { 23 | request, e := http.NewRequest(method, urlStr, body) 24 | if e != nil { 25 | panic(e) 26 | } 27 | return &Request{*request} 28 | } 29 | 30 | func (request *Request) WithBasicAuth(username, password string) *Request { 31 | request.SetBasicAuth(username, password) 32 | return request 33 | } 34 | 35 | func (request *Request) WithHeader(key, value string) *Request { 36 | request.Header.Add(key, value) 37 | return request 38 | } 39 | 40 | func NewPutRequest(url string, formFiles map[string]map[string]io.Reader) (*http.Request, error) { 41 | bodyBuf := &bytes.Buffer{} 42 | contentType, e := AddFormFileTo(bodyBuf, formFiles) 43 | if e != nil { 44 | return nil, errors.Wrapf(e, "url=%v", url) 45 | } 46 | request, e := http.NewRequest("PUT", url, bodyBuf) 47 | if e != nil { 48 | return nil, errors.Wrapf(e, "url=%v", url) 49 | } 50 | request.Header.Add("Content-Type", contentType) 51 | return request, nil 52 | } 53 | 54 | func NewPostRequest(url string, formFiles map[string]map[string]io.Reader) (*http.Request, error) { 55 | bodyBuf := &bytes.Buffer{} 56 | contentType, e := AddFormFileTo(bodyBuf, formFiles) 57 | if e != nil { 58 | return nil, errors.Wrapf(e, "url=%v", url) 59 | } 60 | request, e := http.NewRequest("POST", url, bodyBuf) 61 | if e != nil { 62 | return nil, errors.Wrapf(e, "url=%v", url) 63 | } 64 | request.Header.Add("Content-Type", contentType) 65 | return request, nil 66 | } 67 | 68 | func AddFormFileTo(body io.Writer, formFiles map[string]map[string]io.Reader) (contentType string, err error) { 69 | multipartWriter := multipart.NewWriter(body) 70 | for name, filenameAndReader := range formFiles { 71 | for filename, reader := range filenameAndReader { 72 | formFileWriter, e := multipartWriter.CreateFormFile(name, filename) 73 | if e != nil { 74 | err = fmt.Errorf("Could not CreateFormFile with name %v and filename %v", name, filename) 75 | return 76 | } 77 | io.Copy(formFileWriter, reader) 78 | } 79 | } 80 | multipartWriter.Close() 81 | contentType = multipartWriter.FormDataContentType() 82 | return 83 | } 84 | 85 | func MustParse(rawUrl string) *url.URL { 86 | u, e := url.ParseRequestURI(rawUrl) 87 | if e != nil { 88 | panic(e) 89 | } 90 | return u 91 | } 92 | -------------------------------------------------------------------------------- /images/GET-request-speed-comparison-10-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/images/GET-request-speed-comparison-10-1.png -------------------------------------------------------------------------------- /images/GET-request-speed-comparison-100-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/images/GET-request-speed-comparison-100-10.png -------------------------------------------------------------------------------- /images/GET-request-speed-comparison-100-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/images/GET-request-speed-comparison-100-100.png -------------------------------------------------------------------------------- /images/PUT-request-speed-comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/images/PUT-request-speed-comparison.png -------------------------------------------------------------------------------- /images/bits-service-ruby-mem-consumption.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/images/bits-service-ruby-mem-consumption.png -------------------------------------------------------------------------------- /images/routes-and-stores.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/images/routes-and-stores.png -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "net/http" 7 | 8 | "go.uber.org/zap" 9 | ) 10 | 11 | var Log = setUpDefaultLogger().Sugar() 12 | 13 | func setUpDefaultLogger() *zap.Logger { 14 | cfg := zap.NewDevelopmentConfig() 15 | cfg.DisableStacktrace = true 16 | logger, e := cfg.Build() 17 | if e != nil { 18 | panic(e) 19 | } 20 | return logger 21 | } 22 | 23 | func SetLogger(logger *zap.Logger) { 24 | Log = logger.Sugar() 25 | } 26 | 27 | func From(r *http.Request) *zap.SugaredLogger { 28 | log := r.Context().Value("logger") 29 | if log != nil { 30 | return log.(*zap.SugaredLogger) 31 | } else { 32 | return Log 33 | } 34 | } 35 | 36 | // Copied from go.uber.org/zap/global.go and changed to use Error instead of Info: 37 | func NewStdLog(l *zap.Logger) *log.Logger { 38 | const ( 39 | _stdLogDefaultDepth = 2 40 | _loggerWriterDepth = 1 41 | ) 42 | return log.New(&loggerWriter{l.WithOptions( 43 | zap.AddCallerSkip(_stdLogDefaultDepth + _loggerWriterDepth), 44 | )}, "" /* prefix */, 0 /* flags */) 45 | } 46 | 47 | type loggerWriter struct{ logger *zap.Logger } 48 | 49 | func (l *loggerWriter) Write(p []byte) (int, error) { 50 | p = bytes.TrimSpace(p) 51 | l.logger.Error(string(p)) 52 | return len(p), nil 53 | } 54 | -------------------------------------------------------------------------------- /matchers/slice_of_byte.go: -------------------------------------------------------------------------------- 1 | // Code generated by pegomock. DO NOT EDIT. 2 | package matchers 3 | 4 | import ( 5 | "reflect" 6 | "github.com/petergtz/pegomock" 7 | 8 | ) 9 | 10 | func AnySliceOfByte() []byte { 11 | pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*([]byte))(nil)).Elem())) 12 | var nullValue []byte 13 | return nullValue 14 | } 15 | 16 | func EqSliceOfByte(value []byte) []byte { 17 | pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) 18 | var nullValue []byte 19 | return nullValue 20 | } 21 | -------------------------------------------------------------------------------- /metrics_service.go: -------------------------------------------------------------------------------- 1 | package bitsgo 2 | 3 | import "time" 4 | 5 | type MetricsService interface { 6 | SendTimingMetric(name string, duration time.Duration) 7 | SendGaugeMetric(name string, value int64) 8 | SendCounterMetric(name string, value int64) 9 | } 10 | -------------------------------------------------------------------------------- /middlewares/basic_auth_middleware.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "crypto/subtle" 5 | "net/http" 6 | ) 7 | 8 | type Credential struct { 9 | Username, Password string 10 | } 11 | 12 | type BasicAuthMiddleware struct { 13 | credentials []Credential 14 | basicAuthHeaderMissingHandler http.Handler 15 | unauthorizedHandler http.Handler 16 | } 17 | 18 | func NewBasicAuthMiddleWare(credentials ...Credential) *BasicAuthMiddleware { 19 | return &BasicAuthMiddleware{credentials: credentials} 20 | } 21 | 22 | func (middleware *BasicAuthMiddleware) WithBasicAuthHeaderMissingHandler(handler http.Handler) *BasicAuthMiddleware { 23 | middleware.basicAuthHeaderMissingHandler = handler 24 | return middleware 25 | } 26 | 27 | func (middleware *BasicAuthMiddleware) WithUnauthorizedHandler(handler http.Handler) *BasicAuthMiddleware { 28 | middleware.unauthorizedHandler = handler 29 | return middleware 30 | } 31 | 32 | func (middleware *BasicAuthMiddleware) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request, next http.HandlerFunc) { 33 | username, password, ok := request.BasicAuth() 34 | if !ok { 35 | if middleware.basicAuthHeaderMissingHandler == nil { 36 | responseWriter.Header().Set("WWW-Authenticate", `Basic realm="bits-service"`) 37 | responseWriter.WriteHeader(http.StatusUnauthorized) 38 | return 39 | } 40 | middleware.basicAuthHeaderMissingHandler.ServeHTTP(responseWriter, request) 41 | return 42 | } 43 | 44 | if !middleware.authorized(username, password) { 45 | if middleware.unauthorizedHandler == nil { 46 | responseWriter.Header().Set("WWW-Authenticate", `Basic realm="bits-service"`) 47 | responseWriter.WriteHeader(http.StatusUnauthorized) 48 | return 49 | } 50 | middleware.unauthorizedHandler.ServeHTTP(responseWriter, request) 51 | } 52 | next(responseWriter, request) 53 | } 54 | 55 | func (middleware *BasicAuthMiddleware) authorized(username, password string) bool { 56 | for _, credential := range middleware.credentials { 57 | if subtle.ConstantTimeCompare([]byte(username), []byte(credential.Username)) == 1 && 58 | subtle.ConstantTimeCompare([]byte(password), []byte(credential.Password)) == 1 { 59 | return true 60 | } 61 | } 62 | return false 63 | } 64 | -------------------------------------------------------------------------------- /middlewares/body_size_limit_middleware.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "net/http" 8 | ) 9 | 10 | func NewBodySizeLimitMiddleware(bodySizeLimit uint64) *BodySizeLimitMiddleware { 11 | return &BodySizeLimitMiddleware{bodySizeLimit: bodySizeLimit} 12 | } 13 | 14 | type BodySizeLimitMiddleware struct { 15 | bodySizeLimit uint64 16 | } 17 | 18 | func (middleware *BodySizeLimitMiddleware) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request, next http.HandlerFunc) { 19 | if middleware.bodySizeLimit != 0 { 20 | if request.ContentLength == -1 { 21 | responseWriter.WriteHeader(http.StatusBadRequest) 22 | fmt.Fprintf(responseWriter, "Please provide the header field Content-Length") 23 | return 24 | } 25 | if uint64(request.ContentLength) > middleware.bodySizeLimit { 26 | 27 | // Reading the body here is really just to make Ruby's RestClient happy. 28 | // For some reason it crashes if we don't read the body. 29 | defer request.Body.Close() 30 | io.Copy(ioutil.Discard, request.Body) 31 | 32 | responseWriter.WriteHeader(http.StatusRequestEntityTooLarge) 33 | return 34 | } 35 | } 36 | next(responseWriter, request) 37 | } 38 | -------------------------------------------------------------------------------- /middlewares/body_size_limit_middleware_test.go: -------------------------------------------------------------------------------- 1 | package middlewares_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "net/http" 8 | "net/http/httptest" 9 | 10 | "github.com/cloudfoundry-incubator/bits-service/middlewares" 11 | "github.com/onsi/ginkgo" 12 | "github.com/onsi/gomega" 13 | "github.com/petergtz/pegomock" 14 | ) 15 | 16 | func TestMiddleWares(t *testing.T) { 17 | gomega.RegisterFailHandler(ginkgo.Fail) 18 | pegomock.RegisterMockFailHandler(ginkgo.Fail) 19 | ginkgo.RunSpecs(t, "Middleware") 20 | } 21 | 22 | var _ = Describe("BodySizeLimitMiddleWare", func() { 23 | Context("Body is larger than limit", func() { 24 | It("Fails when body is bigger than limit", func() { 25 | responseWriter := httptest.NewRecorder() 26 | 27 | middlewares.NewBodySizeLimitMiddleware(10).ServeHTTP( 28 | responseWriter, 29 | httptest.NewRequest("PUT", "/some/path", strings.NewReader("This is longer than the limit")), 30 | func(responseWriter http.ResponseWriter, request *http.Request) { 31 | Fail("Unexpected call to the next handler") 32 | }) 33 | 34 | Expect(responseWriter.Code).To(Equal(http.StatusRequestEntityTooLarge), responseWriter.Body.String()) 35 | }) 36 | 37 | }) 38 | 39 | Context("Body is smaller than limit", func() { 40 | It("succeeds", func() { 41 | responseWriter := httptest.NewRecorder() 42 | downstreamHandlerResponse := http.StatusOK 43 | 44 | middlewares.NewBodySizeLimitMiddleware(10000).ServeHTTP( 45 | responseWriter, 46 | httptest.NewRequest("PUT", "/some/path", strings.NewReader("This is shorter than the limit")), 47 | func(responseWriter http.ResponseWriter, request *http.Request) { 48 | responseWriter.WriteHeader(downstreamHandlerResponse) 49 | }) 50 | 51 | Expect(responseWriter.Code).To(Equal(downstreamHandlerResponse), responseWriter.Body.String()) 52 | }) 53 | }) 54 | 55 | Context("Limit is 0", func() { 56 | It("succeeds, even though body is technically larger than limi", func() { 57 | responseWriter := httptest.NewRecorder() 58 | downstreamHandlerResponse := http.StatusOK 59 | 60 | middlewares.NewBodySizeLimitMiddleware(0).ServeHTTP( 61 | responseWriter, 62 | httptest.NewRequest("PUT", "/some/path", strings.NewReader("This is longer than the limit")), 63 | func(responseWriter http.ResponseWriter, request *http.Request) { 64 | responseWriter.WriteHeader(downstreamHandlerResponse) 65 | }) 66 | 67 | Expect(responseWriter.Code).To(Equal(downstreamHandlerResponse), responseWriter.Body.String()) 68 | }) 69 | }) 70 | }) 71 | -------------------------------------------------------------------------------- /middlewares/logger_middleware.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "net/http" 5 | 6 | "math/rand" 7 | "time" 8 | 9 | "github.com/cloudfoundry-incubator/bits-service/util" 10 | "github.com/urfave/negroni" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | type ZapLoggerMiddleware struct { 15 | logger *zap.SugaredLogger 16 | } 17 | 18 | func NewZapLoggerMiddleware(logger *zap.SugaredLogger) *ZapLoggerMiddleware { 19 | rand.Seed(time.Now().Unix()) 20 | return &ZapLoggerMiddleware{logger: logger} 21 | } 22 | 23 | func (middleware *ZapLoggerMiddleware) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request, next http.HandlerFunc) { 24 | startTime := time.Now() 25 | 26 | requestId := rand.Int63() 27 | requestLogger := middleware.logger.With( 28 | "request-id", requestId, 29 | "vcap-request-id", request.Header.Get("X-Vcap-Request-Id")) 30 | 31 | requestLogger.Infow( 32 | "HTTP Request started", 33 | "host", request.Host, 34 | "method", request.Method, 35 | "path", request.URL.RequestURI(), 36 | "user-agent", request.UserAgent(), 37 | ) 38 | 39 | negroniResponseWriter, ok := responseWriter.(negroni.ResponseWriter) 40 | if !ok { 41 | negroniResponseWriter = negroni.NewResponseWriter(responseWriter) 42 | } 43 | 44 | next(negroniResponseWriter, util.RequestWithContextValues(request, 45 | "logger", requestLogger, 46 | "vcap-request-id", request.Header.Get("X-Vcap-Request-Id"), 47 | "request-id", requestId, 48 | )) 49 | 50 | fields := []interface{}{ 51 | "host", request.Host, 52 | "method", request.Method, 53 | "path", request.URL.RequestURI(), 54 | "status-code", negroniResponseWriter.Status(), 55 | "body-size", negroniResponseWriter.Size(), 56 | "duration", time.Since(startTime), 57 | "user-agent", request.UserAgent(), 58 | } 59 | if negroniResponseWriter.Status() >= 300 && negroniResponseWriter.Status() < 400 { 60 | fields = append(fields, zap.String("Location", negroniResponseWriter.Header().Get("Location"))) 61 | } 62 | requestLogger.Infow("HTTP Request completed", fields...) 63 | } 64 | -------------------------------------------------------------------------------- /middlewares/matchers/time_duration.go: -------------------------------------------------------------------------------- 1 | // Code generated by pegomock. DO NOT EDIT. 2 | package matchers 3 | 4 | import ( 5 | "reflect" 6 | "github.com/petergtz/pegomock" 7 | time "time" 8 | ) 9 | 10 | func AnyTimeDuration() time.Duration { 11 | pegomock.RegisterMatcher(pegomock.NewAnyMatcher(reflect.TypeOf((*(time.Duration))(nil)).Elem())) 12 | var nullValue time.Duration 13 | return nullValue 14 | } 15 | 16 | func EqTimeDuration(value time.Duration) time.Duration { 17 | pegomock.RegisterMatcher(&pegomock.EqMatcher{Value: value}) 18 | var nullValue time.Duration 19 | return nullValue 20 | } 21 | -------------------------------------------------------------------------------- /middlewares/metrics_middleware.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/urfave/negroni" 8 | 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "github.com/cloudfoundry-incubator/bits-service" 14 | ) 15 | 16 | type MetricsMiddleware struct { 17 | metricsService bitsgo.MetricsService 18 | } 19 | 20 | func NewMetricsMiddleware(metricsService bitsgo.MetricsService) *MetricsMiddleware { 21 | return &MetricsMiddleware{metricsService: metricsService} 22 | } 23 | 24 | func (middleware *MetricsMiddleware) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request, next http.HandlerFunc) { 25 | startTime := time.Now() 26 | negroniResponseWriter, ok := responseWriter.(negroni.ResponseWriter) 27 | if !ok { 28 | negroniResponseWriter = negroni.NewResponseWriter(responseWriter) 29 | } 30 | 31 | next(negroniResponseWriter, request) 32 | 33 | responseStatus := strconv.Itoa(negroniResponseWriter.Status()) 34 | middleware.metricsService.SendCounterMetric("status-"+responseStatus, 1) 35 | resourceType := ResourceTypeFrom(request.URL.Path) 36 | if resourceType != "" { 37 | duration := time.Since(startTime) 38 | middleware.metricsService.SendTimingMetric(request.Method+"-"+resourceType+"-time", duration) 39 | middleware.metricsService.SendTimingMetric(request.Method+"-"+resourceType+"-"+responseStatus+"-time", duration) 40 | middleware.metricsService.SendGaugeMetric(request.Method+"-"+resourceType+"-size", int64(negroniResponseWriter.Size())) 41 | middleware.metricsService.SendGaugeMetric(request.Method+"-"+resourceType+"-request-size", int64(request.ContentLength)) 42 | } 43 | } 44 | 45 | var resourceURLPathPattern = regexp.MustCompile(`^/(packages|droplets|app_stash|buildpacks|buildpack_cache)/`) 46 | 47 | func ResourceTypeFrom(path string) string { 48 | return strings.Trim(resourceURLPathPattern.FindString(path), "/") 49 | } 50 | -------------------------------------------------------------------------------- /middlewares/metrics_middleware_test.go: -------------------------------------------------------------------------------- 1 | package middlewares_test 2 | 3 | import ( 4 | http "net/http" 5 | "net/http/httptest" 6 | 7 | "github.com/cloudfoundry-incubator/bits-service/middlewares" 8 | . "github.com/cloudfoundry-incubator/bits-service/middlewares/matchers" 9 | . "github.com/petergtz/pegomock" 10 | ) 11 | 12 | var _ = Describe("MetricsMiddleWare", func() { 13 | It("can properly extract resource types from URL path", func() { 14 | Expect(middlewares.ResourceTypeFrom("/packages/123456")).To(Equal("packages")) 15 | Expect(middlewares.ResourceTypeFrom("/packages/123456/789")).To(Equal("packages")) 16 | Expect(middlewares.ResourceTypeFrom("/droplets/123456/789")).To(Equal("droplets")) 17 | Expect(middlewares.ResourceTypeFrom("/app_stash/some-sha")).To(Equal("app_stash")) 18 | Expect(middlewares.ResourceTypeFrom("/buildpacks/123456/789")).To(Equal("buildpacks")) 19 | Expect(middlewares.ResourceTypeFrom("/buildpack_cache/entries/123456/789")).To(Equal("buildpack_cache")) 20 | Expect(middlewares.ResourceTypeFrom("/")).To(BeEmpty()) 21 | Expect(middlewares.ResourceTypeFrom("/something/else")).To(BeEmpty()) 22 | }) 23 | 24 | It("sends all required metrics", func() { 25 | metricsService := NewMockMetricsService() 26 | middleware := middlewares.NewMetricsMiddleware(metricsService) 27 | req, e := http.NewRequest("GET", "http://example.com/packages/someguid", nil) 28 | Expect(e).NotTo(HaveOccurred()) 29 | 30 | middleware.ServeHTTP(httptest.NewRecorder(), req, func(rw http.ResponseWriter, r *http.Request) { 31 | rw.WriteHeader(http.StatusForbidden) 32 | }) 33 | 34 | metricsService.VerifyWasCalledOnce().SendCounterMetric("status-403", 1) 35 | metricsService.VerifyWasCalledOnce().SendGaugeMetric("GET-packages-size", 0) 36 | metricsService.VerifyWasCalledOnce().SendGaugeMetric("GET-packages-request-size", 0) 37 | metricsService.VerifyWasCalledOnce().SendTimingMetric(EqString("GET-packages-time"), AnyTimeDuration()) 38 | metricsService.VerifyWasCalledOnce().SendTimingMetric(EqString("GET-packages-403-time"), AnyTimeDuration()) 39 | }) 40 | 41 | It("sends metrics only for valid endpoints", func() { 42 | metricsService := NewMockMetricsService() 43 | middleware := middlewares.NewMetricsMiddleware(metricsService) 44 | req, e := http.NewRequest("FOO", "http://example.com/some/invalid/endpoint", nil) 45 | Expect(e).NotTo(HaveOccurred()) 46 | 47 | middleware.ServeHTTP(httptest.NewRecorder(), req, func(rw http.ResponseWriter, r *http.Request) { 48 | rw.WriteHeader(http.StatusBadRequest) 49 | }) 50 | 51 | metricName, _ := metricsService.VerifyWasCalledOnce().SendCounterMetric(AnyString(), AnyInt64()).GetCapturedArguments() 52 | Expect(metricName).To(Equal("status-400")) 53 | metricsService.VerifyWasCalled(Never()).SendGaugeMetric(AnyString(), AnyInt64()) 54 | metricsService.VerifyWasCalled(Never()).SendTimingMetric(AnyString(), AnyTimeDuration()) 55 | }) 56 | }) 57 | -------------------------------------------------------------------------------- /middlewares/mock_handler_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by pegomock. DO NOT EDIT! 2 | // Source: net/http (interfaces: Handler) 3 | 4 | package middlewares_test 5 | 6 | import ( 7 | http "net/http" 8 | "reflect" 9 | 10 | pegomock "github.com/petergtz/pegomock" 11 | ) 12 | 13 | type MockHandler struct { 14 | fail func(message string, callerSkip ...int) 15 | } 16 | 17 | func NewMockHandler() *MockHandler { 18 | return &MockHandler{fail: pegomock.GlobalFailHandler} 19 | } 20 | 21 | func (mock *MockHandler) ServeHTTP(_param0 http.ResponseWriter, _param1 *http.Request) { 22 | params := []pegomock.Param{_param0, _param1} 23 | pegomock.GetGenericMockFrom(mock).Invoke("ServeHTTP", params, []reflect.Type{}) 24 | } 25 | 26 | func (mock *MockHandler) VerifyWasCalledOnce() *VerifierHandler { 27 | return &VerifierHandler{mock, pegomock.Times(1), nil} 28 | } 29 | 30 | func (mock *MockHandler) VerifyWasCalled(invocationCountMatcher pegomock.Matcher) *VerifierHandler { 31 | return &VerifierHandler{mock, invocationCountMatcher, nil} 32 | } 33 | 34 | func (mock *MockHandler) VerifyWasCalledInOrder(invocationCountMatcher pegomock.Matcher, inOrderContext *pegomock.InOrderContext) *VerifierHandler { 35 | return &VerifierHandler{mock, invocationCountMatcher, inOrderContext} 36 | } 37 | 38 | type VerifierHandler struct { 39 | mock *MockHandler 40 | invocationCountMatcher pegomock.Matcher 41 | inOrderContext *pegomock.InOrderContext 42 | } 43 | 44 | func (verifier *VerifierHandler) ServeHTTP(_param0 http.ResponseWriter, _param1 *http.Request) *Handler_ServeHTTP_OngoingVerification { 45 | params := []pegomock.Param{_param0, _param1} 46 | methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "ServeHTTP", params) 47 | return &Handler_ServeHTTP_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} 48 | } 49 | 50 | type Handler_ServeHTTP_OngoingVerification struct { 51 | mock *MockHandler 52 | methodInvocations []pegomock.MethodInvocation 53 | } 54 | 55 | func (c *Handler_ServeHTTP_OngoingVerification) GetCapturedArguments() (http.ResponseWriter, *http.Request) { 56 | _param0, _param1 := c.GetAllCapturedArguments() 57 | return _param0[len(_param0)-1], _param1[len(_param1)-1] 58 | } 59 | 60 | func (c *Handler_ServeHTTP_OngoingVerification) GetAllCapturedArguments() (_param0 []http.ResponseWriter, _param1 []*http.Request) { 61 | params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) 62 | if len(params) > 0 { 63 | _param0 = make([]http.ResponseWriter, len(params[0])) 64 | for u, param := range params[0] { 65 | _param0[u] = param.(http.ResponseWriter) 66 | } 67 | _param1 = make([]*http.Request, len(params[1])) 68 | for u, param := range params[1] { 69 | _param1[u] = param.(*http.Request) 70 | } 71 | } 72 | return 73 | } 74 | -------------------------------------------------------------------------------- /middlewares/multipart_middleware.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/cloudfoundry-incubator/bits-service/util" 8 | 9 | "github.com/cloudfoundry-incubator/bits-service/logger" 10 | ) 11 | 12 | type MultipartMiddleware struct{} 13 | 14 | // This middleware is needed, because changing request contexts while passing request along different handlers creates 15 | // new requests objects. So if we only call request.ParseMultipartForm at in the last handler, only that copy of the request contains 16 | // the information about the temp files. By the time all the handlers return, and the server calls finishRequest(), that request 17 | // does not contain the information about the temp files. 18 | func (m *MultipartMiddleware) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request, next http.HandlerFunc) { 19 | if strings.Contains(request.Header.Get("Content-Type"), "multipart/form-data") { 20 | e := request.ParseMultipartForm(32 << 20) 21 | // lack of formalized error handling in the standard library forces us to do this fragile error string comparison 22 | // to see if it's a request problem, or if there is some generic error on server side. 23 | // We'll have to see if we need more special casing around this. 24 | if e != nil && e.Error() == "multipart: NextPart: unexpected EOF" { 25 | logger.From(request).Infow("Could not parse multipart", "error", e) 26 | responseWriter.WriteHeader(http.StatusBadRequest) 27 | return 28 | } 29 | 30 | util.PanicOnError(e) 31 | 32 | defer func() { 33 | if request.MultipartForm != nil { 34 | e := request.MultipartForm.RemoveAll() 35 | if e != nil { 36 | logger.From(request).Errorw("Could not delete multipart temporary files", "error", e) 37 | } 38 | } 39 | }() 40 | } 41 | 42 | next(responseWriter, request) 43 | } 44 | -------------------------------------------------------------------------------- /middlewares/negroni_gorilla_adapter.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/urfave/negroni" 7 | ) 8 | 9 | func GorillaMiddlewareFrom(negroniHandler negroni.Handler) func(next http.Handler) http.Handler { 10 | return func(next http.Handler) http.Handler { 11 | return http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { 12 | negroniHandler.ServeHTTP(responseWriter, request, next.ServeHTTP) 13 | }) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /middlewares/panic_middleware.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "runtime/debug" 9 | 10 | "github.com/cloudfoundry-incubator/bits-service/logger" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type PanicMiddleware struct{} 15 | 16 | type internalServerErrorResponseBody struct { 17 | Service string `json:"service"` 18 | Error string `json:"error"` 19 | VcapRequestID string `json:"vcap-request-id"` 20 | RequestID int64 `json:"request-id"` 21 | } 22 | 23 | func (middleware *PanicMiddleware) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request, next http.HandlerFunc) { 24 | defer func() { 25 | if e := recover(); e != nil { 26 | if _, ok := e.(interface { 27 | StackTrace() errors.StackTrace 28 | }); ok { 29 | logger.From(request).Errorw("Internal Server Error.", "error", fmt.Sprintf("%+v", e)) 30 | } else { 31 | logger.From(request).Errorw("Internal Server Error.", "error", fmt.Sprintf("%v\n%s", e, debug.Stack())) 32 | } 33 | responseWriter.WriteHeader(http.StatusInternalServerError) 34 | body, e := json.Marshal(internalServerErrorResponseBody{ 35 | Service: "Bits-Service", 36 | Error: "Internal Server Error", 37 | VcapRequestID: safeGetStringValueFrom(request.Context(), "vcap-request-id"), 38 | RequestID: safeGetInt64ValueFrom(request.Context(), "request-id"), 39 | }) 40 | if e != nil { 41 | // Nothing we can do at this point 42 | return 43 | } 44 | responseWriter.Write(body) 45 | } 46 | }() 47 | 48 | next(responseWriter, request) 49 | } 50 | 51 | func safeGetStringValueFrom(c context.Context, key string) string { 52 | if c.Value(key) == nil { 53 | return "" 54 | } 55 | if value, ok := c.Value(key).(string); ok { 56 | return value 57 | } 58 | return "" 59 | } 60 | 61 | func safeGetInt64ValueFrom(c context.Context, key string) int64 { 62 | if c.Value(key) == nil { 63 | return 0 64 | } 65 | if value, ok := c.Value(key).(int64); ok { 66 | return value 67 | } 68 | return 0 69 | } 70 | -------------------------------------------------------------------------------- /middlewares/panic_middleware_test.go: -------------------------------------------------------------------------------- 1 | package middlewares_test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | 7 | "github.com/cloudfoundry-incubator/bits-service/middlewares" 8 | "github.com/cloudfoundry-incubator/bits-service/util" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | var _ = Describe("PanicMiddleWare", func() { 13 | Context("Handler panics", func() { 14 | It("catches a panic and responds with an Internal Server Error", func() { 15 | responseWriter := httptest.NewRecorder() 16 | 17 | (&middlewares.PanicMiddleware{}).ServeHTTP( 18 | responseWriter, 19 | util.RequestWithContextValues( 20 | httptest.NewRequest("GET", "http://example.com/some/request", nil), 21 | "request-id", int64(123456), 22 | "vcap-request-id", "123456-7890-1234"), 23 | func(http.ResponseWriter, *http.Request) { 24 | panic(errors.New("Some unexpected error")) 25 | }) 26 | 27 | Expect(responseWriter.Code).To(Equal(http.StatusInternalServerError)) 28 | Expect(responseWriter.Body.String()).To(MatchJSON(`{ 29 | "service": "Bits-Service", 30 | "error": "Internal Server Error", 31 | "request-id": 123456, 32 | "vcap-request-id":"123456-7890-1234" 33 | }`)) 34 | }) 35 | }) 36 | 37 | Context("Handler succeeds", func() { 38 | It("responds with handler's response", func() { 39 | responseWriter := httptest.NewRecorder() 40 | 41 | (&middlewares.PanicMiddleware{}).ServeHTTP( 42 | responseWriter, 43 | httptest.NewRequest("GET", "http://example.com/some/request", nil), 44 | func(rw http.ResponseWriter, r *http.Request) { 45 | rw.WriteHeader(http.StatusOK) 46 | }) 47 | 48 | Expect(responseWriter.Code).To(Equal(http.StatusOK)) 49 | }) 50 | }) 51 | }) 52 | -------------------------------------------------------------------------------- /middlewares/signature_verification_middleware.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/cloudfoundry-incubator/bits-service/pathsigner" 7 | ) 8 | 9 | type SignatureVerificationMiddleware struct { 10 | SignatureValidator pathsigner.PathSignatureValidator 11 | } 12 | 13 | func (middleware *SignatureVerificationMiddleware) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request, next http.HandlerFunc) { 14 | if request.URL.Query().Get("signature") == "" { 15 | responseWriter.WriteHeader(403) 16 | return 17 | } 18 | if !middleware.SignatureValidator.SignatureValid(request.Method, request.URL) { 19 | responseWriter.WriteHeader(403) 20 | return 21 | } 22 | next(responseWriter, request) 23 | } 24 | -------------------------------------------------------------------------------- /middlewares/signature_verification_middleware_test.go: -------------------------------------------------------------------------------- 1 | package middlewares_test 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "github.com/benbjohnson/clock" 8 | "github.com/cloudfoundry-incubator/bits-service/pathsigner" 9 | "github.com/gorilla/mux" 10 | "github.com/urfave/negroni" 11 | 12 | "net/http/httptest" 13 | 14 | "time" 15 | 16 | . "github.com/cloudfoundry-incubator/bits-service/blobstores/local" 17 | . "github.com/cloudfoundry-incubator/bits-service/middlewares" 18 | "github.com/onsi/ginkgo" 19 | 20 | "github.com/onsi/gomega" 21 | ) 22 | 23 | func TestLocalBlobstore(t *testing.T) { 24 | gomega.RegisterFailHandler(ginkgo.Fail) 25 | ginkgo.RunSpecs(t, "LocalBlobstore") 26 | } 27 | 28 | const ( 29 | longerThanExpirationDuration = 2 * time.Hour 30 | ) 31 | 32 | var _ = Describe("Signing URLs", func() { 33 | var ( 34 | mockClock *clock.Mock 35 | pathSignerValidator *pathsigner.PathSignerValidator 36 | handler *LocalResourceSigner 37 | ) 38 | 39 | BeforeEach(func() { 40 | mockClock = clock.NewMock() 41 | pathSignerValidator = pathsigner.Validate(&pathsigner.PathSignerValidator{"geheim", mockClock, nil, ""}) 42 | handler = &LocalResourceSigner{ 43 | Signer: pathSignerValidator, 44 | DelegateEndpoint: "http://example.com", 45 | ResourcePathPrefix: "/my/", 46 | } 47 | }) 48 | 49 | It("signs and verifies URLs", func() { 50 | // signing 51 | responseBody := handler.Sign("path", "GET", mockClock.Now().Add(1*time.Hour)) 52 | 53 | Expect(responseBody).To(ContainSubstring("http://example.com/my/path?signature=")) 54 | Expect(responseBody).To(ContainSubstring("expires")) 55 | 56 | // verifying 57 | responseWriter := httptest.NewRecorder() 58 | delegateHandler := NewMockHandler() 59 | 60 | r := mux.NewRouter() 61 | r.Path("/my/path").Methods("GET").Handler(negroni.New( 62 | &SignatureVerificationMiddleware{pathSignerValidator}, 63 | negroni.Wrap(delegateHandler), 64 | )) 65 | r.ServeHTTP(responseWriter, httptest.NewRequest("GET", responseBody, nil)) 66 | 67 | Expect(responseWriter.Code).To(Equal(http.StatusOK)) 68 | }) 69 | 70 | It("signs and returns an error when URL has expired", func() { 71 | // signing 72 | responseBody := handler.Sign("path", "get", mockClock.Now().Add(1*time.Hour)) 73 | 74 | Expect(responseBody).To(ContainSubstring("http://example.com/my/path?signature=")) 75 | Expect(responseBody).To(ContainSubstring("expires")) 76 | 77 | mockClock.Add(longerThanExpirationDuration) 78 | 79 | // verifying 80 | responseWriter := httptest.NewRecorder() 81 | delegateHandler := NewMockHandler() 82 | 83 | r := mux.NewRouter() 84 | r.Path("/my/path").Methods("GET").Handler(negroni.New( 85 | &SignatureVerificationMiddleware{pathSignerValidator}, 86 | negroni.Wrap(delegateHandler), 87 | )) 88 | r.ServeHTTP(responseWriter, httptest.NewRequest("GET", responseBody, nil)) 89 | 90 | Expect(responseWriter.Code).To(Equal(http.StatusForbidden)) 91 | }) 92 | 93 | }) 94 | -------------------------------------------------------------------------------- /mock_readcloser_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by pegomock. DO NOT EDIT. 2 | // Source: io (interfaces: ReadCloser) 3 | 4 | package bitsgo_test 5 | 6 | import ( 7 | pegomock "github.com/petergtz/pegomock" 8 | "reflect" 9 | ) 10 | 11 | type MockReadCloser struct { 12 | fail func(message string, callerSkip ...int) 13 | } 14 | 15 | func NewMockReadCloser() *MockReadCloser { 16 | return &MockReadCloser{fail: pegomock.GlobalFailHandler} 17 | } 18 | 19 | func (mock *MockReadCloser) Close() error { 20 | params := []pegomock.Param{} 21 | result := pegomock.GetGenericMockFrom(mock).Invoke("Close", params, []reflect.Type{reflect.TypeOf((*error)(nil)).Elem()}) 22 | var ret0 error 23 | if len(result) != 0 { 24 | if result[0] != nil { 25 | ret0 = result[0].(error) 26 | } 27 | } 28 | return ret0 29 | } 30 | 31 | func (mock *MockReadCloser) Read(_param0 []byte) (int, error) { 32 | params := []pegomock.Param{_param0} 33 | result := pegomock.GetGenericMockFrom(mock).Invoke("Read", params, []reflect.Type{reflect.TypeOf((*int)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) 34 | var ret0 int 35 | var ret1 error 36 | if len(result) != 0 { 37 | if result[0] != nil { 38 | ret0 = result[0].(int) 39 | } 40 | if result[1] != nil { 41 | ret1 = result[1].(error) 42 | } 43 | } 44 | return ret0, ret1 45 | } 46 | 47 | func (mock *MockReadCloser) VerifyWasCalledOnce() *VerifierReadCloser { 48 | return &VerifierReadCloser{mock, pegomock.Times(1), nil} 49 | } 50 | 51 | func (mock *MockReadCloser) VerifyWasCalled(invocationCountMatcher pegomock.Matcher) *VerifierReadCloser { 52 | return &VerifierReadCloser{mock, invocationCountMatcher, nil} 53 | } 54 | 55 | func (mock *MockReadCloser) VerifyWasCalledInOrder(invocationCountMatcher pegomock.Matcher, inOrderContext *pegomock.InOrderContext) *VerifierReadCloser { 56 | return &VerifierReadCloser{mock, invocationCountMatcher, inOrderContext} 57 | } 58 | 59 | type VerifierReadCloser struct { 60 | mock *MockReadCloser 61 | invocationCountMatcher pegomock.Matcher 62 | inOrderContext *pegomock.InOrderContext 63 | } 64 | 65 | func (verifier *VerifierReadCloser) Close() *ReadCloser_Close_OngoingVerification { 66 | params := []pegomock.Param{} 67 | methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "Close", params) 68 | return &ReadCloser_Close_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} 69 | } 70 | 71 | type ReadCloser_Close_OngoingVerification struct { 72 | mock *MockReadCloser 73 | methodInvocations []pegomock.MethodInvocation 74 | } 75 | 76 | func (c *ReadCloser_Close_OngoingVerification) GetCapturedArguments() { 77 | } 78 | 79 | func (c *ReadCloser_Close_OngoingVerification) GetAllCapturedArguments() { 80 | } 81 | 82 | func (verifier *VerifierReadCloser) Read(_param0 []byte) *ReadCloser_Read_OngoingVerification { 83 | params := []pegomock.Param{_param0} 84 | methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "Read", params) 85 | return &ReadCloser_Read_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} 86 | } 87 | 88 | type ReadCloser_Read_OngoingVerification struct { 89 | mock *MockReadCloser 90 | methodInvocations []pegomock.MethodInvocation 91 | } 92 | 93 | func (c *ReadCloser_Read_OngoingVerification) GetCapturedArguments() []byte { 94 | _param0 := c.GetAllCapturedArguments() 95 | return _param0[len(_param0)-1] 96 | } 97 | 98 | func (c *ReadCloser_Read_OngoingVerification) GetAllCapturedArguments() (_param0 [][]byte) { 99 | params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) 100 | if len(params) > 0 { 101 | _param0 = make([][]byte, len(params[0])) 102 | for u, param := range params[0] { 103 | _param0[u] = param.([]byte) 104 | } 105 | } 106 | return 107 | } 108 | -------------------------------------------------------------------------------- /mock_resourcesigner_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by pegomock. DO NOT EDIT. 2 | // Source: github.com/cloudfoundry-incubator/bits-service (interfaces: ResourceSigner) 3 | 4 | package bitsgo_test 5 | 6 | import ( 7 | pegomock "github.com/petergtz/pegomock" 8 | "reflect" 9 | time "time" 10 | ) 11 | 12 | type MockResourceSigner struct { 13 | fail func(message string, callerSkip ...int) 14 | } 15 | 16 | func NewMockResourceSigner() *MockResourceSigner { 17 | return &MockResourceSigner{fail: pegomock.GlobalFailHandler} 18 | } 19 | 20 | func (mock *MockResourceSigner) Sign(resource string, method string, expirationTime time.Time) string { 21 | if mock == nil { 22 | panic("mock must not be nil. Use myMock := NewMockMockResourceSigner().") 23 | } 24 | params := []pegomock.Param{resource, method, expirationTime} 25 | result := pegomock.GetGenericMockFrom(mock).Invoke("Sign", params, []reflect.Type{reflect.TypeOf((*string)(nil)).Elem()}) 26 | var ret0 string 27 | if len(result) != 0 { 28 | if result[0] != nil { 29 | ret0 = result[0].(string) 30 | } 31 | } 32 | return ret0 33 | } 34 | 35 | func (mock *MockResourceSigner) VerifyWasCalledOnce() *VerifierResourceSigner { 36 | return &VerifierResourceSigner{mock, pegomock.Times(1), nil} 37 | } 38 | 39 | func (mock *MockResourceSigner) VerifyWasCalled(invocationCountMatcher pegomock.Matcher) *VerifierResourceSigner { 40 | return &VerifierResourceSigner{mock, invocationCountMatcher, nil} 41 | } 42 | 43 | func (mock *MockResourceSigner) VerifyWasCalledInOrder(invocationCountMatcher pegomock.Matcher, inOrderContext *pegomock.InOrderContext) *VerifierResourceSigner { 44 | return &VerifierResourceSigner{mock, invocationCountMatcher, inOrderContext} 45 | } 46 | 47 | type VerifierResourceSigner struct { 48 | mock *MockResourceSigner 49 | invocationCountMatcher pegomock.Matcher 50 | inOrderContext *pegomock.InOrderContext 51 | } 52 | 53 | func (verifier *VerifierResourceSigner) Sign(resource string, method string, expirationTime time.Time) *ResourceSigner_Sign_OngoingVerification { 54 | params := []pegomock.Param{resource, method, expirationTime} 55 | methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "Sign", params) 56 | return &ResourceSigner_Sign_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} 57 | } 58 | 59 | type ResourceSigner_Sign_OngoingVerification struct { 60 | mock *MockResourceSigner 61 | methodInvocations []pegomock.MethodInvocation 62 | } 63 | 64 | func (c *ResourceSigner_Sign_OngoingVerification) GetCapturedArguments() (string, string, time.Time) { 65 | resource, method, expirationTime := c.GetAllCapturedArguments() 66 | return resource[len(resource)-1], method[len(method)-1], expirationTime[len(expirationTime)-1] 67 | } 68 | 69 | func (c *ResourceSigner_Sign_OngoingVerification) GetAllCapturedArguments() (_param0 []string, _param1 []string, _param2 []time.Time) { 70 | params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) 71 | if len(params) > 0 { 72 | _param0 = make([]string, len(params[0])) 73 | for u, param := range params[0] { 74 | _param0[u] = param.(string) 75 | } 76 | _param1 = make([]string, len(params[1])) 77 | for u, param := range params[1] { 78 | _param1[u] = param.(string) 79 | } 80 | _param2 = make([]time.Time, len(params[2])) 81 | for u, param := range params[2] { 82 | _param2[u] = param.(time.Time) 83 | } 84 | } 85 | return 86 | } 87 | -------------------------------------------------------------------------------- /oci_registry/assets/example_droplet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/oci_registry/assets/example_droplet -------------------------------------------------------------------------------- /oci_registry/models/docker/mediatype/models.go: -------------------------------------------------------------------------------- 1 | package mediatype 2 | 3 | const ( 4 | DistributionManifestListV2Json = "application/vnd.docker.distribution.manifest.list.v2+json" 5 | DistributionManifestV2Json = "application/vnd.docker.distribution.manifest.v2+json" 6 | ImageManifestV2Json = "application/vnd.docker.image.manifest.v2+json" 7 | ContainerImageV1Json = "application/vnd.docker.container.image.v1+json" 8 | ImageRootfsTar = "application/vnd.docker.image.rootfs.diff.tar" 9 | ImageRootfsTarGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip" 10 | ) 11 | -------------------------------------------------------------------------------- /oci_registry/models/docker/models.go: -------------------------------------------------------------------------------- 1 | package docker 2 | 3 | type Versioned struct { 4 | SchemaVersion int `json:"schemaVersion"` 5 | MediaType string `json:"mediaType,omitempty"` 6 | } 7 | type Manifest struct { 8 | Versioned 9 | Config Content `json:"config"` 10 | Layers []Content `json:"layers"` 11 | } 12 | 13 | type Content struct { 14 | MediaType string `json:"mediaType"` 15 | Digest string `json:"digest"` 16 | Size int64 `json:"size"` 17 | } 18 | 19 | type ManifestList struct { 20 | Versioned 21 | Manifests []ManifestDescriptor `json:"manifests"` 22 | } 23 | 24 | type ManifestDescriptor struct { 25 | Content 26 | Platform PlatformSpec `json:"platform"` 27 | } 28 | 29 | type PlatformSpec struct { 30 | Architecture string `json:"architecture"` 31 | OS string `json:"os"` 32 | OSVersion string `json:"os.version,omitempty"` 33 | OSFeatures []string `json:"os.features,omitempty"` 34 | Variant string `json:"variant,omitempty"` 35 | Features []string `json:"features,omitempty"` 36 | } 37 | -------------------------------------------------------------------------------- /oci_registry/oci_registry_suite_test.go: -------------------------------------------------------------------------------- 1 | package oci_registry_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestOciRegistry(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "OciRegistry Suite") 13 | } 14 | -------------------------------------------------------------------------------- /package_bundle.go: -------------------------------------------------------------------------------- 1 | package bitsgo 2 | 3 | import ( 4 | "archive/zip" 5 | "crypto/sha1" 6 | "encoding/hex" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "strconv" 11 | "time" 12 | 13 | "go.uber.org/zap" 14 | 15 | "github.com/pkg/errors" 16 | 17 | "github.com/cenkalti/backoff" 18 | ) 19 | 20 | func CreateTempZipFileFrom(bundlesPayload []Fingerprint, 21 | zipReader *zip.Reader, 22 | minimumSize, maximumSize uint64, 23 | blobstore Blobstore, 24 | metricsService MetricsService, 25 | logger *zap.SugaredLogger, 26 | ) (tempFilename string, err error) { 27 | tempZipFile, e := ioutil.TempFile("", "bundles") 28 | if e != nil { 29 | return "", errors.Wrap(e, "Could not create temp file") 30 | } 31 | defer func() { 32 | if err != nil { 33 | os.Remove(tempZipFile.Name()) 34 | } 35 | }() 36 | defer tempZipFile.Close() 37 | zipWriter := zip.NewWriter(tempZipFile) 38 | 39 | if zipReader != nil { 40 | for _, zipInputFileEntry := range zipReader.File { 41 | if !zipInputFileEntry.FileInfo().Mode().IsRegular() { 42 | continue 43 | } 44 | zipFileEntryWriter, e := zipWriter.CreateHeader(zipEntryHeaderWithModifiedTime(zipInputFileEntry.Name, zipInputFileEntry.FileInfo().Mode(), zipInputFileEntry.FileHeader.Modified)) 45 | if e != nil { 46 | return "", errors.Wrap(e, "Could not create header in zip file") 47 | } 48 | zipEntryReader, e := zipInputFileEntry.Open() 49 | if e != nil { 50 | return "", errors.Wrap(e, "Could not open zip file entry") 51 | } 52 | defer zipEntryReader.Close() 53 | 54 | tempFile, e := ioutil.TempFile("", "app-stash") 55 | if e != nil { 56 | return "", errors.Wrap(e, "Could not create tempfile") 57 | } 58 | defer os.Remove(tempFile.Name()) 59 | defer tempFile.Close() 60 | 61 | sha := sha1.New() 62 | tempFileSize, e := io.Copy(io.MultiWriter(zipFileEntryWriter, tempFile, sha), zipEntryReader) 63 | if e != nil { 64 | return "", errors.Wrap(e, "Could not copy content from zip entry") 65 | } 66 | e = tempFile.Close() 67 | if e != nil { 68 | return "", errors.Wrap(e, "Could not close temp file") 69 | } 70 | e = zipEntryReader.Close() 71 | if e != nil { 72 | return "", errors.Wrap(e, "Could not close zip entry reader") 73 | } 74 | e = backoff.RetryNotify(func() error { 75 | tempFile, e = os.Open(tempFile.Name()) 76 | if e != nil { 77 | return errors.Wrap(e, "Could not open temp file for reading") 78 | } 79 | defer tempFile.Close() 80 | if uint64(tempFileSize) >= minimumSize && uint64(tempFileSize) <= maximumSize { 81 | sha := hex.EncodeToString(sha.Sum(nil)) 82 | e = blobstore.Put(sha, tempFile) 83 | if e != nil { 84 | if _, ok := e.(*NoSpaceLeftError); ok { 85 | return backoff.Permanent(e) 86 | } 87 | return errors.Wrapf(e, "Could not upload file to blobstore. SHA: '%v'", sha) 88 | } 89 | } 90 | return nil 91 | }, backoff.NewExponentialBackOff(), func(e error, backOffDelay time.Duration) { 92 | metricsService.SendCounterMetric("appStashPutRetries", 1) 93 | }) 94 | if e != nil { 95 | return "", e 96 | } 97 | os.Remove(tempFile.Name()) 98 | } 99 | } 100 | 101 | for _, entry := range bundlesPayload { 102 | zipEntry, e := zipWriter.CreateHeader(zipEntryHeaderWithModifiedTime(entry.Fn, fileModeFrom(entry.Mode), time.Now())) 103 | if e != nil { 104 | return "", errors.Wrap(e, "Could create header in zip file") 105 | } 106 | 107 | e = backoff.RetryNotify(func() error { 108 | b, e := blobstore.Get(entry.Sha1) 109 | 110 | if e != nil { 111 | if _, ok := e.(*NotFoundError); ok { 112 | return backoff.Permanent(NewNotFoundErrorWithKey(entry.Sha1)) 113 | } 114 | return errors.Wrapf(e, "Could not get file from blobstore. SHA: '%v'", entry.Sha1) 115 | } 116 | defer b.Close() 117 | 118 | _, e = io.Copy(zipEntry, b) 119 | if e != nil { 120 | return errors.Wrapf(e, "Could not copy file to zip entry. SHA: %v", entry.Sha1) 121 | } 122 | return nil 123 | }, 124 | backoff.NewExponentialBackOff(), 125 | func(e error, backOffDelay time.Duration) { 126 | metricsService.SendCounterMetric("appStashGetRetries", 1) 127 | }, 128 | ) 129 | if e != nil { 130 | return "", e 131 | } 132 | } 133 | zipWriter.Close() 134 | return tempZipFile.Name(), nil 135 | } 136 | 137 | func fileModeFrom(s string) os.FileMode { 138 | mode, e := strconv.ParseInt(s, 8, 32) 139 | if e != nil { 140 | return 0744 141 | } 142 | return os.FileMode(mode) 143 | } 144 | 145 | func zipEntryHeaderWithModifiedTime(name string, mode os.FileMode, modified time.Time) *zip.FileHeader { 146 | header := &zip.FileHeader{ 147 | Name: name, 148 | Method: zip.Deflate, 149 | Modified: modified, 150 | } 151 | header.SetMode(mode) 152 | return header 153 | } 154 | -------------------------------------------------------------------------------- /pathsigner/pathsigner.go: -------------------------------------------------------------------------------- 1 | package pathsigner 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha256" 6 | "crypto/subtle" 7 | "encoding/hex" 8 | "errors" 9 | "fmt" 10 | "net/url" 11 | "strconv" 12 | "strings" 13 | "time" 14 | 15 | "github.com/benbjohnson/clock" 16 | ) 17 | 18 | type PathSigner interface { 19 | Sign(method string, path string, expires time.Time) string 20 | } 21 | 22 | type PathSignatureValidator interface { 23 | SignatureValid(method string, u *url.URL) bool 24 | } 25 | 26 | type PathSignerValidator struct { 27 | Secret string 28 | Clock clock.Clock 29 | SigningKeys map[string]string 30 | ActiveKeyID string 31 | } 32 | 33 | func Validate(signer *PathSignerValidator) *PathSignerValidator { 34 | if signer.Secret == "" && len(signer.SigningKeys) == 0 { 35 | panic(errors.New("must provide either \"Secret\" or \"SigningKeys\" with at least one element")) 36 | } 37 | if len(signer.SigningKeys) > 0 && signer.ActiveKeyID == "" { 38 | panic(errors.New("when providing SigningKeys, you must also provide ActiveKeyID")) 39 | } 40 | return signer 41 | } 42 | 43 | func (signer *PathSignerValidator) Sign(method string, path string, expires time.Time) string { 44 | method = strings.ToUpper(method) 45 | if len(signer.SigningKeys) > 0 { 46 | return fmt.Sprintf("%s?signature=%x&expires=%v&AccessKeyId=%v", path, signatureWithHMACFor(method, path, signer.SigningKeys[signer.ActiveKeyID], expires), expires.Unix(), signer.ActiveKeyID) 47 | } 48 | return fmt.Sprintf("%s?signature=%x&expires=%v", path, signatureWithHMACFor(method, path, signer.Secret, expires), expires.Unix()) 49 | } 50 | 51 | func (signer *PathSignerValidator) SignatureValid(method string, u *url.URL) bool { 52 | method = strings.ToUpper(method) 53 | expires, e := strconv.ParseInt(u.Query().Get("expires"), 10, 64) 54 | if e != nil { 55 | return false 56 | } 57 | if signer.Clock.Now().After(time.Unix(expires, 0)) { 58 | return false 59 | } 60 | 61 | querySignature, e := hex.DecodeString(u.Query().Get("signature")) 62 | if e != nil { 63 | return false 64 | } 65 | 66 | accessKeyID := u.Query().Get("AccessKeyId") 67 | if accessKeyID != "" { 68 | if _, exist := signer.SigningKeys[accessKeyID]; !exist { 69 | return false 70 | } 71 | if subtle.ConstantTimeCompare(querySignature, signatureWithHMACFor(method, u.Path, signer.SigningKeys[accessKeyID], time.Unix(expires, 0))) == 0 { 72 | return false 73 | } 74 | } else { 75 | if subtle.ConstantTimeCompare(querySignature, signatureWithHMACFor(method, u.Path, signer.Secret, time.Unix(expires, 0))) == 0 { 76 | return false 77 | } 78 | } 79 | return true 80 | } 81 | 82 | func signatureWithHMACFor(method string, path string, secret string, expires time.Time) []byte { 83 | hash := hmac.New(sha256.New, []byte(secret)) 84 | hash.Write([]byte(fmt.Sprintf("%v %v %v %v", method, path, secret, expires.Unix()))) 85 | return hash.Sum(nil) 86 | } 87 | -------------------------------------------------------------------------------- /pathsigner/pathsigner_test.go: -------------------------------------------------------------------------------- 1 | package pathsigner_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/benbjohnson/clock" 9 | . "github.com/benbjohnson/clock" 10 | "github.com/cloudfoundry-incubator/bits-service/httputil" 11 | "github.com/cloudfoundry-incubator/bits-service/pathsigner" 12 | . "github.com/cloudfoundry-incubator/bits-service/pathsigner" 13 | "github.com/onsi/ginkgo" 14 | . "github.com/onsi/ginkgo" 15 | "github.com/onsi/gomega" 16 | . "github.com/onsi/gomega" 17 | ) 18 | 19 | func TestPathSigner(t *testing.T) { 20 | gomega.RegisterFailHandler(ginkgo.Fail) 21 | ginkgo.RunSpecs(t, "PathSigner") 22 | } 23 | 24 | var _ = Describe("PathSigner", func() { 25 | 26 | var ( 27 | clock *clock.Mock 28 | signer *PathSignerValidator 29 | ) 30 | 31 | BeforeEach(func() { 32 | clock = NewMock() 33 | }) 34 | 35 | Context("Only secret is used. SigningKeys is empty", func() { 36 | BeforeEach(func() { 37 | signer = pathsigner.Validate(&PathSignerValidator{Secret: "thesecret", Clock: clock}) 38 | }) 39 | 40 | It("can sign a path and validate its signature", func() { 41 | signedPath := signer.Sign("GET", "/some/path", time.Unix(200, 0)) 42 | 43 | Expect(signer.SignatureValid("GET", httputil.MustParse(signedPath))).To(BeTrue()) 44 | }) 45 | 46 | It("can sign a path and will not validate a path when it has expired", func() { 47 | signedPath := signer.Sign("GET", "/some/path", time.Unix(200, 0)) 48 | 49 | clock.Add(time.Hour) 50 | 51 | Expect(signer.SignatureValid("GET", httputil.MustParse(signedPath))).To(BeFalse()) 52 | }) 53 | 54 | It("can sign a path and will not allow to tamper with the expiration time", func() { 55 | signedPath := signer.Sign("GET", "/some/path", time.Unix(200, 0)) 56 | 57 | clock.Add(time.Hour) 58 | 59 | u := httputil.MustParse(signedPath) 60 | q := u.Query() 61 | q.Set("expires", fmt.Sprintf("%v", clock.Now().Add(time.Hour).Unix())) 62 | u.RawQuery = q.Encode() 63 | 64 | Expect(signer.SignatureValid("GET", u)).To(BeFalse()) 65 | }) 66 | }) 67 | 68 | Context("SigningKeys is used. Secret is irrelevant.", func() { 69 | BeforeEach(func() { 70 | signer = pathsigner.Validate(&PathSignerValidator{ 71 | Secret: "thesecret", 72 | Clock: clock, 73 | SigningKeys: map[string]string{ 74 | "key1": "secret1", 75 | "key2": "secret2", 76 | "key3": "secret3", 77 | }, 78 | ActiveKeyID: "key2", 79 | }) 80 | }) 81 | 82 | It("can sign a path and validate its signature", func() { 83 | signedPath := signer.Sign("GET", "/some/path", time.Unix(200, 0)) 84 | 85 | Expect(signedPath).To(ContainSubstring("AccessKeyId=key2")) 86 | Expect(signer.SignatureValid("GET", httputil.MustParse(signedPath))).To(BeTrue()) 87 | }) 88 | 89 | It("can sign a path and will not allow to tamper with the signature", func() { 90 | signedPath := signer.Sign("GET", "/some/path", time.Unix(200, 0)) 91 | 92 | u := httputil.MustParse(signedPath) 93 | q := u.Query() 94 | q.Set("signature", "InventedSignature") 95 | u.RawQuery = q.Encode() 96 | 97 | Expect(signer.SignatureValid("GET", u)).To(BeFalse()) 98 | }) 99 | 100 | It("can sign a path and will not allow to tamper with the AccessKeyId", func() { 101 | signedPath := signer.Sign("GET", "/some/path", time.Unix(200, 0)) 102 | 103 | u := httputil.MustParse(signedPath) 104 | q := u.Query() 105 | q.Set("AccessKeyId", "SomethingElse") 106 | u.RawQuery = q.Encode() 107 | 108 | Expect(signer.SignatureValid("GET", u)).To(BeFalse()) 109 | }) 110 | 111 | Context("Signing and validation method differs", func() { 112 | It("fails signature validation", func() { 113 | signedPath := signer.Sign("GET", "/some/path", time.Unix(200, 0)) 114 | 115 | Expect(signedPath).To(ContainSubstring("AccessKeyId=key2")) 116 | Expect(signer.SignatureValid("PUT", httputil.MustParse(signedPath))).To(BeFalse()) 117 | }) 118 | }) 119 | }) 120 | 121 | }) 122 | -------------------------------------------------------------------------------- /performance-comparison.md: -------------------------------------------------------------------------------- 1 | # Performance Comparison 2 | 3 | ## Speed 4 | 5 | The performance test was done in an AWS environment with an S3 blobstore backend. The bits-service was deployed as one VM with the Ruby implementation in it, and another VM with the Go implementation in it. 6 | 7 | ### GET requests 8 | 9 | GET requests are about **3 orders of magnitude faster** in this Go implementation than in the [original Ruby implementation](https://github.com/cloudfoundry-incubator/bits-service). 10 | 11 | #### 100 requests with concurrency 10: 12 | 13 | ![](images/GET-request-speed-comparison-100-10.png) 14 | 15 | #### 10 requests with concurrency 1: 16 | 17 | ![](images/GET-request-speed-comparison-10-1.png) 18 | 19 | #### 100 requests with concurrency 100: 20 | 21 | ![](images/GET-request-speed-comparison-100-100.png) 22 | 23 | ### PUT requests 24 | 25 | PUT requests are about **2 orders of magnitude faster** in this Go implementation than in the [original Ruby implementation](https://github.com/cloudfoundry-incubator/bits-service) when doing 100 requests with concurrency 10: 26 | 27 | ![](images/PUT-request-speed-comparison.png) 28 | 29 | ## Memory Consumption 30 | 31 | ### Ruby Implementation 32 | 33 | The [Ruby implementation](https://github.com/cloudfoundry-incubator/bits-service) uses more or less by default 1GB of memory. 34 | 35 | ![](images/bits-service-ruby-mem-consumption.png) 36 | 37 | ### Go 38 | 39 | TBD 40 | 41 | -------------------------------------------------------------------------------- /routes/test_data/test_archive.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudfoundry-incubator/bits-service/cc4ef14ec36197c495fc706b8da840aa5a5d318b/routes/test_data/test_archive.zip -------------------------------------------------------------------------------- /scripts/build-and-run: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | pushd $(dirname $0)/../cmd/bitsgo 4 | echo `pwd` 5 | go install 6 | popd 7 | 8 | bitsgo -c $1 9 | -------------------------------------------------------------------------------- /scripts/install-git-hooks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | brew list git-secrets || brew install git-secrets 4 | brew list lastpass-cli || brew install lastpass-cli --with-pinentry 5 | git secrets --install 6 | git secrets --register-aws || echo "Could not register AWS patterns (maybe they're already in .git/config)" 7 | git secrets --add-provider "$(cd "$(dirname "$0")" && pwd -P)"/list-lastpass-passwords.sh 8 | -------------------------------------------------------------------------------- /scripts/list-lastpass-passwords.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | export IGNORE='(RSA PRIVATE KEY---|see below|c1oudc0w|skip-pw-check)' 4 | 5 | set +e 6 | lpass status --quiet 7 | EXIT_CODE=$? 8 | set -e 9 | 10 | if [[ $EXIT_CODE -ne 0 ]]; then 11 | echo "ERROR: Not logged in to LastPass" 12 | exit 1 13 | fi 14 | 15 | lpass ls Shared-Flintstone --format 'Shared-Flintstone/'"'"'%an'"'"'' --color=never | \ 16 | xargs lpass show --color=never {} | \ 17 | # manually remove false alarms: 18 | grep -vE "$IGNORE" | \ 19 | # take any line that contains an actual password: 20 | grep -i -E '(.*pass.*|.*key.*|.*secret.*|.*token.*)' | \ 21 | # Use only last column ater splitting on ':': 22 | rev | cut -d: -f1 | rev | \ 23 | # trim whitespace and quotes: 24 | xargs -n1 | \ 25 | # remove duplicate lines: 26 | sort -u 27 | -------------------------------------------------------------------------------- /scripts/run-unit-tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname $0)/.. 4 | 5 | ginkgo -r --skipPackage=blobstores/contract_integ_test -keepGoing 6 | -------------------------------------------------------------------------------- /sign_handler.go: -------------------------------------------------------------------------------- 1 | package bitsgo 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/benbjohnson/clock" 9 | ) 10 | 11 | type ResourceSigner interface { 12 | Sign(resource string, method string, expirationTime time.Time) (signedURL string) 13 | } 14 | 15 | type SignResourceHandler struct { 16 | clock clock.Clock 17 | putResourceSigner, getResourceSigner ResourceSigner 18 | } 19 | 20 | func NewSignResourceHandler(getResourceSigner, putResourceSigner ResourceSigner) *SignResourceHandler { 21 | return &SignResourceHandler{ 22 | getResourceSigner: getResourceSigner, 23 | putResourceSigner: putResourceSigner, 24 | clock: clock.New(), 25 | } 26 | } 27 | 28 | func (handler *SignResourceHandler) Sign(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) { 29 | method := params["verb"] 30 | var signer ResourceSigner 31 | 32 | if method == "" { 33 | method = "get" 34 | } 35 | 36 | switch method { 37 | case "get": 38 | signer = handler.getResourceSigner 39 | case "put", "post": 40 | signer = handler.putResourceSigner 41 | default: 42 | responseWriter.WriteHeader(http.StatusBadRequest) 43 | responseWriter.Write([]byte("Invalid verb: " + method)) 44 | return 45 | } 46 | 47 | signature := signer.Sign(params["resource"], method, handler.clock.Now().Add(1*time.Hour)) 48 | fmt.Fprint(responseWriter, signature) 49 | } 50 | -------------------------------------------------------------------------------- /sign_handler_test.go: -------------------------------------------------------------------------------- 1 | package bitsgo_test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "reflect" 7 | "time" 8 | 9 | "github.com/cloudfoundry-incubator/bits-service" 10 | "github.com/cloudfoundry-incubator/bits-service/httputil" 11 | . "github.com/petergtz/pegomock" 12 | ) 13 | 14 | func AnyTime() (result time.Time) { 15 | RegisterMatcher(NewAnyMatcher(reflect.TypeOf((*time.Time)(nil)).Elem())) 16 | return 17 | } 18 | 19 | var _ = Describe("SignResourceHandler", func() { 20 | var ( 21 | getSigner *MockResourceSigner 22 | putSigner *MockResourceSigner 23 | recorder *httptest.ResponseRecorder 24 | ) 25 | 26 | BeforeEach(func() { 27 | getSigner = NewMockResourceSigner() 28 | putSigner = NewMockResourceSigner() 29 | recorder = httptest.NewRecorder() 30 | }) 31 | 32 | It("Signs a GET URL", func() { 33 | When(getSigner.Sign(AnyString(), AnyString(), AnyTime())).ThenReturn("Some get signature") 34 | handler := bitsgo.NewSignResourceHandler(getSigner, putSigner) 35 | request := httputil.NewRequest("GET", "/foo", nil).Build() 36 | 37 | handler.Sign(recorder, request, map[string]string{"verb": "get", "resource": "bar"}) 38 | Expect(recorder.Code).To(Equal(http.StatusOK)) 39 | Expect(recorder.Body.String()).To(Equal("Some get signature")) 40 | }) 41 | 42 | Context("Invalid method", func() { 43 | It("Responds with error code", func() { 44 | handler := bitsgo.NewSignResourceHandler(getSigner, putSigner) 45 | request := httputil.NewRequest("", "/does", nil).Build() 46 | 47 | handler.Sign(recorder, request, map[string]string{"verb": "something invalid", "resource": "foobar"}) 48 | Expect(recorder.Code).To(Equal(http.StatusBadRequest)) 49 | Expect(recorder.Body.String()).To(Equal("Invalid verb: something invalid")) 50 | }) 51 | }) 52 | 53 | It("Signs a PUT URL", func() { 54 | When(putSigner.Sign(AnyString(), AnyString(), AnyTime())).ThenReturn("Some put signature") 55 | 56 | handler := bitsgo.NewSignResourceHandler(getSigner, putSigner) 57 | request := httputil.NewRequest("PUT", "/bar", nil).Build() 58 | 59 | handler.Sign(recorder, request, map[string]string{"verb": "put", "resource": "foobar"}) 60 | Expect(recorder.Code).To(Equal(http.StatusOK)) 61 | Expect(recorder.Body.String()).To(Equal("Some put signature")) 62 | }) 63 | }) 64 | -------------------------------------------------------------------------------- /statsd/metrics_service.go: -------------------------------------------------------------------------------- 1 | package statsd 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/tecnickcom/statsd" 7 | ) 8 | 9 | type MetricsService struct { 10 | statsdClient *statsd.Client 11 | prefix string 12 | } 13 | 14 | func NewMetricsService() *MetricsService { 15 | statsdClient, e := statsd.New() // Connect to the UDP port 8125 by default. 16 | if e != nil { 17 | panic(e) 18 | } 19 | return &MetricsService{statsdClient: statsdClient, prefix: "bits."} 20 | } 21 | 22 | func (service *MetricsService) SendTimingMetric(name string, duration time.Duration) { 23 | service.statsdClient.Timing(service.prefix+name, duration.Seconds()*1000) 24 | // we send this additional metric, because our test envs use metrics.ng.bluemix.net 25 | // and for aggregation purposes this service needs this suffix. 26 | service.statsdClient.Timing(service.prefix+name+".sparse-avg", duration.Seconds()*1000) 27 | } 28 | func (service *MetricsService) SendGaugeMetric(name string, value int64) { 29 | service.statsdClient.Gauge(service.prefix+name, value) 30 | } 31 | 32 | func (service *MetricsService) SendCounterMetric(name string, value int64) { 33 | service.statsdClient.Count(service.prefix+name, value) 34 | } 35 | -------------------------------------------------------------------------------- /testutil/http.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | 7 | "bytes" 8 | 9 | . "github.com/onsi/gomega" 10 | "github.com/onsi/gomega/gstruct" 11 | "github.com/onsi/gomega/types" 12 | ) 13 | 14 | func HaveStatusCodeAndBody(statusCode types.GomegaMatcher, body types.GomegaMatcher) types.GomegaMatcher { 15 | return SatisfyAny( 16 | gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ 17 | "Code": statusCode, 18 | "Body": WithTransform(func(body *bytes.Buffer) string { return body.String() }, body), 19 | }), 20 | gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ 21 | "StatusCode": statusCode, 22 | "Body": WithTransform(func(body io.Reader) string { 23 | content, e := ioutil.ReadAll(body) 24 | if e != nil { 25 | panic(e) 26 | } 27 | return string(content) 28 | }, body), 29 | }), 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /testutil/zip.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "archive/tar" 5 | "archive/zip" 6 | "bytes" 7 | "compress/gzip" 8 | "io/ioutil" 9 | "strings" 10 | 11 | . "github.com/onsi/ginkgo" 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | func CreateZip(contents map[string]string) *bytes.Buffer { 16 | var result bytes.Buffer 17 | zipWriter := zip.NewWriter(&result) 18 | defer zipWriter.Close() 19 | for filename, fileContents := range contents { 20 | entryWriter, e := zipWriter.Create(filename) 21 | Expect(e).NotTo(HaveOccurred()) 22 | entryWriter.Write([]byte(fileContents)) 23 | 24 | } 25 | e := zipWriter.Close() 26 | Expect(e).NotTo(HaveOccurred()) 27 | return &result 28 | } 29 | 30 | func CreateGZip(contents map[string]string) *bytes.Buffer { 31 | var result bytes.Buffer 32 | zipWriter := gzip.NewWriter(&result) 33 | t := tar.NewWriter(zipWriter) 34 | 35 | defer zipWriter.Close() 36 | for filename, fileContents := range contents { 37 | e := t.WriteHeader(&tar.Header{ 38 | Name: filename, 39 | Mode: 0600, 40 | Size: int64(len(fileContents)), 41 | }) 42 | Expect(e).NotTo(HaveOccurred()) 43 | t.Write([]byte(fileContents)) 44 | 45 | } 46 | e := zipWriter.Close() 47 | Expect(e).NotTo(HaveOccurred()) 48 | return &result 49 | } 50 | 51 | func VerifyZipFileEntry(reader *zip.Reader, expectedFilename string, expectedContent string) { 52 | var foundEntries []string 53 | for _, entry := range reader.File { 54 | if entry.Name == expectedFilename { 55 | content, e := entry.Open() 56 | Expect(e).NotTo(HaveOccurred()) 57 | Expect(ioutil.ReadAll(content)).To(MatchRegexp(expectedContent), "for filename "+expectedFilename) 58 | return 59 | } 60 | foundEntries = append(foundEntries, entry.Name) 61 | } 62 | Fail("Did not find entry with name " + expectedFilename + ". Found only: " + strings.Join(foundEntries, ", ")) 63 | } 64 | -------------------------------------------------------------------------------- /util/context.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | ) 7 | 8 | func RequestWithContextValues(r *http.Request, keysAndValues ...interface{}) *http.Request { 9 | if len(keysAndValues)%2 != 0 { 10 | panic("keysAndValues must have an even number of elements") 11 | } 12 | 13 | c := r.Context() 14 | for i := 0; i < len(keysAndValues); i += 2 { 15 | c = context.WithValue(c, keysAndValues[i], keysAndValues[i+1]) 16 | } 17 | return r.WithContext(c) 18 | } 19 | -------------------------------------------------------------------------------- /util/panic.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | func PanicOnError(e error) { 4 | if e != nil { 5 | panic(e) 6 | } 7 | } 8 | 9 | var Must = PanicOnError 10 | -------------------------------------------------------------------------------- /util/response.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | ) 8 | 9 | func FprintDescriptionAndCodeAsJSON(responseWriter http.ResponseWriter, code int, description string, a ...interface{}) { 10 | fmt.Fprintf(responseWriter, DescriptionAndCodeAsJSON(code, description, a...)) 11 | } 12 | 13 | func DescriptionAndCodeAsJSON(code int, description string, a ...interface{}) string { 14 | 15 | m, e := json.Marshal(struct { 16 | Description string `json:"description"` 17 | Code int `json:"code"` 18 | }{ 19 | Description: fmt.Sprintf(description, a...), 20 | Code: code, 21 | }) 22 | PanicOnError(e) 23 | return string(m) 24 | } 25 | 26 | func FprintDescriptionAsJSON(responseWriter http.ResponseWriter, description string, a ...interface{}) { 27 | fmt.Fprintf(responseWriter, DescriptionAsJSON(description, a...)) 28 | } 29 | 30 | func DescriptionAsJSON(description string, a ...interface{}) string { 31 | m, e := json.Marshal(struct { 32 | Description string `json:"description"` 33 | }{ 34 | Description: fmt.Sprintf(description, a...), 35 | }) 36 | PanicOnError(e) 37 | return string(m) 38 | } 39 | -------------------------------------------------------------------------------- /util/safecloser.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "io" 4 | 5 | type SafeCloser bool 6 | 7 | func (safeCloser *SafeCloser) Close(closer io.Closer) error { 8 | if *safeCloser { 9 | return nil 10 | } 11 | *safeCloser = true 12 | return closer.Close() 13 | } 14 | --------------------------------------------------------------------------------