├── .github └── workflows │ └── test.yml ├── LICENSE.txt ├── README.md ├── go.mod ├── go.sum ├── integration_test.go ├── main.go └── main_test.go /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: '**' 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | with: 15 | persist-credentials: false 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v4 19 | with: 20 | go-version: "1.21.0" 21 | 22 | - name: test 23 | run: go test -v ./ 24 | 25 | staticcheck: 26 | runs-on: ubuntu-latest 27 | steps: 28 | - uses: actions/checkout@v3 29 | with: 30 | persist-credentials: false 31 | 32 | - name: Set up Go 33 | uses: actions/setup-go@v4 34 | with: 35 | go-version: "1.21.0" 36 | 37 | - uses: dominikh/staticcheck-action@v1.3.0 38 | with: 39 | install-go: false 40 | version: "2023.1.5" 41 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2023 ISRG. All rights reserved. 2 | 3 | Mozilla Public License Version 2.0 4 | ================================== 5 | 6 | 1. Definitions 7 | -------------- 8 | 9 | 1.1. "Contributor" 10 | means each individual or legal entity that creates, contributes to 11 | the creation of, or owns Covered Software. 12 | 13 | 1.2. "Contributor Version" 14 | means the combination of the Contributions of others (if any) used 15 | by a Contributor and that particular Contributor's Contribution. 16 | 17 | 1.3. "Contribution" 18 | means Covered Software of a particular Contributor. 19 | 20 | 1.4. "Covered Software" 21 | means Source Code Form to which the initial Contributor has attached 22 | the notice in Exhibit A, the Executable Form of such Source Code 23 | Form, and Modifications of such Source Code Form, in each case 24 | including portions thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | (a) that the initial Contributor has attached the notice described 30 | in Exhibit B to the Covered Software; or 31 | 32 | (b) that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the 34 | terms of a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | means any form of the work other than Source Code Form. 38 | 39 | 1.7. "Larger Work" 40 | means a work that combines Covered Software with other material, in 41 | a separate file or files, that is not Covered Software. 42 | 43 | 1.8. "License" 44 | means this document. 45 | 46 | 1.9. "Licensable" 47 | means having the right to grant, to the maximum extent possible, 48 | whether at the time of the initial grant or subsequently, any and 49 | all of the rights conveyed by this License. 50 | 51 | 1.10. "Modifications" 52 | means any of the following: 53 | 54 | (a) any file in Source Code Form that results from an addition to, 55 | deletion from, or modification of the contents of Covered 56 | Software; or 57 | 58 | (b) any new file in Source Code Form that contains any Covered 59 | Software. 60 | 61 | 1.11. "Patent Claims" of a Contributor 62 | means any patent claim(s), including without limitation, method, 63 | process, and apparatus claims, in any patent Licensable by such 64 | Contributor that would be infringed, but for the grant of the 65 | License, by the making, using, selling, offering for sale, having 66 | made, import, or transfer of either its Contributions or its 67 | Contributor Version. 68 | 69 | 1.12. "Secondary License" 70 | means either the GNU General Public License, Version 2.0, the GNU 71 | Lesser General Public License, Version 2.1, the GNU Affero General 72 | Public License, Version 3.0, or any later versions of those 73 | licenses. 74 | 75 | 1.13. "Source Code Form" 76 | means the form of the work preferred for making modifications. 77 | 78 | 1.14. "You" (or "Your") 79 | means an individual or a legal entity exercising rights under this 80 | License. For legal entities, "You" includes any entity that 81 | controls, is controlled by, or is under common control with You. For 82 | purposes of this definition, "control" means (a) the power, direct 83 | or indirect, to cause the direction or management of such entity, 84 | whether by contract or otherwise, or (b) ownership of more than 85 | fifty percent (50%) of the outstanding shares or beneficial 86 | ownership of such entity. 87 | 88 | 2. License Grants and Conditions 89 | -------------------------------- 90 | 91 | 2.1. Grants 92 | 93 | Each Contributor hereby grants You a world-wide, royalty-free, 94 | non-exclusive license: 95 | 96 | (a) under intellectual property rights (other than patent or trademark) 97 | Licensable by such Contributor to use, reproduce, make available, 98 | modify, display, perform, distribute, and otherwise exploit its 99 | Contributions, either on an unmodified basis, with Modifications, or 100 | as part of a Larger Work; and 101 | 102 | (b) under Patent Claims of such Contributor to make, use, sell, offer 103 | for sale, have made, import, and otherwise transfer either its 104 | Contributions or its Contributor Version. 105 | 106 | 2.2. Effective Date 107 | 108 | The licenses granted in Section 2.1 with respect to any Contribution 109 | become effective for each Contribution on the date the Contributor first 110 | distributes such Contribution. 111 | 112 | 2.3. Limitations on Grant Scope 113 | 114 | The licenses granted in this Section 2 are the only rights granted under 115 | this License. No additional rights or licenses will be implied from the 116 | distribution or licensing of Covered Software under this License. 117 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 118 | Contributor: 119 | 120 | (a) for any code that a Contributor has removed from Covered Software; 121 | or 122 | 123 | (b) for infringements caused by: (i) Your and any other third party's 124 | modifications of Covered Software, or (ii) the combination of its 125 | Contributions with other software (except as part of its Contributor 126 | Version); or 127 | 128 | (c) under Patent Claims infringed by Covered Software in the absence of 129 | its Contributions. 130 | 131 | This License does not grant any rights in the trademarks, service marks, 132 | or logos of any Contributor (except as may be necessary to comply with 133 | the notice requirements in Section 3.4). 134 | 135 | 2.4. Subsequent Licenses 136 | 137 | No Contributor makes additional grants as a result of Your choice to 138 | distribute the Covered Software under a subsequent version of this 139 | License (see Section 10.2) or under the terms of a Secondary License (if 140 | permitted under the terms of Section 3.3). 141 | 142 | 2.5. Representation 143 | 144 | Each Contributor represents that the Contributor believes its 145 | Contributions are its original creation(s) or it has sufficient rights 146 | to grant the rights to its Contributions conveyed by this License. 147 | 148 | 2.6. Fair Use 149 | 150 | This License is not intended to limit any rights You have under 151 | applicable copyright doctrines of fair use, fair dealing, or other 152 | equivalents. 153 | 154 | 2.7. Conditions 155 | 156 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 157 | in Section 2.1. 158 | 159 | 3. Responsibilities 160 | ------------------- 161 | 162 | 3.1. Distribution of Source Form 163 | 164 | All distribution of Covered Software in Source Code Form, including any 165 | Modifications that You create or to which You contribute, must be under 166 | the terms of this License. You must inform recipients that the Source 167 | Code Form of the Covered Software is governed by the terms of this 168 | License, and how they can obtain a copy of this License. You may not 169 | attempt to alter or restrict the recipients' rights in the Source Code 170 | Form. 171 | 172 | 3.2. Distribution of Executable Form 173 | 174 | If You distribute Covered Software in Executable Form then: 175 | 176 | (a) such Covered Software must also be made available in Source Code 177 | Form, as described in Section 3.1, and You must inform recipients of 178 | the Executable Form how they can obtain a copy of such Source Code 179 | Form by reasonable means in a timely manner, at a charge no more 180 | than the cost of distribution to the recipient; and 181 | 182 | (b) You may distribute such Executable Form under the terms of this 183 | License, or sublicense it under different terms, provided that the 184 | license for the Executable Form does not attempt to limit or alter 185 | the recipients' rights in the Source Code Form under this License. 186 | 187 | 3.3. Distribution of a Larger Work 188 | 189 | You may create and distribute a Larger Work under terms of Your choice, 190 | provided that You also comply with the requirements of this License for 191 | the Covered Software. If the Larger Work is a combination of Covered 192 | Software with a work governed by one or more Secondary Licenses, and the 193 | Covered Software is not Incompatible With Secondary Licenses, this 194 | License permits You to additionally distribute such Covered Software 195 | under the terms of such Secondary License(s), so that the recipient of 196 | the Larger Work may, at their option, further distribute the Covered 197 | Software under the terms of either this License or such Secondary 198 | License(s). 199 | 200 | 3.4. Notices 201 | 202 | You may not remove or alter the substance of any license notices 203 | (including copyright notices, patent notices, disclaimers of warranty, 204 | or limitations of liability) contained within the Source Code Form of 205 | the Covered Software, except that You may alter any license notices to 206 | the extent required to remedy known factual inaccuracies. 207 | 208 | 3.5. Application of Additional Terms 209 | 210 | You may choose to offer, and to charge a fee for, warranty, support, 211 | indemnity or liability obligations to one or more recipients of Covered 212 | Software. However, You may do so only on Your own behalf, and not on 213 | behalf of any Contributor. You must make it absolutely clear that any 214 | such warranty, support, indemnity, or liability obligation is offered by 215 | You alone, and You hereby agree to indemnify every Contributor for any 216 | liability incurred by such Contributor as a result of warranty, support, 217 | indemnity or liability terms You offer. You may include additional 218 | disclaimers of warranty and limitations of liability specific to any 219 | jurisdiction. 220 | 221 | 4. Inability to Comply Due to Statute or Regulation 222 | --------------------------------------------------- 223 | 224 | If it is impossible for You to comply with any of the terms of this 225 | License with respect to some or all of the Covered Software due to 226 | statute, judicial order, or regulation then You must: (a) comply with 227 | the terms of this License to the maximum extent possible; and (b) 228 | describe the limitations and the code they affect. Such description must 229 | be placed in a text file included with all distributions of the Covered 230 | Software under this License. Except to the extent prohibited by statute 231 | or regulation, such description must be sufficiently detailed for a 232 | recipient of ordinary skill to be able to understand it. 233 | 234 | 5. Termination 235 | -------------- 236 | 237 | 5.1. The rights granted under this License will terminate automatically 238 | if You fail to comply with any of its terms. However, if You become 239 | compliant, then the rights granted under this License from a particular 240 | Contributor are reinstated (a) provisionally, unless and until such 241 | Contributor explicitly and finally terminates Your grants, and (b) on an 242 | ongoing basis, if such Contributor fails to notify You of the 243 | non-compliance by some reasonable means prior to 60 days after You have 244 | come back into compliance. Moreover, Your grants from a particular 245 | Contributor are reinstated on an ongoing basis if such Contributor 246 | notifies You of the non-compliance by some reasonable means, this is the 247 | first time You have received notice of non-compliance with this License 248 | from such Contributor, and You become compliant prior to 30 days after 249 | Your receipt of the notice. 250 | 251 | 5.2. If You initiate litigation against any entity by asserting a patent 252 | infringement claim (excluding declaratory judgment actions, 253 | counter-claims, and cross-claims) alleging that a Contributor Version 254 | directly or indirectly infringes any patent, then the rights granted to 255 | You by any and all Contributors for the Covered Software under Section 256 | 2.1 of this License shall terminate. 257 | 258 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 259 | end user license agreements (excluding distributors and resellers) which 260 | have been validly granted by You or Your distributors under this License 261 | prior to termination shall survive termination. 262 | 263 | ************************************************************************ 264 | * * 265 | * 6. Disclaimer of Warranty * 266 | * ------------------------- * 267 | * * 268 | * Covered Software is provided under this License on an "as is" * 269 | * basis, without warranty of any kind, either expressed, implied, or * 270 | * statutory, including, without limitation, warranties that the * 271 | * Covered Software is free of defects, merchantable, fit for a * 272 | * particular purpose or non-infringing. The entire risk as to the * 273 | * quality and performance of the Covered Software is with You. * 274 | * Should any Covered Software prove defective in any respect, You * 275 | * (not any Contributor) assume the cost of any necessary servicing, * 276 | * repair, or correction. This disclaimer of warranty constitutes an * 277 | * essential part of this License. No use of any Covered Software is * 278 | * authorized under this License except under this disclaimer. * 279 | * * 280 | ************************************************************************ 281 | 282 | ************************************************************************ 283 | * * 284 | * 7. Limitation of Liability * 285 | * -------------------------- * 286 | * * 287 | * Under no circumstances and under no legal theory, whether tort * 288 | * (including negligence), contract, or otherwise, shall any * 289 | * Contributor, or anyone who distributes Covered Software as * 290 | * permitted above, be liable to You for any direct, indirect, * 291 | * special, incidental, or consequential damages of any character * 292 | * including, without limitation, damages for lost profits, loss of * 293 | * goodwill, work stoppage, computer failure or malfunction, or any * 294 | * and all other commercial damages or losses, even if such party * 295 | * shall have been informed of the possibility of such damages. This * 296 | * limitation of liability shall not apply to liability for death or * 297 | * personal injury resulting from such party's negligence to the * 298 | * extent applicable law prohibits such limitation. Some * 299 | * jurisdictions do not allow the exclusion or limitation of * 300 | * incidental or consequential damages, so this exclusion and * 301 | * limitation may not apply to You. * 302 | * * 303 | ************************************************************************ 304 | 305 | 8. Litigation 306 | ------------- 307 | 308 | Any litigation relating to this License may be brought only in the 309 | courts of a jurisdiction where the defendant maintains its principal 310 | place of business and such litigation shall be governed by laws of that 311 | jurisdiction, without reference to its conflict-of-law provisions. 312 | Nothing in this Section shall prevent a party's ability to bring 313 | cross-claims or counter-claims. 314 | 315 | 9. Miscellaneous 316 | ---------------- 317 | 318 | This License represents the complete agreement concerning the subject 319 | matter hereof. If any provision of this License is held to be 320 | unenforceable, such provision shall be reformed only to the extent 321 | necessary to make it enforceable. Any law or regulation which provides 322 | that the language of a contract shall be construed against the drafter 323 | shall not be used to construe this License against a Contributor. 324 | 325 | 10. Versions of the License 326 | --------------------------- 327 | 328 | 10.1. New Versions 329 | 330 | Mozilla Foundation is the license steward. Except as provided in Section 331 | 10.3, no one other than the license steward has the right to modify or 332 | publish new versions of this License. Each version will be given a 333 | distinguishing version number. 334 | 335 | 10.2. Effect of New Versions 336 | 337 | You may distribute the Covered Software under the terms of the version 338 | of the License under which You originally received the Covered Software, 339 | or under the terms of any subsequent version published by the license 340 | steward. 341 | 342 | 10.3. Modified Versions 343 | 344 | If you create software not governed by this License, and you want to 345 | create a new license for such software, you may create and use a 346 | modified version of this License if you rename the license and remove 347 | any references to the name of the license steward (except to note that 348 | such modified license differs from this License). 349 | 350 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 351 | Licenses 352 | 353 | If You choose to distribute Source Code Form that is Incompatible With 354 | Secondary Licenses under the terms of this version of the License, the 355 | notice described in Exhibit B of this License must be attached. 356 | 357 | Exhibit A - Source Code Form License Notice 358 | ------------------------------------------- 359 | 360 | This Source Code Form is subject to the terms of the Mozilla Public 361 | License, v. 2.0. If a copy of the MPL was not distributed with this 362 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 363 | 364 | If it is not possible or desirable to put the notice in a particular 365 | file, then You may include the notice in a location (such as a LICENSE 366 | file in a relevant directory) where a recipient would be likely to look 367 | for such a notice. 368 | 369 | You may add additional accurate notices of copyright ownership. 370 | 371 | Exhibit B - "Incompatible With Secondary Licenses" Notice 372 | --------------------------------------------------------- 373 | 374 | This Source Code Form is "Incompatible With Secondary Licenses", as 375 | defined by the Mozilla Public License, v. 2.0. 376 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CTile 2 | 3 | This is a caching proxy for the get-entries endpoint of a CT log, which uses S3 4 | as its backing store. It uses the concept of "tiles" of entries, where each tile 5 | is a fixed size N (e.g. 256 log entries) and the sequence of tiles starts at 0. 6 | Regardless of what `start` and `end` parameters CTile receives for a request, it 7 | will transform those into a tile-sized request to its backend, by rounding down 8 | `start` to the nearest multiple of N and requesting exactly N items from the 9 | backend. If the request is successful, CTile checks that the response contains 10 | exactly N items, re-encodes as gzipped CBOR, and stores the result in S3. It then 11 | returns modified JSON to the user, removing items from the head and tail to ensure 12 | that the first entry actually corresponds to the first entry requested by the user 13 | and that the response includes at most as many entries as requested. 14 | 15 | When looking up entries in the cache, CTile also rounds `start` down to the 16 | nearest multiple of N, and requests a single tile from the S3 backend. The CT 17 | protocol allows the server to return fewer results than requested, so CTile does 18 | not attempt to request multiple tiles to fulfil a large request. If a request's 19 | `start` parameter is one less than the end of a tile, CTile will respond with a 20 | single entry. This is similar to how Trillian's [align_getentries 21 | flag](https://github.com/google/certificate-transparency-go/blob/6e118585d9d9757b739353829becec378f47e10b/trillian/ctfe/handlers.go#L50) 22 | works, and is in fact compatible with that flag, so long as CTile's tile size is 23 | less than or equal to Trillian's max_get_entries flag. 24 | 25 | When a user requests a range of get-entries near the end of the log, CTile 26 | usually won't be able to get a full tile's worth of entries from the backend, 27 | because the requisite number of entries haven't been sequenced yet. In this 28 | case, CTile does not write anything to the S3 backend and simply passes 29 | through the entries returned from the server (after appropriate tweaks to match 30 | the start and end parameters from the user request). 31 | 32 | # How To 33 | 34 | You must have an S3 bucket set up, and AWS credentials for a role that has read 35 | and write access to that S3 bucket. CTile uses the AWS Go SDK with the default 36 | credential provider, and so will [pull credential 37 | information](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials) 38 | from environment variables, an AWS config file, or ambient credentials for an 39 | EC2 instance. You'll need to manually specify the AWS region for your S3 bucket 40 | by setting the environment variable AWS_REGION. 41 | 42 | You must also know the maximum get-entries size for the log you are mirroring. 43 | If you operate the log, you will know this from your own configs. Otherwise, you 44 | can figure it out by making a get-entries request with `end` much larger than 45 | `start`, and counting the entries. You should set CTile's tile-size to exactly 46 | equal this maximum. It's possible to set a tile-size lower, but only if the log 47 | is not using `align_getentries`. 48 | 49 | Example invocation: 50 | 51 | ``` 52 | export AWS_REGION=us-west-2 53 | go run . -log-url https://oak.ct.letsencrypt.org/2023 \ 54 | -tile-size 256 -s3-bucket some-bucket -full-request-timeout 30s -s3-prefix oak2023 55 | ``` 56 | 57 | ``` 58 | curl 'localhost:8080/ct/v1/get-entries?start=0&end=999999999' -i | less 59 | ``` 60 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/letsencrypt/ctile 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/NYTimes/gziphandler v1.1.1 7 | github.com/aws/aws-sdk-go-v2 v1.21.0 8 | github.com/aws/aws-sdk-go-v2/config v1.18.37 9 | github.com/aws/aws-sdk-go-v2/credentials v1.13.35 10 | github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 11 | github.com/fxamacker/cbor/v2 v2.5.0 12 | github.com/prometheus/client_golang v1.16.0 13 | golang.org/x/sync v0.3.0 14 | ) 15 | 16 | require ( 17 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect 18 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect 19 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect 20 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect 21 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect 22 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect 23 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect 24 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect 25 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect 26 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect 27 | github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect 28 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect 29 | github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect 30 | github.com/aws/smithy-go v1.14.2 // indirect 31 | github.com/beorn7/perks v1.0.1 // indirect 32 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 33 | github.com/davecgh/go-spew v1.1.1 // indirect 34 | github.com/golang/protobuf v1.5.3 // indirect 35 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 36 | github.com/prometheus/client_model v0.3.0 // indirect 37 | github.com/prometheus/common v0.42.0 // indirect 38 | github.com/prometheus/procfs v0.10.1 // indirect 39 | github.com/x448/float16 v0.8.4 // indirect 40 | golang.org/x/sys v0.8.0 // indirect 41 | google.golang.org/protobuf v1.30.0 // indirect 42 | ) 43 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= 2 | github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= 3 | github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= 4 | github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= 5 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= 6 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= 7 | github.com/aws/aws-sdk-go-v2/config v1.18.37 h1:RNAfbPqw1CstCooHaTPhScz7z1PyocQj0UL+l95CgzI= 8 | github.com/aws/aws-sdk-go-v2/config v1.18.37/go.mod h1:8AnEFxW9/XGKCbjYDCJy7iltVNyEI9Iu9qC21UzhhgQ= 9 | github.com/aws/aws-sdk-go-v2/credentials v1.13.35 h1:QpsNitYJu0GgvMBLUIYu9H4yryA5kMksjeIVQfgXrt8= 10 | github.com/aws/aws-sdk-go-v2/credentials v1.13.35/go.mod h1:o7rCaLtvK0hUggAGclf76mNGGkaG5a9KWlp+d9IpcV8= 11 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= 12 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= 13 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= 14 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= 15 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= 16 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= 17 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= 18 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= 19 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= 20 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= 21 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= 22 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= 23 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= 24 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= 25 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= 26 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= 27 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= 28 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= 29 | github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= 30 | github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= 31 | github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 h1:oCvTFSDi67AX0pOX3PuPdGFewvLRU2zzFSrTsgURNo0= 32 | github.com/aws/aws-sdk-go-v2/service/sso v1.13.5/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= 33 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 h1:dnInJb4S0oy8aQuri1mV6ipLlnZPfnsDNB9BGO9PDNY= 34 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= 35 | github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= 36 | github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= 37 | github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= 38 | github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= 39 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 40 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 41 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 42 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 43 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 44 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 45 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 46 | github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= 47 | github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= 48 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 49 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 50 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 51 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 52 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 53 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 54 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 55 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 56 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 57 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 58 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 59 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 60 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 61 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 62 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 63 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 64 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 65 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 66 | github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= 67 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 68 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 69 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 70 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 71 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 72 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 73 | github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= 74 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 75 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 76 | golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= 77 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 78 | golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= 79 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 80 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 81 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 82 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 83 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= 84 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 85 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 86 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 87 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 88 | -------------------------------------------------------------------------------- /integration_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "context" 7 | "encoding/binary" 8 | "encoding/json" 9 | "errors" 10 | "io" 11 | "net" 12 | "net/http" 13 | "net/http/httptest" 14 | "os/exec" 15 | "reflect" 16 | "strconv" 17 | "strings" 18 | "syscall" 19 | "testing" 20 | "time" 21 | 22 | "github.com/aws/aws-sdk-go-v2/aws" 23 | "github.com/aws/aws-sdk-go-v2/config" 24 | "github.com/aws/aws-sdk-go-v2/credentials" 25 | "github.com/aws/aws-sdk-go-v2/service/s3" 26 | "github.com/prometheus/client_golang/prometheus" 27 | "github.com/prometheus/client_golang/prometheus/testutil" 28 | ) 29 | 30 | const containerName string = "ctile_integration_test_minio" 31 | const testLogSaysPastTheEnd string = "oh no! we fell off the end of the log!" 32 | 33 | func startContainer(t *testing.T) { 34 | _, err := exec.Command("podman", "run", "--rm", "--detach", "-p", "19085:9000", "--name", containerName, "quay.io/minio/minio", "server", "/data").Output() 35 | if err != nil { 36 | t.Fatalf("minio failed to come up: %v", err) 37 | } 38 | for i := 0; i < 1000; i++ { 39 | _, err := net.Dial("tcp", "localhost:19085") 40 | if errors.Is(err, syscall.ECONNREFUSED) { 41 | t.Log("sleeping 10ms waiting for minio to come up") 42 | time.Sleep(10 * time.Millisecond) 43 | continue 44 | } 45 | if err != nil { 46 | t.Fatalf("failed to connect to minio: %v", err) 47 | } 48 | t.Log("minio is up") 49 | return 50 | } 51 | t.Fatalf("failed to connect to minio: %v", err) 52 | } 53 | 54 | // cleanupContainer stops a running named container and removes its assigned 55 | // name. This is helpful in the event that a container wasn't properly killed 56 | // during a previous test run or if manual testing was being performed and not 57 | // cleaned up. 58 | func cleanupContainer() { 59 | // Unconditionally stop the container. 60 | _, _ = exec.Command("podman", "stop", containerName).Output() 61 | 62 | // Unconditionally remove the container name if the operator did manual 63 | // container testing, but didn't clean up the name. 64 | _, _ = exec.Command("podman", "rm", containerName).Output() 65 | } 66 | 67 | func TestIntegration(t *testing.T) { 68 | cleanupContainer() // Clean up old containers and names just in case. 69 | startContainer(t) 70 | defer cleanupContainer() 71 | 72 | // A test CT server that responds to get-entries requests with appropriately JSON-formatted 73 | // data, where base64-decoding the LeafInput and ExtraData fields yields a binary encoding 74 | // of the position of the given element. 75 | // 76 | // This acts like a CT log with a max_getentries limit of 3 and 10 elements in total. 77 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 78 | if r.URL.Path != "/ct/v1/get-entries" { 79 | w.WriteHeader(http.StatusNotFound) 80 | return 81 | } 82 | startInt, _ := strconv.ParseInt(r.URL.Query().Get("start"), 10, 64) 83 | endInt, _ := strconv.ParseInt(r.URL.Query().Get("end"), 10, 64) 84 | var entries entries 85 | 86 | // Behave as if the CT server has a max_get_entries limit of 3. 87 | // The +1 and -1 are because CT uses closed intervals. 88 | if endInt-startInt+1 > 3 { 89 | endInt = startInt + 3 - 1 90 | } 91 | 92 | // Behave as if the CT server has a total of 10 entries 93 | if startInt > 10 { 94 | w.WriteHeader(http.StatusBadRequest) 95 | w.Write([]byte(testLogSaysPastTheEnd)) 96 | return 97 | } 98 | 99 | if endInt > 10 { 100 | endInt = 10 101 | } 102 | 103 | for i := startInt; i <= endInt; i++ { 104 | // Put fake data in leafInput and extraData. Normally these would contain 105 | // certificates, but instead we encode the offset within the log, which 106 | // allows us to check later that we got the correct log offsets. 107 | leafInput := make([]byte, 8) 108 | binary.PutVarint(leafInput, i) 109 | extraData := make([]byte, 8) 110 | binary.PutVarint(extraData, i) 111 | entries.Entries = append(entries.Entries, entry{ 112 | LeafInput: leafInput, 113 | ExtraData: extraData, 114 | }) 115 | } 116 | 117 | encoder := json.NewEncoder(w) 118 | encoder.Encode(entries) 119 | })) 120 | defer server.Close() 121 | 122 | const defaultRegion = "fakeRegion" 123 | hostAddress := "http://localhost:19085" 124 | 125 | resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { 126 | return aws.Endpoint{ 127 | PartitionID: "aws", 128 | URL: hostAddress, 129 | SigningRegion: defaultRegion, 130 | HostnameImmutable: true, 131 | }, nil 132 | }) 133 | 134 | cfg, err := config.LoadDefaultConfig(context.Background(), 135 | config.WithRegion(defaultRegion), 136 | config.WithEndpointResolverWithOptions(resolver), 137 | config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "")), 138 | ) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | s3Service := s3.NewFromConfig(cfg) 143 | 144 | _, err = s3Service.CreateBucket(context.Background(), &s3.CreateBucketInput{ 145 | Bucket: aws.String("bucket"), 146 | }) 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | 151 | ctile := makeTCH(t, server.URL, s3Service) 152 | 153 | // Invalid URL; should 404 passed through to backend and 400 154 | resp := getResp(ctile, "/foo") 155 | if resp.StatusCode != 404 { 156 | t.Errorf("expected 404 got %d", resp.StatusCode) 157 | } 158 | 159 | // Malformed queries; should 400 160 | malformed := []string{ 161 | "/ct/v1/get-entries?start=a&end=b", 162 | "/ct/v1/get-entries?start=1&end=b", 163 | "/ct/v1/get-entries?start=a&end=1", 164 | "/ct/v1/get-entries?start=1&end=0", 165 | "/ct/v1/get-entries?start=-1&end=1", 166 | "/ct/v1/get-entries?start=1&end=-1", 167 | "/ct/v1/get-entries?start=1", 168 | "/ct/v1/get-entries?end=1", 169 | } 170 | for _, m := range malformed { 171 | resp := getResp(ctile, m) 172 | if resp.StatusCode != 400 { 173 | t.Errorf("%q: expected 400 got %d", m, resp.StatusCode) 174 | } 175 | } 176 | 177 | // Valid query; should 200 178 | twoEntriesA, headers, err := getAndParseResp(t, ctile, "/ct/v1/get-entries?start=3&end=4") 179 | if err != nil { 180 | t.Error(err) 181 | } 182 | 183 | expectHeader(t, headers, "Content-Type", "application/json") 184 | expectHeader(t, headers, "X-Source", "CT log") 185 | 186 | if len(twoEntriesA.Entries) != 2 { 187 | t.Errorf("expected 2 entries got %d", len(twoEntriesA.Entries)) 188 | } 189 | 190 | n, err := binary.ReadVarint(bytes.NewReader(twoEntriesA.Entries[0].LeafInput)) 191 | if err != nil { 192 | t.Error(err) 193 | } 194 | 195 | if n != 3 { 196 | t.Errorf("expected first leafinput in response to be 3rd in log overall got %d", n) 197 | } 198 | 199 | n, err = binary.ReadVarint(bytes.NewReader(twoEntriesA.Entries[1].LeafInput)) 200 | if err != nil { 201 | t.Error(err) 202 | } 203 | 204 | if n != 4 { 205 | t.Errorf("expected second leaf_input in response to be 4th in log overall got %d", n) 206 | } 207 | 208 | successes := testutil.ToFloat64(ctile.requestsMetric.WithLabelValues("success", "ct_log_get")) 209 | if successes != 1 { 210 | t.Errorf("expected 1 success from ct_log_get, got %g", successes) 211 | } 212 | ctile.requestsMetric.Reset() 213 | 214 | // Same query again; should come from S3 this time. 215 | twoEntriesB, headers, err := getAndParseResp(t, ctile, "/ct/v1/get-entries?start=3&end=4") 216 | if err != nil { 217 | t.Error(err) 218 | } 219 | 220 | expectHeader(t, headers, "Content-Type", "application/json") 221 | expectHeader(t, headers, "X-Source", "S3") 222 | expectAndResetMetric(t, ctile.requestsMetric, 1, "success", "s3_get") 223 | 224 | if len(twoEntriesB.Entries) != 2 { 225 | t.Errorf("expected 2 entries got %d", len(twoEntriesB.Entries)) 226 | } 227 | 228 | // Same query with a different prefix; should succeed 229 | _, _, err = getAndParseResp(t, ctile, "/ctile/ct/v1/get-entries?start=3&end=4") 230 | if err != nil { 231 | t.Error(err) 232 | } 233 | ctile.requestsMetric.Reset() 234 | 235 | // The results from the first and second queries should be the same 236 | if !reflect.DeepEqual(twoEntriesA, twoEntriesB) { 237 | t.Errorf("expected equal responses got %#v != %#v", twoEntriesA, twoEntriesB) 238 | } 239 | 240 | // The third entry in this first tile should also be served from S3 now, because it 241 | // was pulled into cache by the previous requests. 242 | oneEntry, headers, err := getAndParseResp(t, ctile, "/ct/v1/get-entries?start=5&end=5") 243 | if err != nil { 244 | t.Error(err) 245 | } 246 | 247 | expectHeader(t, headers, "X-Source", "S3") 248 | expectAndResetMetric(t, ctile.requestsMetric, 1, "success", "s3_get") 249 | 250 | if len(oneEntry.Entries) != 1 { 251 | t.Errorf("expected 1 entry got %d", len(oneEntry.Entries)) 252 | } 253 | 254 | // Tiles fetched from the end of the log will be partial. CTile should not cache. 255 | _, headers, err = getAndParseResp(t, ctile, "/ct/v1/get-entries?start=9&end=11") 256 | if err != nil { 257 | t.Error(err) 258 | } 259 | 260 | expectHeader(t, headers, "X-Source", "CT log") 261 | expectAndResetMetric(t, ctile.requestsMetric, 1, "success", "ct_log_get") 262 | 263 | _, headers, err = getAndParseResp(t, ctile, "/ct/v1/get-entries?start=9&end=11") 264 | if err != nil { 265 | t.Error(err) 266 | } 267 | 268 | // This should still come from the CT log rather than from S3, even though it was 269 | // requested twice in a row. 270 | expectHeader(t, headers, "X-Source", "CT log") 271 | expectAndResetMetric(t, ctile.requestsMetric, 1, "success", "ct_log_get") 272 | 273 | // Tiles fetched past the end of the log will get a 400 from our test CT log; ctile 274 | // should pass that through, along with the body. 275 | resp = getResp(ctile, "/ct/v1/get-entries?start=99&end=100") 276 | if resp.StatusCode != 400 { 277 | t.Errorf("expected 400 got %d", resp.StatusCode) 278 | } 279 | gzReader, err := gzip.NewReader(resp.Body) 280 | if err != nil { 281 | t.Fatal(err) 282 | } 283 | body, _ := io.ReadAll(gzReader) 284 | if !strings.Contains(string(body), testLogSaysPastTheEnd) { 285 | t.Errorf("expected response to contain %q got %q", testLogSaysPastTheEnd, body) 286 | } 287 | expectAndResetMetric(t, ctile.requestsMetric, 1, "bad_request", "ct_log_get") 288 | 289 | // A request where the _tile_ starts inside the log but the requested `start` value is 290 | // outside the log. In this case ctile synthesizes a 400. 291 | resp = getResp(ctile, "/ct/v1/get-entries?start=11&end=12") 292 | if resp.StatusCode != 400 { 293 | t.Errorf("expected 400 got %d", resp.StatusCode) 294 | } 295 | body, _ = io.ReadAll(resp.Body) 296 | pastTheEnd := "requested range is past the end of the log" 297 | if !strings.Contains(string(body), pastTheEnd) { 298 | t.Errorf("expected response to contain %q got %q", pastTheEnd, body) 299 | } 300 | expectAndResetMetric(t, ctile.requestsMetric, 1, "bad_request", "past_the_end_partial_tile") 301 | 302 | // simulate a down backend 303 | errorCTLog := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 304 | w.WriteHeader(http.StatusInternalServerError) 305 | })) 306 | defer server.Close() 307 | 308 | erroringCTile := makeTCH(t, errorCTLog.URL, s3Service) 309 | resp = getResp(erroringCTile, "/ct/v1/get-entries?start=0&end=1") 310 | if resp.StatusCode != 500 { 311 | t.Errorf("expected 500 got %d", resp.StatusCode) 312 | } 313 | expectAndResetMetric(t, erroringCTile.requestsMetric, 1, "error", "ct_log_get") 314 | } 315 | 316 | func getResp(ctile *tileCachingHandler, url string) *http.Response { 317 | req := httptest.NewRequest("GET", url, nil) 318 | req.Header.Set("Accept-Encoding", "gzip") 319 | w := httptest.NewRecorder() 320 | 321 | ctile.ServeHTTP(w, req) 322 | 323 | return w.Result() 324 | } 325 | 326 | func getAndParseResp(t *testing.T, ctile *tileCachingHandler, url string) (entries, http.Header, error) { 327 | t.Helper() 328 | resp := getResp(ctile, url) 329 | body, _ := io.ReadAll(resp.Body) 330 | if resp.StatusCode != 200 { 331 | t.Fatalf("%q: expected status code 200 got %d with body: %q", url, resp.StatusCode, body) 332 | } 333 | if resp.Header.Get("Content-Encoding") != "gzip" { 334 | t.Fatalf("expected Content-Encoding: gzip, got %q", resp.Header.Get("Content-Encoding")) 335 | } 336 | gzipReader, err := gzip.NewReader(bytes.NewReader(body)) 337 | if err != nil { 338 | t.Fatal(err) 339 | } 340 | jsonBytes, err := io.ReadAll(gzipReader) 341 | if err != nil { 342 | t.Fatal(err) 343 | } 344 | 345 | var entries entries 346 | err = json.Unmarshal(jsonBytes, &entries) 347 | return entries, resp.Header, err 348 | } 349 | 350 | func expectHeader(t *testing.T, headers http.Header, key, expected string) { 351 | t.Helper() 352 | if headers.Get(key) != expected { 353 | t.Errorf("header %q: expected %q got %q", key, expected, headers.Get(key)) 354 | } 355 | } 356 | 357 | func expectAndResetMetric(t *testing.T, metric *prometheus.CounterVec, expected float64, labels ...string) { 358 | value := testutil.ToFloat64(metric.WithLabelValues(labels...)) 359 | if value != expected { 360 | t.Errorf("expected Prometheus counter value of %g got %g with labels %s", expected, value, labels) 361 | } 362 | metric.Reset() 363 | } 364 | 365 | func makeTCH(t *testing.T, url string, s3Service *s3.Client) *tileCachingHandler { 366 | tch, err := newTileCachingHandler(url, 3, s3Service, "test", "bucket", 10*time.Second, prometheus.NewRegistry()) 367 | if err != nil { 368 | t.Fatal(err) 369 | } 370 | return tch 371 | } 372 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // main is the entrypoint for the ctile binary. 2 | package main 3 | 4 | import ( 5 | "bytes" 6 | "compress/gzip" 7 | "context" 8 | "encoding/json" 9 | "errors" 10 | "flag" 11 | "fmt" 12 | "io" 13 | "log" 14 | "net/http" 15 | "net/url" 16 | "os" 17 | "strconv" 18 | "strings" 19 | "time" 20 | 21 | "github.com/NYTimes/gziphandler" 22 | "github.com/aws/aws-sdk-go-v2/aws" 23 | "github.com/aws/aws-sdk-go-v2/config" 24 | "github.com/aws/aws-sdk-go-v2/service/s3" 25 | "github.com/aws/aws-sdk-go-v2/service/s3/types" 26 | "github.com/fxamacker/cbor/v2" 27 | "github.com/prometheus/client_golang/prometheus" 28 | "github.com/prometheus/client_golang/prometheus/collectors" 29 | "github.com/prometheus/client_golang/prometheus/promhttp" 30 | "golang.org/x/sync/singleflight" 31 | ) 32 | 33 | // parseQueryParams returns the start and end values, or an error. 34 | // 35 | // The end value it returns is one greater than in the request, 36 | // because CT uses closed intervals while we use half-open intervals 37 | // internally for simpler math. 38 | func parseQueryParams(values url.Values) (int64, int64, error) { 39 | start := values.Get("start") 40 | end := values.Get("end") 41 | if start == "" { 42 | return 0, 0, errors.New("missing start parameter") 43 | } 44 | if end == "" { 45 | return 0, 0, errors.New("missing end parameter") 46 | } 47 | startInt, err := strconv.ParseInt(start, 10, 64) 48 | if err != nil || startInt < 0 { 49 | return 0, 0, fmt.Errorf("invalid start parameter: %w", err) 50 | } 51 | endInt, err := strconv.ParseInt(end, 10, 64) 52 | if err != nil || endInt < 0 { 53 | return 0, 0, fmt.Errorf("invalid end parameter: %w", err) 54 | } 55 | if endInt < startInt { 56 | return 0, 0, errors.New("end must be greater than or equal to start") 57 | } 58 | return startInt, endInt + 1, nil 59 | } 60 | 61 | // tile represents important info about a tile: where it starts, where it ends, its size, 62 | // what CT backend URL it exists on (or is anticipated to exist on), and what s3 prefix 63 | // it should be stored/retrieved under. 64 | // 65 | // `start` is inclusive, and `end` is exclusive, unlike in the CT protocol. 66 | // In other words, they represent the half-open interval [start, end). 67 | type tile struct { 68 | start int64 69 | end int64 70 | size int64 71 | logURL string 72 | } 73 | 74 | // makeTile returns a tile of size `size` that contains the given `start` position. 75 | // The resulting tile's `start` will be equal to or less than the requested `start`. 76 | func makeTile(start, size int64, logURL string) tile { 77 | tileOffset := start % size 78 | tileStart := start - tileOffset 79 | return tile{ 80 | start: tileStart, 81 | end: tileStart + size, 82 | size: size, 83 | logURL: logURL, 84 | } 85 | } 86 | 87 | // key returns the S3 key for the tile. 88 | func (t tile) key() string { 89 | return fmt.Sprintf("tile_size=%d/%d.cbor.gz", t.size, t.start) 90 | } 91 | 92 | // url returns the URL to fetch the tile from the backend. 93 | func (t tile) url() string { 94 | // Use end-1 because our internal representation uses half-open intervals, while the 95 | // CT protocol uses closed intervals. https://datatracker.ietf.org/doc/html/rfc6962#section-4.6 96 | return fmt.Sprintf("%s/ct/v1/get-entries?start=%d&end=%d", t.logURL, t.start, t.end-1) 97 | } 98 | 99 | // entries corresponds to the JSON response to the CT get-entries endpoint. 100 | // https://datatracker.ietf.org/doc/html/rfc6962#section-4.6 101 | // 102 | // It is marshaled and unmarshaled to/from JSON and CBOR. 103 | // 104 | // This type must not be mutated, because pointers to the same value may be in use 105 | // across multiple goroutines. 106 | type entries struct { 107 | Entries []entry `json:"entries"` 108 | } 109 | 110 | type pastTheEndError struct{} 111 | 112 | func (p pastTheEndError) Error() string { 113 | return "requested range is past the end of the log" 114 | } 115 | 116 | // trimForDisplay takes a set of entries corresponding to `tile`, and returns a new 117 | // object suitable for a request for entries in the range [start, end). 118 | // 119 | // This does not mutate the original object. It is suitable for calling when the set 120 | // of entries represents a partial tile. 121 | func (e *entries) trimForDisplay(start, end int64, tile tile) (*entries, error) { 122 | if start < tile.start || start >= tile.end || end <= start || len(e.Entries) > int(tile.size) { 123 | return nil, fmt.Errorf("internal inconsistency: start = %d, end = %d, tile = %v, len(e.Entries) = %d", start, end, tile, len(e.Entries)) 124 | } 125 | 126 | // Truncate to match the request 127 | prefixToRemove := start - tile.start 128 | if prefixToRemove >= int64(len(e.Entries)) { 129 | // In this case, the requested range is entirely outside the current log, 130 | // but the _tile_'s beginning was inside the log. For instance, a log with 131 | // size 1000 and max_getentries of 256, where ctile is handling a request 132 | // for start=1001&end=1001; the tile starts at offset 768, but is partial so 133 | // it doesn't include the requested range. 134 | // 135 | // When Trillian gets a request that is past the end of the log, it returns 136 | // 400 (for better or worse), so we emulate that here. 137 | return nil, pastTheEndError{} 138 | } 139 | 140 | requestedLen := end - start 141 | if prefixToRemove+requestedLen > int64(len(e.Entries)) { 142 | requestedLen = int64(len(e.Entries)) - prefixToRemove 143 | } 144 | return &entries{ 145 | Entries: e.Entries[prefixToRemove : prefixToRemove+requestedLen], 146 | }, nil 147 | } 148 | 149 | // entry corresponds to a single entry in the CT get-entries endpoint. 150 | // 151 | // Note: the JSON fields are base64. For fields of type `[]byte`, Go's encoding/json 152 | // automagically decodes base64. 153 | // 154 | // This type must not be mutated, because pointers to the same value may be in use 155 | // across multiple goroutines. 156 | type entry struct { 157 | LeafInput []byte `json:"leaf_input"` 158 | ExtraData []byte `json:"extra_data"` 159 | } 160 | 161 | // statusCodeError indicates the backend returned a non-200 status code, and contains 162 | // the response body. This allows passing through that status code and body to the requester. 163 | type statusCodeError struct { 164 | statusCode int 165 | body []byte 166 | } 167 | 168 | func (s statusCodeError) Error() string { 169 | return fmt.Sprintf("backend responded with status code %d and body:\n%s", s.statusCode, string(s.body)) 170 | } 171 | 172 | // getTileFromBackend fetches a tile of entries from the backend. 173 | // 174 | // If the backend returns a non-200 status code, it returns a statusCodeError, 175 | // so the caller can handle that case specially by propagating the backend's 176 | // status code (for instance, 400 or 404). 177 | func getTileFromBackend(ctx context.Context, t tile) (*entries, error) { 178 | url := t.url() 179 | r, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 180 | if err != nil { 181 | return nil, fmt.Errorf("unable to create backend Request object: %w", err) 182 | } 183 | resp, err := http.DefaultClient.Do(r) 184 | if err != nil { 185 | return nil, fmt.Errorf("fetching %s: %w", url, err) 186 | } 187 | 188 | if resp.StatusCode != http.StatusOK { 189 | body, err := io.ReadAll(resp.Body) 190 | if err != nil { 191 | return nil, fmt.Errorf("reading body from %s: %w", url, err) 192 | } 193 | return nil, statusCodeError{resp.StatusCode, body} 194 | } 195 | 196 | var entries entries 197 | err = json.NewDecoder(resp.Body).Decode(&entries) 198 | if err != nil { 199 | return nil, fmt.Errorf("reading body from %s: %w", url, err) 200 | } 201 | 202 | if len(entries.Entries) > int(t.size) || len(entries.Entries) == 0 { 203 | return nil, fmt.Errorf("expected %d entries, got %d", t.size, len(entries.Entries)) 204 | } 205 | 206 | return &entries, nil 207 | } 208 | 209 | // writeToS3 stores the entries corresponding to the given tile in s3. 210 | func (tch *tileCachingHandler) writeToS3(ctx context.Context, t tile, e *entries) error { 211 | if len(e.Entries) != int(t.size) || t.end != t.start+t.size { 212 | return fmt.Errorf("internal inconsistency: len(entries) == %d; tile = %v", len(e.Entries), t) 213 | } 214 | 215 | var body bytes.Buffer 216 | w := gzip.NewWriter(&body) 217 | err := cbor.NewEncoder(w).Encode(e) 218 | if err != nil { 219 | return nil 220 | } 221 | 222 | err = w.Close() 223 | if err != nil { 224 | return fmt.Errorf("closing gzip writer: %w", err) 225 | } 226 | 227 | key := tch.s3Prefix + t.key() 228 | _, err = tch.s3Service.PutObject(ctx, &s3.PutObjectInput{ 229 | Bucket: aws.String(tch.s3Bucket), 230 | Key: aws.String(key), 231 | Body: bytes.NewReader(body.Bytes()), 232 | }) 233 | if err != nil { 234 | return fmt.Errorf("putting in bucket %q with key %q: %s", tch.s3Bucket, key, err) 235 | } 236 | return nil 237 | } 238 | 239 | // noSuchKey indicates the requested key does not exist. 240 | type noSuchKey struct{} 241 | 242 | func (noSuchKey) Error() string { 243 | return "no such key" 244 | } 245 | 246 | // getFromS3 retrieves the entries corresponding to the given tile from s3. 247 | // If the tile isn't already stored in s3, it returns a noSuchKey error. 248 | func (tch *tileCachingHandler) getFromS3(ctx context.Context, t tile) (*entries, error) { 249 | key := tch.s3Prefix + t.key() 250 | resp, err := tch.s3Service.GetObject(ctx, &s3.GetObjectInput{ 251 | Bucket: aws.String(tch.s3Bucket), 252 | Key: aws.String(key), 253 | }) 254 | if err != nil { 255 | var nsk *types.NoSuchKey 256 | if errors.As(err, &nsk) { 257 | return nil, noSuchKey{} 258 | } 259 | return nil, fmt.Errorf("getting from bucket %q with key %q: %w", tch.s3Bucket, key, err) 260 | } 261 | 262 | var entries entries 263 | gzipReader, err := gzip.NewReader(resp.Body) 264 | if err != nil { 265 | return nil, fmt.Errorf("making gzipReader: %w", err) 266 | } 267 | err = cbor.NewDecoder(gzipReader).Decode(&entries) 268 | if err != nil { 269 | return nil, fmt.Errorf("reading body from bucket %q with key %q: %w", tch.s3Bucket, key, err) 270 | } 271 | 272 | if len(entries.Entries) != int(t.size) || t.end != t.start+t.size { 273 | return nil, fmt.Errorf("internal inconsistency: len(entries) == %d; tile = %v", len(entries.Entries), t) 274 | } 275 | 276 | return &entries, nil 277 | } 278 | 279 | // tileCachingHandler is the main HTTP handler that serves CT tiles it fetches 280 | // from a backend server and from the cache tiles it maintains in S3. 281 | type tileCachingHandler struct { 282 | logURL string // The string form of the HTTP host and path prefix to add incoming request paths to in order to fetch tiles from the backing CT log. Must not be empty. 283 | tileSize int // The CT tile size used here and in the backing CT log. Must be the same as the backing CT log's value and must not be zero. 284 | 285 | s3Service *s3.Client // The S3 service to use for caching tiles. Must not be nil. 286 | s3Prefix string // The prefix to add to the path when caching tiles in S3. Must not be empty. 287 | s3Bucket string // The S3 bucket to use for caching tiles. Must not be empty. 288 | 289 | cacheGroup *singleflight.Group // The singleflight.Group to use for deduplicating simultaneous requests (a.k.a. "request collapsing") for tiles. Must not be nil. 290 | 291 | requestsMetric *prometheus.CounterVec 292 | partialTiles prometheus.Counter 293 | singleFlightShared prometheus.Counter 294 | latencyMetric prometheus.Histogram 295 | backendLatencyMetric *prometheus.HistogramVec 296 | 297 | fullRequestTimeout time.Duration 298 | 299 | gzipHandler http.Handler 300 | } 301 | 302 | func newTileCachingHandler( 303 | logURL string, 304 | tileSize int, 305 | s3Service *s3.Client, 306 | s3Prefix string, 307 | s3Bucket string, 308 | fullRequestTimeout time.Duration, 309 | promRegisterer prometheus.Registerer, 310 | ) (*tileCachingHandler, error) { 311 | if logURL == "" { 312 | return nil, errors.New("logURL must not be empty") 313 | } 314 | if tileSize == 0 { 315 | return nil, errors.New("tileSize must not be zero") 316 | } 317 | if s3Service == nil { 318 | return nil, errors.New("s3Service must not be nil") 319 | } 320 | if s3Prefix == "" { 321 | return nil, errors.New("s3Prefix must not be empty") 322 | } 323 | if s3Bucket == "" { 324 | return nil, errors.New("s3Bucket must not be empty") 325 | } 326 | if fullRequestTimeout == 0 { 327 | return nil, errors.New("fullRequestTimeout must not be zero") 328 | } 329 | requestsMetric := prometheus.NewCounterVec( 330 | prometheus.CounterOpts{ 331 | Name: "ctile_requests", 332 | Help: "total number of requests, by result and source", 333 | }, 334 | []string{"result", "source"}, 335 | ) 336 | promRegisterer.MustRegister(requestsMetric) 337 | 338 | partialTiles := prometheus.NewCounter( 339 | prometheus.CounterOpts{ 340 | Name: "ctile_partial_tiles", 341 | Help: "number of requests not cached due to partial tile returned from CT log", 342 | }) 343 | promRegisterer.MustRegister(partialTiles) 344 | 345 | singleFlightShared := prometheus.NewCounter( 346 | prometheus.CounterOpts{ 347 | Name: "ctile_single_flight_shared", 348 | Help: "number of inbound requests coalesced into a single set of backend requests", 349 | }) 350 | promRegisterer.MustRegister(singleFlightShared) 351 | 352 | latencyMetric := prometheus.NewHistogram( 353 | prometheus.HistogramOpts{ 354 | Name: "ctile_response_latency_seconds", 355 | Help: "overall latency of responses, including all backend requests", 356 | Buckets: prometheus.DefBuckets, 357 | }) 358 | promRegisterer.MustRegister(latencyMetric) 359 | 360 | backendLatencyMetric := prometheus.NewHistogramVec( 361 | prometheus.HistogramOpts{ 362 | Name: "ctile_backend_latency_seconds", 363 | Help: "latency of each backend request", 364 | Buckets: prometheus.DefBuckets, 365 | }, 366 | []string{"backend"}) 367 | promRegisterer.MustRegister(backendLatencyMetric) 368 | 369 | tch := tileCachingHandler{ 370 | logURL: logURL, 371 | tileSize: tileSize, 372 | s3Service: s3Service, 373 | s3Prefix: s3Prefix, 374 | s3Bucket: s3Bucket, 375 | cacheGroup: &singleflight.Group{}, 376 | requestsMetric: requestsMetric, 377 | partialTiles: partialTiles, 378 | singleFlightShared: singleFlightShared, 379 | fullRequestTimeout: fullRequestTimeout, 380 | latencyMetric: latencyMetric, 381 | backendLatencyMetric: backendLatencyMetric, 382 | } 383 | 384 | handlerMaker, err := gziphandler.NewGzipLevelAndMinSize(gzip.BestSpeed, 100) 385 | if err != nil { 386 | return nil, err 387 | } 388 | 389 | tch.gzipHandler = handlerMaker(http.HandlerFunc(tch.serveHTTPInner)) 390 | 391 | return &tch, nil 392 | } 393 | 394 | func (tch *tileCachingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 395 | tch.gzipHandler.ServeHTTP(w, r) 396 | } 397 | 398 | func (tch *tileCachingHandler) serveHTTPInner(w http.ResponseWriter, r *http.Request) { 399 | begin := time.Now() 400 | defer func() { 401 | tch.latencyMetric.Observe(time.Since(begin).Seconds()) 402 | }() 403 | 404 | if !strings.HasSuffix(r.URL.Path, "/ct/v1/get-entries") { 405 | passthroughHandler{logURL: tch.logURL}.ServeHTTP(w, r) 406 | return 407 | } 408 | start, end, err := parseQueryParams(r.URL.Query()) 409 | if err != nil { 410 | w.WriteHeader(http.StatusBadRequest) 411 | fmt.Fprintln(w, err) 412 | return 413 | } 414 | 415 | ctx, cancel := context.WithTimeout(r.Context(), tch.fullRequestTimeout) 416 | defer cancel() 417 | 418 | tile := makeTile(start, int64(tch.tileSize), tch.logURL) 419 | 420 | contents, source, err := tch.getAndCacheTile(ctx, tile) 421 | if err != nil { 422 | status := http.StatusInternalServerError 423 | var statusCodeErr statusCodeError 424 | if errors.As(err, &statusCodeErr) { 425 | status = statusCodeErr.statusCode 426 | } 427 | // Send errors to our stdout as well as to the user. 428 | if status != http.StatusBadRequest { 429 | log.Println(err) 430 | } 431 | w.WriteHeader(status) 432 | fmt.Fprintln(w, err) 433 | return 434 | } 435 | 436 | if tch.isPartialTile(contents) { 437 | w.Header().Set("X-Partial-Tile", "true") 438 | } 439 | 440 | w.Header().Set("X-Source", string(source)) 441 | 442 | contents, err = contents.trimForDisplay(start, end, tile) 443 | if err != nil { 444 | if errors.As(err, &pastTheEndError{}) { 445 | tch.requestsMetric.WithLabelValues("bad_request", "past_the_end_partial_tile").Inc() 446 | } else { 447 | tch.requestsMetric.WithLabelValues("error", "internal_inconsistency").Inc() 448 | } 449 | w.WriteHeader(http.StatusBadRequest) 450 | fmt.Fprintln(w, err) 451 | return 452 | } 453 | 454 | if w.Header().Get("X-Source") == "S3" { 455 | tch.requestsMetric.WithLabelValues("success", "s3_get").Inc() 456 | } else { 457 | tch.requestsMetric.WithLabelValues("success", "ct_log_get").Inc() 458 | } 459 | 460 | w.Header().Set("X-Response-Len", fmt.Sprintf("%d", len(contents.Entries))) 461 | w.Header().Set("Content-Type", "application/json") 462 | w.WriteHeader(http.StatusOK) 463 | 464 | encoder := json.NewEncoder(w) 465 | encoder.SetIndent("", " ") 466 | encoder.Encode(contents) 467 | } 468 | 469 | // tileSource is a helper enum to indicate to the user whether the tile returned 470 | // to them was found in S3 or in the CT log. 471 | type tileSource string 472 | 473 | const ( 474 | sourceCTLog tileSource = "CT log" 475 | sourceS3 tileSource = "S3" 476 | ) 477 | 478 | // getAndCacheTile fetches the requested tile from S3 if it exists there, or, if 479 | // it doesn't exist in S3, from the backing CT log and then caches it in S3. 480 | // Under the hood, it collapses requests for the same tile into one single 481 | // request. It should be preferred over getAndCacheTileUncollapsed. 482 | func (tch *tileCachingHandler) getAndCacheTile(ctx context.Context, tile tile) (*entries, tileSource, error) { 483 | dedupKey := fmt.Sprintf("logURL-%s-tile-%d-%d", tile.logURL, tile.start, tile.end) 484 | 485 | type entriesAndSource struct { 486 | entries *entries 487 | source tileSource 488 | } 489 | 490 | innerContents, err, shared := singleflightDo(tch.cacheGroup, dedupKey, func() (entriesAndSource, error) { 491 | contents, source, err := tch.getAndCacheTileUncollapsed(ctx, tile) 492 | return entriesAndSource{contents, source}, err 493 | }) 494 | 495 | if shared { 496 | tch.singleFlightShared.Inc() 497 | } 498 | 499 | // The value from our singleflightDo closure is always non-nil, so we don't 500 | // need an err != nil check here. 501 | return innerContents.entries, innerContents.source, err 502 | } 503 | 504 | // getAndCacheTileUncollapsed is the core of getAndCacheTile (and is used by it) 505 | // without the request collapsing. Use getAndCacheTile instead of this method. 506 | func (tch *tileCachingHandler) getAndCacheTileUncollapsed(ctx context.Context, tile tile) (*entries, tileSource, error) { 507 | beginS3Get := time.Now() 508 | contents, err := tch.getFromS3(ctx, tile) 509 | tch.backendLatencyMetric.WithLabelValues("s3_get").Observe(time.Since(beginS3Get).Seconds()) 510 | 511 | if err == nil { 512 | return contents, sourceS3, nil 513 | } 514 | 515 | if !errors.Is(err, noSuchKey{}) { 516 | tch.requestsMetric.WithLabelValues("error", "s3_get").Inc() 517 | return nil, sourceS3, fmt.Errorf("error reading tile from s3: %w", err) 518 | } 519 | 520 | beginCTLogGet := time.Now() 521 | contents, err = getTileFromBackend(ctx, tile) 522 | tch.backendLatencyMetric.WithLabelValues("ct_log_get").Observe(time.Since(beginCTLogGet).Seconds()) 523 | 524 | if err != nil { 525 | var statusCodeErr statusCodeError 526 | // Requests for tiles past the end of the log will get a 400 from CTFE, so report those 527 | // separately. 528 | if errors.As(err, &statusCodeErr) && statusCodeErr.statusCode == http.StatusBadRequest { 529 | tch.requestsMetric.WithLabelValues("bad_request", "ct_log_get").Inc() 530 | } else { 531 | tch.requestsMetric.WithLabelValues("error", "ct_log_get").Inc() 532 | } 533 | return nil, sourceCTLog, fmt.Errorf("error reading tile from backend: %w", err) 534 | } 535 | 536 | // If we got a partial tile, assume we are at the end of the log and the last 537 | // tile isn't filled up yet. In that case, don't write to S3, but still return 538 | // results to the user. 539 | if tch.isPartialTile(contents) { 540 | tch.partialTiles.Inc() 541 | return contents, sourceCTLog, nil 542 | } 543 | 544 | beginS3Put := time.Now() 545 | err = tch.writeToS3(ctx, tile, contents) 546 | tch.backendLatencyMetric.WithLabelValues("s3_put").Observe(time.Since(beginS3Put).Seconds()) 547 | 548 | if err != nil { 549 | tch.requestsMetric.WithLabelValues("error", "s3_put").Inc() 550 | return nil, sourceCTLog, fmt.Errorf("error writing tile to S3: %w", err) 551 | } 552 | 553 | return contents, sourceCTLog, nil 554 | } 555 | 556 | // isPartialTile returns true if there are fewer items in the tile than were 557 | // requested by the tileCachingHandler. 558 | func (tch *tileCachingHandler) isPartialTile(contents *entries) bool { 559 | return len(contents.Entries) < tch.tileSize 560 | } 561 | 562 | // singleflightDo is a wrapper around singleflight.Group.Do that, instead of 563 | // returning an interface{}, returns the exact type of the first return type of 564 | // the function fn. (singleflight was built before generics) 565 | func singleflightDo[V any](group *singleflight.Group, key string, fn func() (V, error)) (V, error, bool) { 566 | out, err, shared := group.Do(key, func() (interface{}, error) { 567 | return fn() 568 | }) 569 | return out.(V), err, shared 570 | } 571 | 572 | // passthroughHandler is an HTTP handler that passes through GET requests to the CT log. 573 | type passthroughHandler struct { 574 | logURL string 575 | } 576 | 577 | func (p passthroughHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 578 | if r.Method != "GET" { 579 | w.WriteHeader(http.StatusMethodNotAllowed) 580 | fmt.Fprintln(w, "only GET is supported") 581 | return 582 | } 583 | url := fmt.Sprintf("%s%s", p.logURL, r.URL.Path) 584 | req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, url, nil) 585 | if err != nil { 586 | w.WriteHeader(http.StatusInternalServerError) 587 | fmt.Fprintf(w, "creating request: %s\n", err) 588 | return 589 | } 590 | resp, err := http.DefaultClient.Do(req) 591 | if err != nil { 592 | w.WriteHeader(http.StatusInternalServerError) 593 | fmt.Fprintf(w, "fetching %s: %s\n", url, err) 594 | return 595 | } 596 | defer resp.Body.Close() 597 | 598 | w.WriteHeader(resp.StatusCode) 599 | _, err = io.Copy(w, resp.Body) 600 | if err != nil { 601 | log.Printf("copying response body to client: %s\n", err) 602 | } 603 | } 604 | 605 | func main() { 606 | logURL := flag.String("log-url", "", "CT log URL. e.g. https://oak.ct.letsencrypt.org/2023") 607 | tileSize := flag.Int("tile-size", 0, "tile size. Must match the value used by the backend") 608 | s3bucket := flag.String("s3-bucket", "", "s3 bucket to use for caching") 609 | s3prefix := flag.String("s3-prefix", "", "prefix for s3 keys. defaults to value of -backend") 610 | listenAddress := flag.String("listen-address", ":7962", "address to listen on") 611 | metricsAddress := flag.String("metrics-address", ":7963", "address to listen on for metrics") 612 | 613 | // fullRequestTimeout is the max allowed time the handler can read from S3 and return or read from S3, read from backend, write to S3, and return. 614 | fullRequestTimeout := flag.Duration("full-request-timeout", 4*time.Second, "max time to spend in the HTTP handler") 615 | 616 | flag.Parse() 617 | 618 | if *logURL == "" { 619 | log.Fatal("missing required flag: -log-url") 620 | } 621 | 622 | if *s3bucket == "" { 623 | log.Fatal("missing required flag: -s3-bucket") 624 | } 625 | 626 | if *tileSize == 0 { 627 | log.Fatal("missing required flag: -tile-size") 628 | } 629 | 630 | if *fullRequestTimeout == 0 { 631 | log.Fatal("-full-request-timeout may not have a timeout value of 0") 632 | } 633 | 634 | if *s3prefix == "" { 635 | *s3prefix = *logURL 636 | } 637 | 638 | cfg, err := config.LoadDefaultConfig(context.Background()) 639 | if err != nil { 640 | log.Fatal(err) 641 | } 642 | svc := s3.NewFromConfig(cfg) 643 | 644 | promRegistry := newStatsRegistry(*metricsAddress) 645 | 646 | handler, err := newTileCachingHandler(*logURL, *tileSize, svc, *s3prefix, *s3bucket, *fullRequestTimeout, promRegistry) 647 | if err != nil { 648 | log.Fatal(err) 649 | } 650 | 651 | srv := http.Server{ 652 | Addr: *listenAddress, 653 | ReadTimeout: 5 * time.Second, 654 | WriteTimeout: *fullRequestTimeout + 1*time.Second, // must be a bit larger than the max time spent in the HTTP handler 655 | IdleTimeout: 5 * time.Minute, 656 | ReadHeaderTimeout: 2 * time.Second, 657 | Handler: handler, 658 | } 659 | 660 | log.Fatal(srv.ListenAndServe()) 661 | } 662 | 663 | func newStatsRegistry(listenAddress string) prometheus.Registerer { 664 | registry := prometheus.NewRegistry() 665 | registry.MustRegister(collectors.NewGoCollector()) 666 | registry.MustRegister(collectors.NewProcessCollector( 667 | collectors.ProcessCollectorOpts{})) 668 | 669 | server := http.Server{ 670 | Addr: listenAddress, 671 | ReadTimeout: 5 * time.Second, 672 | WriteTimeout: 5 * time.Second, 673 | IdleTimeout: 5 * time.Minute, 674 | ReadHeaderTimeout: 2 * time.Second, 675 | Handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), 676 | } 677 | go func() { 678 | err := server.ListenAndServe() 679 | if err != nil { 680 | log.Printf("unable to start metrics server on %s: %s\n", listenAddress, err) 681 | os.Exit(1) 682 | } 683 | }() 684 | return registry 685 | } 686 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestTrimForDisplay(t *testing.T) { 9 | entries := &entries{ 10 | Entries: []entry{ 11 | {}, 12 | {}, 13 | {}, 14 | }, 15 | } 16 | _, err := entries.trimForDisplay(1, 2, tile{start: 10, end: 20, size: 10, logURL: "http://example.com"}) 17 | if err == nil { 18 | t.Fatal("expected error, got none") 19 | } 20 | if !strings.Contains(err.Error(), "internal inconsistency") { 21 | t.Errorf("expected internal inconsistency error, got %s", err) 22 | } 23 | 24 | _, err = entries.trimForDisplay(999, 1000, tile{start: 10, end: 20, size: 10, logURL: "http://example.com"}) 25 | if err == nil { 26 | t.Fatal("expected error, got none") 27 | } 28 | if !strings.Contains(err.Error(), "internal inconsistency") { 29 | t.Errorf("expected internal inconsistency error, got %s", err) 30 | } 31 | 32 | _, err = entries.trimForDisplay(1000, 1000, tile{start: 10, end: 20, size: 10, logURL: "http://example.com"}) 33 | if err == nil { 34 | t.Fatal("expected error, got none") 35 | } 36 | if !strings.Contains(err.Error(), "internal inconsistency") { 37 | t.Errorf("expected internal inconsistency error, got %s", err) 38 | } 39 | 40 | _, err = entries.trimForDisplay(10, 20, tile{start: 10, end: 12, size: 2, logURL: "http://example.com"}) 41 | if err == nil { 42 | t.Fatal("expected error, got none") 43 | } 44 | if !strings.Contains(err.Error(), "internal inconsistency") { 45 | t.Errorf("expected internal inconsistency error, got %s", err) 46 | } 47 | 48 | _, err = entries.trimForDisplay(15, 20, tile{start: 10, end: 20, size: 10, logURL: "http://example.com"}) 49 | if err == nil { 50 | t.Fatal("expected error, got none") 51 | } 52 | if !strings.Contains(err.Error(), "past the end of the log") { 53 | t.Errorf("expected 'past the end of the log' error, got %s", err) 54 | } 55 | 56 | e, err := entries.trimForDisplay(10, 20, tile{start: 10, end: 20, size: 10, logURL: "http://example.com"}) 57 | if err != nil { 58 | t.Fatalf("expected success, got %s", err) 59 | } 60 | if len(e.Entries) != 3 { 61 | t.Errorf("expected 3 entries got %d", len(entries.Entries)) 62 | } 63 | 64 | e, err = entries.trimForDisplay(11, 12, tile{start: 10, end: 20, size: 10, logURL: "http://example.com"}) 65 | if err != nil { 66 | t.Fatalf("expected success, got %s", err) 67 | } 68 | if len(e.Entries) != 1 { 69 | t.Errorf("expected 1 entry got %d", len(entries.Entries)) 70 | } 71 | 72 | e, err = entries.trimForDisplay(12, 20, tile{start: 10, end: 20, size: 10, logURL: "http://example.com"}) 73 | if err != nil { 74 | t.Fatalf("expected success, got %s", err) 75 | } 76 | if len(e.Entries) != 1 { 77 | t.Errorf("expected 1 entry got %d", len(entries.Entries)) 78 | } 79 | } 80 | --------------------------------------------------------------------------------