├── .air.toml ├── .env.example ├── .github └── workflows │ └── pr.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── cmd └── cli │ └── main.go ├── docker-compose.yaml ├── go.mod ├── go.sum ├── internal ├── .gitkeep ├── application │ ├── application.go │ └── options.go ├── consts │ ├── async.go │ └── git.go ├── git │ ├── .gitkeep │ ├── async.go │ ├── errors.go │ └── git.go ├── github │ ├── errors.go │ └── github.go ├── model │ └── .gitkeep ├── s3 │ ├── models.go │ └── util.go ├── server │ ├── bucket.go │ ├── catchall.go │ ├── handler.go │ ├── object.go │ ├── object_async.go │ └── router.go └── util │ ├── github.go │ ├── log.go │ └── name.go └── readme.md /.air.toml: -------------------------------------------------------------------------------- 1 | root = "." 2 | testdata_dir = "testdata" 3 | tmp_dir = "tmp" 4 | 5 | [build] 6 | args_bin = [] 7 | bin = "./tmp/main" 8 | cmd = "go build -o ./tmp/main ./cmd/cli/" 9 | delay = 1000 10 | exclude_dir = ["assets", "tmp", "vendor", "testdata"] 11 | exclude_file = [] 12 | exclude_regex = ["_test.go"] 13 | exclude_unchanged = false 14 | follow_symlink = false 15 | full_bin = "" 16 | include_dir = [] 17 | include_ext = ["go", "tpl", "tmpl", "html"] 18 | include_file = [] 19 | kill_delay = "0s" 20 | log = "build-errors.log" 21 | poll = false 22 | poll_interval = 0 23 | post_cmd = [] 24 | pre_cmd = [] 25 | rerun = false 26 | rerun_delay = 500 27 | send_interrupt = false 28 | stop_on_error = false 29 | 30 | [color] 31 | app = "" 32 | build = "yellow" 33 | main = "magenta" 34 | runner = "green" 35 | watcher = "cyan" 36 | 37 | [log] 38 | main_only = false 39 | silent = false 40 | time = false 41 | 42 | [misc] 43 | clean_on_exit = false 44 | 45 | [proxy] 46 | app_port = 0 47 | enabled = false 48 | proxy_port = 0 49 | 50 | [screen] 51 | clear_on_rebuild = false 52 | keep_scroll = true 53 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | GHS3_PORT= 2 | GHS3_ADDRESS= 3 | GITHUB_TOKEN= 4 | GITHUB_OWNER= 5 | 6 | GIT_USERNAME= 7 | GIT_EMAIL= 8 | -------------------------------------------------------------------------------- /.github/workflows/pr.yaml: -------------------------------------------------------------------------------- 1 | name: PR Linting 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | lint: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout code 11 | uses: actions/checkout@v4 12 | 13 | - name: Set up Go 14 | uses: actions/setup-go@v5 15 | with: 16 | go-version: '1.24' # Specify your Go version 17 | 18 | - name: Run golangci-lint 19 | uses: golangci/golangci-lint-action@v8 20 | with: 21 | args: --timeout=5m 22 | 23 | - name: Run go vet 24 | run: go vet ./... 25 | 26 | - name: Run go test 27 | run: go test ./... 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | tmp 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Stage 1: Build the application 2 | FROM golang:1.24-alpine AS builder 3 | 4 | WORKDIR /app 5 | 6 | COPY go.mod go.sum ./ 7 | RUN go mod download 8 | 9 | # Copy the rest of the application source code 10 | COPY . . 11 | 12 | RUN go build -o ./tmp/main ./cmd/cli/ 13 | 14 | FROM alpine:latest 15 | 16 | WORKDIR /app 17 | 18 | COPY --from=builder /app/tmp/main /app/main 19 | 20 | # The application reads configuration from environment variables. 21 | # Refer to your readme.md or .env.example for required variables like: 22 | # GITHUB_TOKEN, GITHUB_OWNER, GHS3_PORT, GHS3_ADDRESS 23 | # These should be provided when running the container, e.g., using 'docker run -e VAR=value'. 24 | 25 | # If your application listens on a specific port (e.g., GHS3_PORT), 26 | # you might want to EXPOSE it. For example, if GHS3_PORT defaults to 8080: 27 | # EXPOSE 8080 28 | 29 | ENTRYPOINT ["/app/main"] 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 ktunprasert 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cmd/cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github-as-s3/internal/application" 5 | "net/http" 6 | "os" 7 | 8 | "github.com/joho/godotenv" 9 | "github.com/rs/zerolog" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | func setup() { 14 | _ = godotenv.Load() 15 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 16 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 17 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 18 | 19 | } 20 | 21 | func main() { 22 | setup() 23 | 24 | app := application.NewApplicationWithOpts( 25 | application.WithDefaultGit(), 26 | application.WithDefaultGithub(), 27 | ) 28 | 29 | if err := app.Start(); err != http.ErrServerClosed { 30 | log.Fatal().Err(err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | ghs3: 5 | build: 6 | context: . 7 | dockerfile: Dockerfile 8 | # If you want the container to restart automatically, uncomment the line below 9 | restart: unless-stopped 10 | ports: 11 | - "8080:${GHS3_PORT:-8080}" 12 | # if you want to use it with .env 13 | env_file: 14 | - .env 15 | # if you want to set manually 16 | # environment: 17 | # # Required 18 | # GITHUB_TOKEN: ${GITHUB_TOKEN} # Your GitHub personal access token 19 | # GITHUB_OWNER: ${GITHUB_OWNER} # The GitHub username or organization 20 | # # Optional 21 | # GHS3_PORT: ${GHS3_PORT:-8080} # Port for the service inside the container 22 | # GHS3_ADDRESS: ${GHS3_ADDRESS:-0.0.0.0} # Address to bind to inside the container 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github-as-s3 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | dario.cat/mergo v1.0.2 // indirect 7 | github.com/Microsoft/go-winio v0.6.2 // indirect 8 | github.com/ProtonMail/go-crypto v1.2.0 // indirect 9 | github.com/cloudflare/circl v1.6.1 // indirect 10 | github.com/cyphar/filepath-securejoin v0.4.1 // indirect 11 | github.com/emirpasic/gods v1.18.1 // indirect 12 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect 13 | github.com/go-git/go-billy/v5 v5.6.2 // indirect 14 | github.com/go-git/go-git/v5 v5.16.0 // indirect 15 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect 16 | github.com/google/go-github/v72 v72.0.0 // indirect 17 | github.com/google/go-querystring v1.1.0 // indirect 18 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 19 | github.com/joho/godotenv v1.5.1 // indirect 20 | github.com/kevinburke/ssh_config v1.2.0 // indirect 21 | github.com/labstack/echo/v4 v4.13.3 // indirect 22 | github.com/labstack/gommon v0.4.2 // indirect 23 | github.com/mattn/go-colorable v0.1.14 // indirect 24 | github.com/mattn/go-isatty v0.0.20 // indirect 25 | github.com/pjbgf/sha1cd v0.3.2 // indirect 26 | github.com/rs/zerolog v1.34.0 // indirect 27 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect 28 | github.com/skeema/knownhosts v1.3.1 // indirect 29 | github.com/valyala/bytebufferpool v1.0.0 // indirect 30 | github.com/valyala/fasttemplate v1.2.2 // indirect 31 | github.com/xanzy/ssh-agent v0.3.3 // indirect 32 | golang.org/x/crypto v0.38.0 // indirect 33 | golang.org/x/net v0.40.0 // indirect 34 | golang.org/x/sys v0.33.0 // indirect 35 | golang.org/x/text v0.25.0 // indirect 36 | golang.org/x/time v0.8.0 // indirect 37 | gopkg.in/warnings.v0 v0.1.2 // indirect 38 | ) 39 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= 2 | dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= 3 | github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= 4 | github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= 5 | github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= 6 | github.com/ProtonMail/go-crypto v1.2.0 h1:+PhXXn4SPGd+qk76TlEePBfOfivE0zkWFenhGhFLzWs= 7 | github.com/ProtonMail/go-crypto v1.2.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= 8 | github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= 9 | github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= 10 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 11 | github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= 12 | github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= 13 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= 16 | github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= 17 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= 18 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= 19 | github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= 20 | github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= 21 | github.com/go-git/go-git/v5 v5.16.0 h1:k3kuOEpkc0DeY7xlL6NaaNg39xdgQbtH5mwCafHO9AQ= 22 | github.com/go-git/go-git/v5 v5.16.0/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= 23 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 24 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= 25 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= 26 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 27 | github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM= 28 | github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg= 29 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 30 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 31 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= 32 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= 33 | github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 34 | github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 35 | github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= 36 | github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= 37 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 38 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 39 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 40 | github.com/labstack/echo/v4 v4.13.3 h1:pwhpCPrTl5qry5HRdM5FwdXnhXSLSY+WE+YQSeCaafY= 41 | github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAfg+Xox9q5g= 42 | github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= 43 | github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= 44 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 45 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 46 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 47 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 48 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 49 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 50 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 51 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 52 | github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= 53 | github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= 54 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 55 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 56 | github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= 57 | github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= 58 | github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= 59 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= 60 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= 61 | github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 62 | github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= 63 | github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= 64 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 65 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 66 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 67 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 68 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 69 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 70 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 71 | github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= 72 | github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= 73 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 74 | golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= 75 | golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= 76 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 77 | golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= 78 | golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= 79 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 80 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 81 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 82 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 83 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 84 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 85 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 86 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 87 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 88 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 89 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 90 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 91 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 92 | golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= 93 | golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= 94 | golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= 95 | golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 96 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 97 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 98 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 99 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 100 | gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= 101 | gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= 102 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 103 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 104 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 105 | -------------------------------------------------------------------------------- /internal/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ktunprasert/github-as-s3/4d9d2318a9c708c2ce6f97e47b4827686947dc12/internal/.gitkeep -------------------------------------------------------------------------------- /internal/application/application.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "fmt" 5 | "github-as-s3/internal/git" 6 | "github-as-s3/internal/github" 7 | "github-as-s3/internal/server" 8 | "os" 9 | "time" 10 | 11 | "github.com/labstack/echo/v4" 12 | "github.com/labstack/echo/v4/middleware" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | type Application struct { 17 | Token string 18 | Echo *echo.Echo 19 | Port string 20 | Address string 21 | Owner string 22 | GitUsername string 23 | GitEmail string 24 | 25 | GH *github.GitHub 26 | Git *git.Git 27 | } 28 | 29 | func newApplication() *Application { 30 | return &Application{ 31 | Port: getEnv("GHS3_PORT", "8080"), 32 | Address: getEnv("GHS3_ADDRESS", "0.0.0.0"), 33 | Token: getEnv("GITHUB_TOKEN", ""), 34 | Owner: getEnv("GITHUB_OWNER", ""), 35 | GitUsername: getEnv("GIT_USERNAME", "GHS3"), 36 | GitEmail: getEnv("GIT_EMAIL", "bot@ghs3.com"), 37 | } 38 | } 39 | 40 | func NewApplicationWithOpts(opts ...applicationOpts) *Application { 41 | app := newApplication() 42 | for _, opt := range opts { 43 | opt(app) 44 | } 45 | 46 | return app 47 | } 48 | 49 | func (app *Application) Start() error { 50 | if err := app.Setup(); err != nil { 51 | return err 52 | } 53 | 54 | // should be a goroutine? 55 | 56 | return app.Echo.Start(fmt.Sprintf("%s:%s", app.Address, app.Port)) 57 | } 58 | 59 | func (app *Application) Setup() error { 60 | // setup routes 61 | e := echo.New() 62 | // e.Use(middleware.Logger()) 63 | e.Use(zerologger()) 64 | e.Use(middleware.Recover()) 65 | e.Use(middleware.AddTrailingSlash()) 66 | e.Use(middleware.RequestID()) 67 | e.Use(middleware.BodyLimit("5M")) 68 | 69 | app.Echo = e 70 | server.RegisterRoutes(e, server.NewS3Handler(app.GH, app.Git, true)) 71 | 72 | return nil 73 | } 74 | 75 | func zerologger() echo.MiddlewareFunc { 76 | return func(next echo.HandlerFunc) echo.HandlerFunc { 77 | return func(c echo.Context) error { 78 | req := c.Request() 79 | res := c.Response() 80 | start := time.Now() 81 | 82 | loggerBuilder := log.Logger.With() 83 | clientRequestID := req.Header.Get(echo.HeaderXRequestID) 84 | if clientRequestID != "" { 85 | loggerBuilder = loggerBuilder.Str("request_id", clientRequestID) 86 | } 87 | requestLogger := loggerBuilder.Logger() 88 | 89 | newCtx := requestLogger.WithContext(req.Context()) 90 | c.SetRequest(req.WithContext(newCtx)) 91 | 92 | log.Info().Str("method", req.Method). 93 | Str("url", req.URL.String()). 94 | Int("status", res.Status). 95 | Str("remote_ip", req.RemoteAddr). 96 | Str("user_agent", req.UserAgent()). 97 | Msg("request received") 98 | 99 | err := next(c) 100 | latency := time.Since(start) 101 | 102 | logEvent := requestLogger.Info() 103 | if err != nil { 104 | logEvent = requestLogger.Error().Err(err) 105 | } 106 | 107 | logEvent.Str("method", req.Method). 108 | Dur("latency", latency). 109 | Str("request_id", res.Header().Get(echo.HeaderXRequestID)). 110 | Msg("request completed") 111 | 112 | return err 113 | } 114 | } 115 | } 116 | 117 | func getEnv(key, fallback string) string { 118 | if value, ok := os.LookupEnv(key); ok { 119 | return value 120 | } 121 | 122 | return fallback 123 | } 124 | -------------------------------------------------------------------------------- /internal/application/options.go: -------------------------------------------------------------------------------- 1 | package application 2 | 3 | import ( 4 | "github-as-s3/internal/git" 5 | "github-as-s3/internal/github" 6 | "os" 7 | ) 8 | 9 | type applicationOpts = func(*Application) 10 | 11 | func WithToken(token string) applicationOpts { 12 | return func(a *Application) { 13 | a.Token = token 14 | } 15 | } 16 | 17 | func WithEnvToken() applicationOpts { 18 | token := os.Getenv("GITHUB_TOKEN") 19 | 20 | return func(a *Application) { 21 | a.Token = token 22 | } 23 | } 24 | 25 | func WithPort(port string) applicationOpts { 26 | return func(a *Application) { 27 | a.Port = port 28 | } 29 | } 30 | 31 | func WithAddress(address string) applicationOpts { 32 | return func(a *Application) { 33 | a.Address = address 34 | } 35 | } 36 | 37 | func WithDefaultGithub() applicationOpts { 38 | return func(a *Application) { 39 | a.GH = github.NewGitHub(a.Token, a.Owner) 40 | } 41 | } 42 | 43 | func WithDefaultGit() applicationOpts { 44 | return func(a *Application) { 45 | a.Git = git.NewGit(a.Token, a.Owner, a.GitUsername, a.GitEmail) 46 | } 47 | } 48 | 49 | func WithGitHub(gh *github.GitHub) applicationOpts { 50 | return func(a *Application) { 51 | a.GH = gh 52 | } 53 | } 54 | 55 | func WithGit(g *git.Git) applicationOpts { 56 | return func(a *Application) { 57 | a.Git = g 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /internal/consts/async.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | const ( 4 | Put = "PUT" 5 | Delete = "DELETE" 6 | ) 7 | -------------------------------------------------------------------------------- /internal/consts/git.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | const ( 4 | Master = "master" 5 | Origin = "origin" 6 | ) 7 | -------------------------------------------------------------------------------- /internal/git/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ktunprasert/github-as-s3/4d9d2318a9c708c2ce6f97e47b4827686947dc12/internal/git/.gitkeep -------------------------------------------------------------------------------- /internal/git/async.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github-as-s3/internal/consts" 7 | "github-as-s3/internal/util" 8 | "io" 9 | "os" 10 | "sync" 11 | "time" 12 | 13 | "github.com/go-git/go-git/v5" 14 | "github.com/go-git/go-git/v5/plumbing" 15 | "github.com/go-git/go-git/v5/plumbing/object" 16 | "github.com/go-git/go-git/v5/plumbing/transport" 17 | "github.com/rs/zerolog/log" 18 | ) 19 | 20 | type ChangeRequest struct { 21 | ctx context.Context 22 | ObjectKey string // e.g., "path/to/my-file.txt" 23 | Type string // e.g., "PUT", "DELETE" 24 | Content []byte // Content for PUT operations, or nil for DELETE 25 | CommitMessage string // Pre-formatted commit message for this change 26 | // You might also need a way to pass any specific S3 headers if your Git mapping requires them. 27 | done chan error 28 | tries int 29 | } 30 | 31 | type RepoWorker struct { 32 | sync.RWMutex 33 | // ctx context.Context 34 | changeQueue chan *ChangeRequest 35 | path string 36 | repo *git.Repository 37 | ga *GitAsync 38 | } 39 | 40 | func (w *RepoWorker) Start(bucket string) { 41 | debounceTimer := time.NewTimer(0) 42 | <-debounceTimer.C // Consume the initial expiration 43 | 44 | logger := log.With().Str("component", "RepoWorker").Str("path", w.path).Str("bucket", bucket).Logger() 45 | 46 | logger.Debug().Msg("RepoWorker Start") 47 | defer logger.Info().Msg("RepoWorker stopped") 48 | 49 | tries := 0 50 | 51 | // defer func() { 52 | // logger.Info().Msg("Removing from workers map") 53 | // repoWorkers.Lock() 54 | // defer repoWorkers.Unlock() 55 | // delete(repoWorkers.channels, bucket) 56 | // }() 57 | 58 | wt, err := w.repo.Worktree() 59 | if err != nil { 60 | logger.Error().Err(err).Msg("Failed to get worktree") 61 | return 62 | } 63 | 64 | remote, err := w.repo.Remote(consts.Origin) 65 | if err != nil { 66 | logger.Error().Err(err).Msg("Failed to get remote") 67 | return 68 | } 69 | 70 | for { 71 | select { 72 | case change := <-w.changeQueue: 73 | if err := change.ctx.Err(); err != nil { 74 | logger.Error().Err(err).Msg("skipping work") 75 | continue 76 | } 77 | 78 | if change.tries > 3 { 79 | logger.Error().Msg("Too many attempts, bumping from queue") 80 | change.done <- ErrTooManyAttempts 81 | close(change.done) 82 | continue 83 | } 84 | 85 | slog := logger.With().Str("change", change.Type).Str("key", change.ObjectKey).Logger() 86 | switch change.Type { 87 | case consts.Put: 88 | dst, err := wt.Filesystem.Create(change.ObjectKey) 89 | if err != nil { 90 | slog.Error().Err(err).Msg("Failed to create file") 91 | // try again 92 | change.tries++ 93 | w.changeQueue <- change 94 | continue 95 | } 96 | 97 | _, err = dst.Write(change.Content) 98 | if err != nil { 99 | slog.Error().Err(err).Msg("Failed to write file") 100 | // try again 101 | change.tries++ 102 | w.changeQueue <- change 103 | _ = dst.Close() 104 | continue 105 | } 106 | _ = dst.Close() 107 | 108 | _, err = wt.Add(change.ObjectKey) 109 | if err != nil { 110 | slog.Error().Err(err).Msg("Failed to add file to git") 111 | // try again 112 | change.tries++ 113 | w.changeQueue <- change 114 | _ = dst.Close() 115 | continue 116 | } 117 | 118 | case consts.Delete: 119 | err := wt.Filesystem.Remove(change.ObjectKey) 120 | if err != nil { 121 | var pathErr *os.PathError 122 | if errors.As(err, &pathErr) { 123 | slog.Error().AnErr("path_err", pathErr).Msg("Skipping delete, file does not exist") 124 | change.done <- ErrFileNotExists 125 | continue 126 | } 127 | slog.Error().Err(err).AnErr("err_serialised", err).Msg("Failed to delete file") 128 | // try again 129 | change.tries++ 130 | w.changeQueue <- change 131 | continue 132 | } 133 | 134 | _, err = wt.Remove(change.ObjectKey) 135 | if err != nil { 136 | slog.Error().Err(err).Msg("Failed to remove file from git") 137 | // try again 138 | change.tries++ 139 | w.changeQueue <- change 140 | continue 141 | } 142 | } 143 | 144 | hash, err := wt.Commit("[GHS3] "+change.CommitMessage, &git.CommitOptions{Author: w.ga.signature()}) 145 | if err != nil && !errors.Is(err, git.ErrEmptyCommit) { 146 | slog.Error().Err(err).Msg("Failed to commit file") 147 | // try again 148 | change.tries++ 149 | w.changeQueue <- change 150 | continue 151 | } 152 | 153 | slog.Info().Any("hash", hash.String()).Msg("change completed") 154 | 155 | if err == nil { 156 | debounceTimer.Reset(500 * time.Millisecond) // Adjust debounce interval as needed 157 | } 158 | 159 | change.done <- nil 160 | 161 | case <-debounceTimer.C: 162 | if tries > 3 { 163 | logger.Error().Msg("Too many tries to push, stopping worker") 164 | return 165 | } 166 | 167 | if w.ga.skipPush { 168 | logger.Debug().Msg("Skipping push to remote") 169 | debounceTimer.Stop() 170 | 171 | continue 172 | } 173 | 174 | err = remote.Push(&git.PushOptions{ 175 | RemoteName: consts.Origin, 176 | Auth: w.ga.auth(), 177 | }) 178 | 179 | if err != nil { 180 | if errors.Is(err, git.NoErrAlreadyUpToDate) { 181 | logger.Debug().Msg("No changes to push") 182 | debounceTimer.Stop() 183 | continue 184 | } 185 | 186 | logger.Error().Err(err).Msg("failed to push change, trying again in 500ms") 187 | tries++ 188 | debounceTimer.Reset(500 * time.Millisecond) 189 | continue 190 | } 191 | 192 | tries = 0 193 | 194 | debounceTimer.Stop() 195 | 196 | // prevents a race if a new request comes in right after processing 197 | debounceTimer.Reset(0) // Reset to immediate expiration for the next cycle 198 | <-debounceTimer.C // Consume it 199 | } 200 | } 201 | } 202 | 203 | func (rw *RepoWorker) Stop() {} 204 | 205 | var repoWorkers = struct { 206 | sync.RWMutex 207 | channels map[string]*RepoWorker 208 | }{channels: make(map[string]*RepoWorker)} 209 | 210 | type GitAsync struct { 211 | *Git 212 | } 213 | 214 | func NewGitAsync(git *Git) *GitAsync { 215 | return &GitAsync{git} 216 | } 217 | 218 | func (ga GitAsync) Head(ctx context.Context, name, filepath, version string) (*object.File, *object.Commit, error) { 219 | logger := log.Ctx(ctx).With().Str("component", "gitasync.Head").Str("repo", name).Str("path", filepath).Logger() 220 | logger.Debug().Msg("gitasync.Head Start") 221 | defer logger.Debug().Msg("gitasync.Head End") 222 | var err error 223 | 224 | w, err := ga.ensureWorker(ctx, name) 225 | if err != nil { 226 | logger.Error().Err(err).Msg("Failed to ensure worker") 227 | return nil, nil, err 228 | } 229 | 230 | var cmt *object.Commit 231 | var file *object.File 232 | 233 | // Attempt to get the file, retry once if not found initially 234 | for i := 0; i < 2; i++ { // Try up to 2 times 235 | if version != "" { 236 | commitHash := plumbing.NewHash(version) 237 | cmt, err = w.repo.CommitObject(commitHash) 238 | } else { 239 | headRef, errRef := w.repo.Head() 240 | if errRef != nil { 241 | logger.Error().Err(errRef).Msg("Failed to get HEAD reference") 242 | return nil, nil, errRef 243 | } 244 | cmt, err = w.repo.CommitObject(headRef.Hash()) 245 | if err != nil { 246 | logger.Error().Err(err).Str("head_hash", headRef.Hash().String()).Msg("Failed to get commit object for HEAD") 247 | return nil, nil, err 248 | } 249 | logger.Debug().Str("commit_hash_for_head", cmt.Hash.String()).Msg("Commit (latest HEAD) being checked by Head") 250 | } 251 | 252 | if err != nil { // Error getting the commit itself 253 | return nil, nil, err 254 | } 255 | 256 | file, err = cmt.File(filepath) 257 | if err == nil { 258 | break // File found, exit loop 259 | } 260 | 261 | if errors.Is(err, object.ErrFileNotFound) { 262 | if i == 0 { // If first attempt and file not found 263 | logger.Warn().Str("filepath", filepath).Str("commit_hash_checked", cmt.Hash.String()).Msg("File not found on first attempt, retrying shortly...") 264 | time.Sleep(500 * time.Millisecond) // Small delay 265 | // Potentially force a re-read of refs, though go-git might not have an explicit API for this. 266 | // The delay itself is often enough for filesystem changes to propagate. 267 | continue 268 | } 269 | // If still not found on second attempt, log tree and return error 270 | tree, treeErr := cmt.Tree() 271 | if treeErr == nil { 272 | logger.Warn().Str("filepath_searched", filepath).Msg("File not found in commit after retry. Listing tree entries:") 273 | _ = tree.Files().ForEach(func(f *object.File) error { 274 | logger.Warn().Str("file_in_tree", f.Name).Msg("File present in commit tree") 275 | return nil 276 | }) 277 | } 278 | } 279 | // For other errors, return immediately 280 | return nil, nil, err 281 | } 282 | 283 | if err != nil { // If loop finished due to error (e.g., file still not found) 284 | logger.Error().Err(err).Str("filepath", filepath).Str("commit_hash_checked", cmt.Hash.String()).Msg("Failed to get file from commit after retries") 285 | return nil, nil, err 286 | } 287 | 288 | return file, cmt, nil 289 | } 290 | 291 | func (ga GitAsync) Clone(ctx context.Context, bucket string) (*git.Repository, error) { 292 | logger := log.Ctx(ctx).With().Str("component", "gitasync.Clone").Str("bucket", bucket).Logger() 293 | logger.Debug().Msg("gitasync.Clone Start") 294 | 295 | worker, err := ga.ensureWorker(ctx, bucket) 296 | if err != nil { 297 | return nil, err 298 | } 299 | 300 | logger.Debug().Msg("gitasync.Clone End") 301 | return worker.repo, nil 302 | } 303 | 304 | func (ga GitAsync) PutRaw(ctx context.Context, repo *git.Repository, bucket, key string, dst io.ReadCloser) error { 305 | logger := log.Ctx(ctx).With().Str("component", "gitasync.PutRaw").Str("bucket", bucket).Str("key", key).Logger() 306 | logger.Debug().Msg("gitasync.PutRaw Start") 307 | defer logger.Debug().Msg("gitasync.PutRaw End") 308 | 309 | if repo == nil { 310 | logger.Error().Msg("repo is nil") 311 | return errors.New("repo is nil") 312 | } 313 | 314 | worker, err := ga.ensureWorker(ctx, bucket) 315 | if err != nil { 316 | return err 317 | } 318 | 319 | fileContent, err := io.ReadAll(dst) 320 | if err != nil { 321 | logger.Error().Err(err).Msg("Failed to read file content") 322 | return err 323 | } 324 | 325 | change := ChangeRequest{ 326 | ctx: ctx, 327 | ObjectKey: key, 328 | Content: fileContent, 329 | Type: consts.Put, 330 | CommitMessage: "Put file " + key, 331 | done: make(chan error), 332 | } 333 | 334 | worker.changeQueue <- &change 335 | if err := <-change.done; err != nil { 336 | return err 337 | } 338 | 339 | return nil 340 | } 341 | 342 | func (ga GitAsync) Delete(ctx context.Context, repo *git.Repository, bucket, relativeFilepath string) error { 343 | logger := log.Ctx(ctx).With().Str("component", "gitasync.Delete").Str("relativeFilepath", relativeFilepath).Logger() 344 | logger.Debug().Msg("gitasync.Delete Start") 345 | 346 | if repo == nil { 347 | logger.Error().Msg("repo is nil") 348 | return errors.New("repo is nil") 349 | } 350 | 351 | worker, err := ga.ensureWorker(ctx, bucket) 352 | if err != nil { 353 | logger.Error().Err(err).Msg("Failed to ensure worker") 354 | return err 355 | } 356 | 357 | change := ChangeRequest{ 358 | ctx: ctx, 359 | ObjectKey: relativeFilepath, 360 | Type: consts.Delete, 361 | CommitMessage: "Delete file " + relativeFilepath, 362 | done: make(chan error), 363 | } 364 | 365 | worker.changeQueue <- &change 366 | if err := <-change.done; err != nil { 367 | return err 368 | } 369 | 370 | logger.Debug().Msg("gitasync.Delete End") 371 | 372 | return nil 373 | } 374 | 375 | func (ga GitAsync) ensureWorker(ctx context.Context, bucket string) (*RepoWorker, error) { 376 | repoWorkers.RLock() 377 | w, exists := repoWorkers.channels[bucket] 378 | repoWorkers.RUnlock() 379 | 380 | if exists { 381 | // Optional: Check if w.isRunning and restart if necessary, 382 | // though the current Start logic seems to run indefinitely until an error. 383 | return w, nil 384 | } 385 | 386 | // Worker doesn't exist, acquire full lock to create it 387 | repoWorkers.Lock() 388 | defer repoWorkers.Unlock() // Ensure lock is released 389 | 390 | // Re-check: Another goroutine might have created it while we were waiting for the lock 391 | if w, exists = repoWorkers.channels[bucket]; exists { 392 | return w, nil 393 | } 394 | 395 | logger := log.Ctx(ctx).With().Str("component", "gitasync.EnsureWorker").Str("bucket", bucket).Logger() 396 | logger.Debug().Msg("gitasync.EnsureWorker Start - Creating new worker") 397 | 398 | path, err := os.MkdirTemp("", "ghs3-"+bucket+"-") // Added a trailing dash for clarity 399 | if err != nil { 400 | logger.Error().Err(err).Msg("Failed to create temp directory") 401 | return nil, err 402 | } 403 | 404 | logger.Debug().Str("path", path).Msg("Temp directory created for Clone") 405 | 406 | repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{ 407 | URL: util.GithubURL(ga.owner, bucket), 408 | Auth: ga.auth(), 409 | RemoteName: consts.Origin, 410 | ReferenceName: consts.Master, 411 | SingleBranch: true, 412 | Progress: nil, 413 | Depth: 1, 414 | }) 415 | 416 | if err != nil { 417 | // If clone fails, attempt to remove the temp directory 418 | _ = os.RemoveAll(path) 419 | if errors.Is(err, transport.ErrEmptyRemoteRepository) { 420 | logger.Debug().Msg("Remote repository is empty, calling InitRepo") 421 | initializedRepo, initErr := ga.InitRepo(ctx, bucket, path) // This might need to use the 'path' 422 | if initErr != nil { 423 | logger.Error().Err(initErr).Msg("Failed to init repo after empty remote error") 424 | return nil, initErr 425 | } 426 | repo = initializedRepo 427 | } else if errors.Is(err, git.ErrRepositoryAlreadyExists) { 428 | logger.Warn().Err(err).Msg("Repository already exists, attempting to open") 429 | repo, err = git.PlainOpen(path) 430 | if err != nil { 431 | logger.Error().Err(err).Msg("Failed to open existing repository") 432 | return nil, err 433 | } 434 | } else { 435 | logger.Error().Err(err).Msg("Failed to clone repository") 436 | return nil, err 437 | } 438 | } 439 | 440 | w = &RepoWorker{ 441 | path: path, 442 | changeQueue: make(chan *ChangeRequest, 100), // Buffered channel 443 | repo: repo, 444 | ga: &ga, 445 | // isRunning: true, // Set isRunning when Start is actually running 446 | } 447 | repoWorkers.channels[bucket] = w 448 | 449 | go w.Start(bucket) // Start the worker goroutine 450 | 451 | logger.Info().Msg("New RepoWorker created and started") 452 | return w, nil 453 | } 454 | -------------------------------------------------------------------------------- /internal/git/errors.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrRepoNil = errors.New("repository is nil") 7 | ErrPathEmpty = errors.New("path is empty") 8 | ErrPathNotExists = errors.New("path does not exist") 9 | ErrFileNil = errors.New("file is nil") 10 | ErrFileNotExists = errors.New("file does not exist") 11 | 12 | ErrTooManyAttempts = errors.New("too many attempts") 13 | ) 14 | -------------------------------------------------------------------------------- /internal/git/git.go: -------------------------------------------------------------------------------- 1 | package git 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github-as-s3/internal/consts" 7 | "github-as-s3/internal/util" 8 | "io" 9 | "mime/multipart" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "time" 14 | 15 | "github.com/go-git/go-git/v5" 16 | "github.com/go-git/go-git/v5/config" 17 | "github.com/go-git/go-git/v5/plumbing/object" 18 | "github.com/go-git/go-git/v5/plumbing/transport" 19 | "github.com/go-git/go-git/v5/plumbing/transport/http" 20 | "github.com/rs/zerolog/log" 21 | ) 22 | 23 | type Git struct { 24 | token string 25 | owner string 26 | skipPush bool 27 | username string 28 | email string 29 | } 30 | 31 | func NewGit(token, owner, username, email string) *Git { 32 | return &Git{ 33 | token: token, 34 | owner: owner, 35 | username: username, 36 | email: email, 37 | } 38 | } 39 | 40 | func (g *Git) SetSkipPush(skipPush bool) { 41 | g.skipPush = skipPush 42 | } 43 | 44 | // Used when we create a new repo via GitHub but 45 | // it's empty 46 | func (g *Git) InitRepo(ctx context.Context, name, path string) (*git.Repository, error) { 47 | slog := util.LogCtx(ctx, "git.InitRepo").With().Str("component", "git.InitRepo").Logger() 48 | 49 | var err error 50 | slog.Debug().Str("repo_name", name).Msg("git.InitRepo.Start") 51 | defer slog.Error().Str("name", name).Str("path", path).AnErr("init_repo_error", err).Msg("git.InitRepo.Error") 52 | 53 | if path == "" { 54 | path, err = os.MkdirTemp("", "ghs3-"+name) 55 | if err != nil { 56 | return nil, err 57 | } 58 | slog.Debug().Str("path", path).Msg("Temp directory created for Clone") 59 | } 60 | 61 | repo, err := git.PlainInit(path, false) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | remote, err := repo.CreateRemote(&config.RemoteConfig{ 67 | Name: consts.Origin, 68 | URLs: []string{util.GithubURL(g.owner, name)}, 69 | }) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | wt, err := repo.Worktree() 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | f, err := wt.Filesystem.Create(".ghs3") 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | _, err = wt.Add(f.Name()) 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | _ = f.Close() 90 | 91 | _, err = wt.Commit("batman", &git.CommitOptions{ 92 | Author: g.signature(), 93 | }) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | err = g.push(ctx, remote, name) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | slog.Debug().Str("repo_name", name).Msg("git.InitRepo.OK") 104 | return repo, nil 105 | } 106 | 107 | func (g *Git) Clone(ctx context.Context, name string) (*git.Repository, error) { 108 | slog := util.LogCtx(ctx, "git.Clone").With().Str("component", "git.Clone").Logger() 109 | slog.Debug().Str("repo_name", name).Msg("git.Clone.Start") 110 | 111 | path, err := os.MkdirTemp("", "ghs3-"+name) 112 | if err != nil { 113 | return nil, err 114 | } 115 | slog.Debug().Str("path", path).Msg("Temp directory created for Clone") 116 | 117 | repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{ 118 | URL: util.GithubURL(g.owner, name), 119 | Auth: g.auth(), 120 | ReferenceName: consts.Master, 121 | SingleBranch: true, 122 | }) 123 | 124 | if err != nil { 125 | if errors.Is(err, transport.ErrEmptyRemoteRepository) { 126 | slog.Debug().Str("repo_name", name).Msg("Remote repository is empty, calling InitRepo") 127 | repo, err = g.InitRepo(ctx, name, path) 128 | if err != nil { 129 | return nil, err 130 | } 131 | 132 | } else { 133 | return nil, err 134 | } 135 | 136 | } 137 | 138 | slog.Debug().Str("repo_name", name).Msg("git.Clone.OK") 139 | return repo, nil 140 | } 141 | 142 | func (g *Git) PutRaw(ctx context.Context, repo *git.Repository, key string, src io.ReadCloser) error { 143 | slog := util.LogCtx(ctx, "git.PutRaw").With().Str("component", "git.PutRaw").Logger() 144 | slog.Debug().Str("filename", key).Msg("git.PutRaw.Start") 145 | 146 | if repo == nil { 147 | slog.Error().Msg("repo is nil") 148 | return errors.New("repo is nil") 149 | } 150 | 151 | wt, err := repo.Worktree() 152 | if err != nil { 153 | slog.Error().Err(err).Msg("failed to get worktree") 154 | return err 155 | } 156 | 157 | dst, err := wt.Filesystem.Create(key) 158 | if err != nil { 159 | slog.Error().Err(err).Str("filename", key).Msg("failed to create destination file") 160 | return err 161 | } 162 | 163 | if _, err := io.Copy(dst, src); err != nil { 164 | slog.Error().Err(err).Str("filename", key).Msg("failed to copy file contents") 165 | return err 166 | } 167 | slog.Debug().Str("filename", key).Msg("file copied successfully") 168 | 169 | _ = src.Close() 170 | _ = dst.Close() 171 | 172 | _, err = wt.Add(key) 173 | if err != nil { 174 | slog.Error().Err(err).Str("filename", key).Msg("failed to add file to git") 175 | return err 176 | } 177 | 178 | _, err = wt.Commit("[GHS3] add file "+key, &git.CommitOptions{Author: g.signature()}) 179 | if err != nil { 180 | if errors.Is(err, git.ErrEmptyCommit) { 181 | slog.Debug().Msg("empty commit, skipping") 182 | return nil 183 | } 184 | 185 | slog.Error().Err(err).Str("filename", key).Msg("failed to commit file") 186 | return err 187 | } 188 | 189 | slog.Debug().Str("filename", key).Msg("file committed") 190 | 191 | remote, err := repo.Remote("origin") 192 | if err != nil { 193 | slog.Error().Err(err).Msg("failed to get remote 'origin'") 194 | return err 195 | } 196 | 197 | err = g.push(ctx, remote, "") 198 | if err != nil { 199 | return err 200 | } 201 | 202 | slog.Debug().Str("filename", key).Msg("git.Put.OK") 203 | 204 | return nil 205 | } 206 | 207 | func (g *Git) Put(ctx context.Context, repo *git.Repository, file *multipart.FileHeader) error { 208 | slog := util.LogCtx(ctx, "git.Put").With().Str("component", "git.Put").Logger() 209 | slog.Debug().Str("filename", func() string { 210 | if file != nil { 211 | return file.Filename 212 | } 213 | return "" 214 | }()).Msg("git.Put.Start") 215 | 216 | if repo == nil { 217 | slog.Error().Msg("repo is nil") 218 | return errors.New("repo is nil") 219 | } 220 | 221 | if file == nil { 222 | slog.Error().Msg("file is nil") 223 | return errors.New("file is nil") 224 | } 225 | 226 | src, err := file.Open() 227 | if err != nil { 228 | slog.Error().Err(err).Msg("failed to open file") 229 | return err 230 | } 231 | slog.Debug().Str("filename", file.Filename).Msg("file opened successfully") 232 | 233 | wt, err := repo.Worktree() 234 | if err != nil { 235 | slog.Error().Err(err).Msg("failed to get worktree") 236 | return err 237 | } 238 | 239 | path := wt.Filesystem.Root() 240 | if path == "" { 241 | slog.Error().Msg("worktree path is empty") 242 | return errors.New("path is empty") 243 | } 244 | slog.Debug().Str("path", path).Msg("worktree path resolved") 245 | 246 | if _, err := os.Stat(path); err != nil { 247 | if os.IsNotExist(err) { 248 | slog.Error().Str("path", path).Msg("worktree path does not exist") 249 | return errors.New("path does not exist") 250 | } 251 | slog.Error().Err(err).Msg("failed to stat worktree path") 252 | return err 253 | } 254 | 255 | dst, err := wt.Filesystem.Create(file.Filename) 256 | if err != nil { 257 | slog.Error().Err(err).Str("filename", file.Filename).Msg("failed to create destination file") 258 | return err 259 | } 260 | 261 | if _, err := io.Copy(dst, src); err != nil { 262 | slog.Error().Err(err).Str("filename", file.Filename).Msg("failed to copy file contents") 263 | return err 264 | } 265 | slog.Debug().Str("filename", file.Filename).Msg("file copied successfully") 266 | 267 | _ = src.Close() 268 | _ = dst.Close() 269 | 270 | _, err = wt.Add(file.Filename) 271 | if err != nil { 272 | slog.Error().Err(err).Str("filename", file.Filename).Msg("failed to add file to git") 273 | return err 274 | } 275 | slog.Debug().Str("filename", file.Filename).Msg("file added to git index") 276 | 277 | _, err = wt.Commit("[GHS3] add file "+file.Filename, &git.CommitOptions{ 278 | Author: g.signature(), 279 | }) 280 | if err != nil { 281 | // if user uploads the same file with the same content 282 | // we should let the user do it 283 | if errors.Is(err, git.ErrEmptyCommit) { 284 | slog.Debug().Msg("empty commit, skipping") 285 | return nil 286 | } 287 | 288 | slog.Error().Err(err).Str("filename", file.Filename).Msg("failed to commit file") 289 | return err 290 | } 291 | slog.Debug().Str("filename", file.Filename).Msg("file committed") 292 | 293 | remote, err := repo.Remote("origin") 294 | if err != nil { 295 | slog.Error().Err(err).Msg("failed to get remote 'origin'") 296 | return err 297 | } 298 | 299 | err = g.push(ctx, remote, "") 300 | if err != nil { 301 | return err 302 | } 303 | 304 | slog.Debug().Str("filename", file.Filename).Msg("git.Put.OK") 305 | return nil 306 | } 307 | 308 | func (g *Git) Get(ctx context.Context, repo *git.Repository, relativeFilepath string) ([]byte, os.FileInfo, error) { 309 | if repo == nil { 310 | return nil, nil, errors.New("repo is nil") 311 | } 312 | 313 | logger := log.Ctx(ctx).With().Str("component", "git.Get").Str("filename", relativeFilepath).Logger() 314 | 315 | wt, err := repo.Worktree() 316 | if err != nil { 317 | return nil, nil, err 318 | } 319 | 320 | f, err := wt.Filesystem.Open(relativeFilepath) 321 | if err != nil { 322 | if os.IsNotExist(err) { 323 | logger.Error().Err(err).Msg("file does not exist") 324 | return nil, nil, ErrFileNotExists 325 | } 326 | 327 | logger.Error().Err(err).Msg("failed to open file") 328 | return nil, nil, err 329 | } 330 | defer func() { 331 | _ = f.Close() 332 | }() 333 | 334 | fileInfo, error := wt.Filesystem.Stat(relativeFilepath) 335 | if error != nil { 336 | logger.Error().Err(error).Msg("failed to get file info") 337 | return nil, nil, error 338 | } 339 | 340 | bytes, err := io.ReadAll(f) 341 | return bytes, fileInfo, err 342 | } 343 | 344 | func (g *Git) List(ctx context.Context, repo *git.Repository) (map[string]os.FileInfo, error) { 345 | slog := util.LogCtx(ctx, "git.List").With().Str("component", "git.List").Logger() 346 | slog.Debug().Msg("git.List.Start") 347 | 348 | wt, err := repo.Worktree() 349 | if err != nil { 350 | slog.Error().Err(err).Msg("failed to get worktree") 351 | return nil, err 352 | } 353 | 354 | root := wt.Filesystem.Root() 355 | files := make(map[string]os.FileInfo) 356 | err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 357 | if err != nil { 358 | slog.Error().Err(err).Str("path", path).Msg("error walking path") 359 | return err 360 | } 361 | 362 | rel, err := filepath.Rel(root, path) 363 | if err != nil { 364 | slog.Error().Err(err).Str("path", path).Msg("error getting relative path") 365 | return err 366 | } 367 | 368 | switch true { 369 | case rel == ".git": 370 | slog.Debug().Str("path", path).Str("rel", rel).Msg("skipping dir") 371 | return filepath.SkipDir 372 | case strings.HasPrefix(rel, ".ghs3"), rel == ".": 373 | return nil 374 | } 375 | 376 | slog.Debug().Str("path", path).Str("rel", rel).Msg("walking path") 377 | 378 | files[rel] = info 379 | return nil 380 | }) 381 | if err != nil { 382 | slog.Error().Err(err).Msg("error walking file tree") 383 | return nil, err 384 | } 385 | slog.Debug().Int("file_count", len(files)).Msg("git.List.OK") 386 | return files, nil 387 | } 388 | 389 | func (g *Git) Delete(ctx context.Context, repo *git.Repository, relativeFilepath string) error { 390 | slog := util.LogCtx(ctx, "git.Delete").With().Str("component", "git.Delete").Logger() 391 | slog.Debug().Str("filename", relativeFilepath).Msg("git.Delete.Start") 392 | 393 | wt, err := repo.Worktree() 394 | if err != nil { 395 | slog.Error().Err(err).Msg("failed to get worktree") 396 | return err 397 | } 398 | err = wt.Filesystem.Remove(relativeFilepath) 399 | if err != nil { 400 | slog.Error().Err(err).Str("filename", relativeFilepath).Msg("failed to remove file from filesystem") 401 | return err 402 | } 403 | slog.Debug().Str("filename", relativeFilepath).Msg("file removed from filesystem") 404 | 405 | _, err = wt.Remove(relativeFilepath) 406 | if err != nil { 407 | slog.Error().Err(err).Str("filename", relativeFilepath).Msg("failed to remove file from git index") 408 | return err 409 | } 410 | slog.Debug().Str("filename", relativeFilepath).Msg("file removed from git index") 411 | 412 | _, err = wt.Commit("[GHS3] delete file "+relativeFilepath, &git.CommitOptions{ 413 | Author: g.signature(), 414 | }) 415 | if err != nil { 416 | slog.Error().Err(err).Str("filename", relativeFilepath).Msg("failed to commit file deletion") 417 | return err 418 | } 419 | slog.Debug().Str("filename", relativeFilepath).Msg("file deletion committed") 420 | 421 | remote, err := repo.Remote("origin") 422 | if err != nil { 423 | slog.Error().Err(err).Msg("failed to get remote 'origin'") 424 | return err 425 | } 426 | 427 | err = g.push(ctx, remote, "") 428 | if err != nil { 429 | slog.Error().Err(err).Msg("failed to push file deletion to remote") 430 | return err 431 | } 432 | slog.Debug().Str("filename", relativeFilepath).Msg("git.Delete.OK") 433 | return nil 434 | } 435 | 436 | func (g *Git) auth() transport.AuthMethod { 437 | return &http.BasicAuth{Username: "non-empty-string", Password: g.token} 438 | } 439 | 440 | func (g *Git) signature() *object.Signature { 441 | return &object.Signature{ 442 | Name: g.username, 443 | Email: g.email, 444 | When: time.Now(), 445 | } 446 | } 447 | 448 | func (g *Git) push(ctx context.Context, remote *git.Remote, reponame string) error { 449 | slog := util.LogCtx(ctx, "git.push").With().Str("component", "git.push").Logger() 450 | if g.skipPush { 451 | log.Ctx(ctx).Debug().Msg("skipping push") 452 | return nil 453 | } 454 | 455 | if remote == nil { 456 | slog.Error().Msg("remote is nil") 457 | return errors.New("remote is nil") 458 | } 459 | 460 | pushOpts := &git.PushOptions{ 461 | RemoteName: consts.Origin, 462 | Auth: g.auth(), 463 | } 464 | 465 | if len(reponame) > 0 { 466 | pushOpts.RemoteURL = util.GithubURL(g.owner, reponame) 467 | } 468 | 469 | err := remote.PushContext(ctx, pushOpts) 470 | if err != nil { 471 | slog.Error().Err(err).Any("pushOpts", pushOpts).Msg("failed to push to remote") 472 | return err 473 | } 474 | 475 | log.Ctx(ctx).Debug().Msg("push OK") 476 | return nil 477 | } 478 | -------------------------------------------------------------------------------- /internal/github/errors.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrRepoAlreadyExists = errors.New("repository already exists") 7 | ErrRepoNotFound = errors.New("repository not found") 8 | ) 9 | -------------------------------------------------------------------------------- /internal/github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github-as-s3/internal/util" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/google/go-github/v72/github" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | var requiredPermissions = []string{"repo", "delete_repo"} 17 | 18 | type GitHub struct { 19 | client *github.Client 20 | owner string 21 | } 22 | 23 | func NewGitHub(token, owner string) *GitHub { 24 | client := github.NewClient(nil).WithAuthToken(token) 25 | 26 | return &GitHub{ 27 | client: client, 28 | owner: owner, 29 | } 30 | } 31 | 32 | // nice to have 33 | func (gh *GitHub) CheckPermissions(ctx context.Context) (bool, error) { 34 | var resp map[string]any 35 | 36 | req, err := http.NewRequest(http.MethodGet, gh.client.BaseURL.String(), nil) 37 | if err != nil { 38 | return false, fmt.Errorf("create request: %w", err) 39 | } 40 | 41 | ghResp, err := gh.client.Do(ctx, req, &resp) 42 | if err != nil { 43 | return false, fmt.Errorf("check permissions: %w", err) 44 | } 45 | 46 | permissionStr := ghResp.Header.Get("X-OAuth-Scopes") 47 | if permissionStr == "" { 48 | return false, fmt.Errorf("no permissions found") 49 | } 50 | 51 | permissions := strings.Split(permissionStr, ",") 52 | 53 | missing := checkMissingPermissions(permissions) 54 | 55 | if len(missing) == 0 { 56 | return true, nil 57 | } 58 | 59 | return false, fmt.Errorf("missing required permissions: %v", missing) 60 | } 61 | 62 | func (gh *GitHub) CreateRepo(ctx context.Context, name string, isPrivate bool) error { 63 | repoName := util.RepoName(name) 64 | 65 | visibility := "public" 66 | if isPrivate { 67 | visibility = "private" 68 | } 69 | 70 | repo, _, err := gh.client.Repositories.Create(ctx, "", &github.Repository{ 71 | Name: repoName, 72 | Owner: &github.User{ 73 | Name: github.Ptr(gh.owner), 74 | }, 75 | Visibility: &visibility, 76 | }) 77 | 78 | if err != nil { 79 | var ghErrResp *github.ErrorResponse 80 | if ok := errors.As(err, &ghErrResp); ok { 81 | log.Ctx(ctx).Debug().Any("github_err", ghErrResp).Msg("GitHub API error") 82 | for _, e := range ghErrResp.Errors { 83 | if e.Code == "custom" && e.Resource == "Repository" && e.Message == "name already exists on this account" { 84 | return fmt.Errorf("%w: %s", ErrRepoAlreadyExists, *repoName) 85 | } 86 | } 87 | } 88 | 89 | return err 90 | } 91 | 92 | // TODO: figure out how to trigger check for this 93 | // dont want to list/search for it 94 | // createdAt==nil is NOT the condition 95 | if repo.CreatedAt == nil { 96 | tries := 0 97 | multiplier := 1 98 | var createdAt *github.Timestamp 99 | for createdAt == nil { 100 | if tries > 3 { 101 | return fmt.Errorf("failed to check whether repo is created") 102 | } 103 | 104 | repo, _, err := gh.client.Repositories.Get(ctx, gh.owner, *repoName) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | createdAt = repo.CreatedAt 110 | 111 | time.Sleep(time.Duration(multiplier) * time.Second) 112 | 113 | multiplier *= 2 114 | } 115 | 116 | } 117 | return nil 118 | } 119 | 120 | func (gh *GitHub) DeleteRepo(ctx context.Context, name string) error { 121 | repoName := util.RepoName(name) 122 | 123 | _, err := gh.client.Repositories.Delete(ctx, gh.owner, *repoName) 124 | if err != nil { 125 | var ghErrResp *github.ErrorResponse 126 | if ok := errors.As(err, &ghErrResp); ok { 127 | log.Ctx(ctx).Debug().Any("github_err", ghErrResp).Msg("GitHub API error") 128 | if ghErrResp.DocumentationURL == "https://docs.github.com/rest/repos/repos#delete-a-repository" && ghErrResp.Message == "Not Found" { 129 | return fmt.Errorf("%w: %s", ErrRepoNotFound, *repoName) 130 | } 131 | } 132 | 133 | return err 134 | } 135 | 136 | return nil 137 | } 138 | 139 | func (gh *GitHub) ListRepos(ctx context.Context, page, perPage int, prefix string) ([]*github.Repository, int, bool, error) { 140 | if page < 1 { 141 | return nil, 0, false, fmt.Errorf("page must be greater than 0") 142 | } 143 | 144 | search := fmt.Sprintf("user:%s ghs3-%s in:name", gh.owner, prefix) 145 | repos, _, err := gh.client.Search.Repositories(ctx, search, &github.SearchOptions{ 146 | Sort: "updated", 147 | Order: "desc", 148 | ListOptions: github.ListOptions{ 149 | Page: page, 150 | PerPage: perPage, 151 | }, 152 | }) 153 | if err != nil { 154 | return nil, 0, false, err 155 | } 156 | 157 | return repos.Repositories, page + 1, repos.GetIncompleteResults(), nil 158 | } 159 | 160 | // TODO: implement HeadBucket 161 | // func (gh *GitHub) HeadRepo(){} 162 | 163 | // can be directory or file 164 | func (gh *GitHub) Head(ctx context.Context, name, filepath, version string) (*github.RepositoryContent, []*github.RepositoryContent, *time.Time, error) { 165 | if name == "" { 166 | return nil, nil, nil, fmt.Errorf("name cannot be empty") 167 | } 168 | 169 | logger := log.Ctx(ctx).With().Str("repo", name).Str("path", filepath).Logger() 170 | 171 | getContentOptions := &github.RepositoryContentGetOptions{} 172 | 173 | if version != "" { 174 | getContentOptions.Ref = version 175 | } 176 | 177 | fileContent, directoryContent, _, err := gh.client.Repositories.GetContents(ctx, gh.owner, *util.RepoName(name), filepath, getContentOptions) 178 | if err != nil { 179 | var ghErrResp *github.ErrorResponse 180 | if ok := errors.As(err, &ghErrResp); ok { 181 | logger.Debug().Any("github_err", ghErrResp).Msg("GitHub API error") 182 | } 183 | return nil, nil, nil, err 184 | } 185 | 186 | lastModified := time.Now() 187 | 188 | // TODO: stop faking time 189 | // d, _, err := gh.client.Git.GetCommit(ctx, gh.owner, *util.RepoName(name), *fileContent.SHA) 190 | // if err != nil { 191 | // var ghErrResp *github.ErrorResponse 192 | // if ok := errors.As(err, &ghErrResp); ok { 193 | // logger.Debug().Any("github_err", ghErrResp).Msg("GitHub API error") 194 | // } 195 | 196 | // // its ok if this doesnt pass we fuzzy the date 197 | // logger.Debug().Msg("could not get commit - using time.Now()") 198 | // } else { 199 | // lastModified = d.Committer.Date.Time 200 | // logger.Debug().Time("last_modified", d.Committer.Date.Time).Msg("last modified") 201 | // } 202 | logger.Debug().Any("filecontent", fileContent).Any("directorycontent", directoryContent).Msg("content") 203 | 204 | return fileContent, directoryContent, &lastModified, nil 205 | } 206 | 207 | func (gh GitHub) GetOwner() string { 208 | return gh.owner 209 | } 210 | 211 | func checkMissingPermissions(permissions []string) []string { 212 | permissionMap := make(map[string]struct{}) 213 | 214 | for _, rp := range requiredPermissions { 215 | permissionMap[rp] = struct{}{} 216 | } 217 | 218 | for _, p := range permissions { 219 | p = strings.TrimSpace(p) 220 | delete(permissionMap, p) 221 | } 222 | 223 | missing := make([]string, 0) 224 | for p := range permissionMap { 225 | missing = append(missing, p) 226 | } 227 | 228 | return missing 229 | } 230 | -------------------------------------------------------------------------------- /internal/model/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ktunprasert/github-as-s3/4d9d2318a9c708c2ce6f97e47b4827686947dc12/internal/model/.gitkeep -------------------------------------------------------------------------------- /internal/s3/models.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "encoding/xml" 5 | ) 6 | 7 | // S3Error represents the XML structure for S3 error responses. 8 | type S3Error struct { 9 | XMLName xml.Name `xml:"Error"` 10 | Code string `xml:"Code"` 11 | Message string `xml:"Message"` 12 | BucketName string `xml:"BucketName,omitempty"` 13 | Resource string `xml:"Resource,omitempty"` 14 | RequestID string `xml:"RequestId"` 15 | HostID string `xml:"HostId"` 16 | } 17 | 18 | type CreateBucketConfiguration struct { 19 | XMLName xml.Name `xml:"CreateBucketConfiguration"` 20 | LocationConstraint string `xml:"LocationConstraint,omitempty"` 21 | } 22 | 23 | type Bucket struct { 24 | XMLName xml.Name `xml:"Bucket"` 25 | Name string `xml:"Name"` 26 | CreationDate string `xml:"CreationDate"` 27 | } 28 | 29 | type Owner struct { 30 | XMLName xml.Name `xml:"Owner"` 31 | ID string `xml:"ID"` 32 | DisplayName string `xml:"DisplayName"` 33 | } 34 | 35 | type ListAllMyBucketsResult struct { 36 | XMLName xml.Name `xml:"ListAllMyBucketsResult"` 37 | Owner Owner `xml:"Owner"` 38 | Buckets []Bucket `xml:"Buckets>Bucket"` 39 | IsTruncated bool `xml:"IsTruncated,omitempty"` 40 | NextContinuationToken int `xml:"NextContinuationToken,omitempty"` 41 | } 42 | 43 | // ListBucketResult is the top-level structure for S3 ListObjectsV2 response 44 | type ListBucketResult struct { 45 | XMLName xml.Name `xml:"ListBucketResult"` 46 | Xmlns string `xml:"xmlns,attr"` 47 | 48 | Name string `xml:"Name"` 49 | Prefix string `xml:"Prefix,omitempty"` 50 | Delimiter string `xml:"Delimiter,omitempty"` 51 | MaxKeys int `xml:"MaxKeys"` 52 | 53 | IsTruncated bool `xml:"IsTruncated"` 54 | NextContinuationToken string `xml:"NextContinuationToken,omitempty"` 55 | 56 | Contents []ContentsType `xml:"Contents,omitempty"` 57 | CommonPrefixes []CommonPrefixType `xml:"CommonPrefixes,omitempty"` 58 | 59 | KeyCount int `xml:"KeyCount"` 60 | 61 | // Echo back request parameters 62 | ContinuationToken string `xml:"ContinuationToken,omitempty"` 63 | StartAfter string `xml:"StartAfter,omitempty"` 64 | } 65 | 66 | // ContentsType represents an object in the S3 bucket 67 | type ContentsType struct { 68 | Key string `xml:"Key"` 69 | LastModified string `xml:"LastModified,omitempty"` // Placeholder: "2006-01-02T15:04:05.000Z" 70 | ETag string `xml:"ETag,omitempty"` // Placeholder: e.g., "\"d41d8cd98f00b204e9800998ecf8427e\"" 71 | Size int64 `xml:"Size"` // Placeholder 72 | StorageClass string `xml:"StorageClass,omitempty"` // e.g., "STANDARD" 73 | } 74 | 75 | // CommonPrefixType represents a common prefix (simulated directory) 76 | type CommonPrefixType struct { 77 | Prefix string `xml:"Prefix"` 78 | } 79 | -------------------------------------------------------------------------------- /internal/s3/util.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | var ( 9 | bucketNameRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`) 10 | ipAddressRegex = regexp.MustCompile(`^(\d{1,3}\.){3}\d{1,3}$`) 11 | disallowedPrefixes = []string{"xn--"} 12 | disallowedSuffixes = []string{"-s3alias", "--ol-s3"} 13 | disallowedSubstrings = []string{"..", ".-", "-."} 14 | ) 15 | 16 | func IsValidBucketName(name string) bool { 17 | if len(name) < 3 || len(name) > 63 { 18 | return false 19 | } 20 | if !bucketNameRegex.MatchString(name) { 21 | return false 22 | } 23 | if ipAddressRegex.MatchString(name) { 24 | return false 25 | } 26 | for _, prefix := range disallowedPrefixes { 27 | if strings.HasPrefix(name, prefix) { 28 | return false 29 | } 30 | } 31 | for _, suffix := range disallowedSuffixes { 32 | if strings.HasSuffix(name, suffix) { 33 | return false 34 | } 35 | } 36 | for _, sub := range disallowedSubstrings { 37 | if strings.Contains(name, sub) { 38 | return false 39 | } 40 | } 41 | return true 42 | } 43 | -------------------------------------------------------------------------------- /internal/server/bucket.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/xml" 5 | "errors" 6 | "github-as-s3/internal/github" 7 | "github-as-s3/internal/s3" 8 | "net/http" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/labstack/echo/v4" 14 | "github.com/rs/zerolog/log" 15 | ) 16 | 17 | func (h *Handler) CreateBucket(c echo.Context) error { 18 | bucketName := c.Param("bucket") 19 | ctx := c.Request().Context() 20 | logger := log.Ctx(ctx).With().Str("bucket", bucketName).Str("command", "CreateBucket").Logger() 21 | 22 | if !s3.IsValidBucketName(bucketName) { 23 | logger.Warn().Str("bucket", bucketName).Msg("Invalid bucket name format") 24 | return h.s3ErrorResponse(c, http.StatusBadRequest, "InvalidBucketName", "The specified bucket is not valid.", bucketName) 25 | } 26 | 27 | var isPrivate bool 28 | aclHeader := c.Request().Header.Get("X-Amz-Acl") 29 | logger.Debug().Str("bucket", bucketName).Str("x-amz-acl", aclHeader).Msg("Processing CreateBucket request") 30 | 31 | switch aclHeader { 32 | case "public-read", "public-read-write": 33 | isPrivate = false 34 | case "private", "": 35 | isPrivate = true 36 | default: 37 | logger.Warn().Str("bucket", bucketName).Str("acl", aclHeader).Msg("Unsupported ACL value received, defaulting to private.") 38 | isPrivate = true 39 | } 40 | 41 | // we dont care about body content tbh 42 | // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html#API_CreateBucket_RequestBody 43 | // if c.Request().ContentLength > 0 { 44 | // bodyBytes, err := io.ReadAll(c.Request().Body) 45 | // if err != nil { 46 | // logger.Error().Err(err).Str("bucket", bucketName).Msg("Failed to read request body for CreateBucketConfiguration") 47 | // return h.s3ErrorResponse(c, http.StatusInternalServerError, "InternalError", "We encountered an internal error. Please try again.", bucketName) 48 | // } 49 | // defer c.Request().Body.Close() 50 | 51 | // if len(bodyBytes) > 0 { 52 | // var config CreateBucketConfiguration 53 | // if err := xml.Unmarshal(bodyBytes, &config); err != nil { 54 | // logger.Warn().Err(err).Str("bucket", bucketName).Msg("Malformed XML in CreateBucketConfiguration") 55 | // return h.s3ErrorResponse(c, http.StatusBadRequest, "MalformedXML", "The XML you provided was not well-formed or did not validate against our published schema.", bucketName) 56 | // } 57 | // logger.Info().Str("bucket", bucketName).Str("locationConstraint", config.LocationConstraint).Msg("Parsed CreateBucketConfiguration (LocationConstraint will be ignored)") 58 | // } 59 | // } 60 | 61 | err := h.gh.CreateRepo(ctx, bucketName, isPrivate) 62 | if err != nil { 63 | if errors.Is(err, github.ErrRepoAlreadyExists) { // Replace with actual error check 64 | logger.Warn().Str("bucket", bucketName).Msg("Attempted to create a bucket that already exists (repository exists)") 65 | return h.s3ErrorResponse(c, http.StatusConflict, "BucketAlreadyOwnedByYou", "Your previous request to create the named bucket succeeded and you already own it.", bucketName) 66 | } 67 | 68 | logger.Error().Err(err).Str("bucket", bucketName).Msg("Failed to create GitHub repository") 69 | return h.s3ErrorResponse(c, http.StatusInternalServerError, "InternalError", "We encountered an internal error creating the repository. Please try again.", bucketName) 70 | } 71 | 72 | _, err = h.git.InitRepo(ctx, bucketName, "") 73 | if err != nil { 74 | logger.Error().Err(err).Str("bucket", bucketName).Msg("Failed to initialize GitHub repository") 75 | return h.s3ErrorResponse(c, http.StatusInternalServerError, "InternalError", "We failed to initialise your bucket please try again", bucketName) 76 | } 77 | 78 | c.Response().Header().Set("Location", "/"+bucketName) 79 | logger.Info().Str("bucket", bucketName).Bool("isPrivate", isPrivate).Msg("Bucket created successfully (GitHub repository created)") 80 | return c.NoContent(http.StatusOK) 81 | } 82 | 83 | func (h *Handler) DeleteBucket(c echo.Context) error { 84 | bucketName := c.Param("bucket") 85 | ctx := c.Request().Context() 86 | logger := log.Ctx(ctx).With().Str("bucket", bucketName).Str("command", "DeleteBucket").Logger() 87 | 88 | logger.Debug().Str("bucket", bucketName).Msg("Processing DeleteBucket request") 89 | 90 | err := h.gh.DeleteRepo(ctx, bucketName) 91 | if err != nil { 92 | if errors.Is(err, github.ErrRepoNotFound) { // Assuming github.ErrRepoNotFound exists 93 | logger.Warn().Str("bucket", bucketName).Msg("Attempted to delete a bucket that does not exist (repository not found)") 94 | return h.s3ErrorResponse(c, http.StatusNotFound, "NoSuchBucket", "The specified bucket does not exist.", bucketName) 95 | } 96 | 97 | logger.Error().Err(err).Str("bucket", bucketName).Msg("Failed to delete GitHub repository") 98 | return h.s3ErrorResponse(c, http.StatusInternalServerError, "InternalError", "We encountered an internal error deleting the repository. Please try again.", bucketName) 99 | } 100 | 101 | logger.Info().Str("bucket", bucketName).Msg("Bucket deleted successfully (GitHub repository deleted)") 102 | return c.NoContent(http.StatusNoContent) 103 | } 104 | 105 | const defaultMaxBuckets = 25 106 | const githubMaxPerPage = 100 107 | 108 | func (h *Handler) ListBuckets(c echo.Context) error { 109 | ctx := c.Request().Context() 110 | logger := log.Ctx(ctx).With().Str("command", "ListBuckets").Logger() 111 | 112 | continuationTokenStr := c.QueryParam("continuation-token") 113 | maxBucketsStr := c.QueryParam("max-buckets") 114 | prefix := c.QueryParam("prefix") 115 | 116 | logger.Debug(). 117 | Str("continuation-token", continuationTokenStr). 118 | Str("max-buckets", maxBucketsStr). 119 | Str("prefix", prefix). 120 | Msg("Processing ListBuckets request") 121 | 122 | page := 1 123 | if continuationTokenStr != "" { 124 | parsedPage, err := strconv.Atoi(continuationTokenStr) 125 | if err == nil && parsedPage > 0 { 126 | page = parsedPage 127 | } else { 128 | logger.Warn().Str("continuation-token", continuationTokenStr).Msg("Invalid continuation-token, using default page 1") 129 | } 130 | } 131 | 132 | perPage := defaultMaxBuckets 133 | if maxBucketsStr != "" { 134 | parsedMax, err := strconv.Atoi(maxBucketsStr) 135 | if err == nil && parsedMax > 0 { 136 | perPage = parsedMax 137 | if perPage > githubMaxPerPage { 138 | logger.Warn().Int("requested_max_buckets", perPage).Int("capped_at", githubMaxPerPage).Msg("max-buckets capped") 139 | perPage = githubMaxPerPage 140 | } 141 | } else { 142 | logger.Warn().Str("max-buckets", maxBucketsStr).Msg("Invalid max-buckets, using default") 143 | } 144 | } 145 | 146 | ghReposPage, nextPageFromGH, incompleteResults, err := h.gh.ListRepos(ctx, page, perPage, prefix) 147 | if err != nil { 148 | logger.Error().Err(err).Msg("Failed to list GitHub repositories") 149 | return h.s3ErrorResponse(c, http.StatusInternalServerError, "InternalError", "We encountered an internal error listing repositories.", "") 150 | } 151 | 152 | s3ApiBuckets := make([]s3.Bucket, 0, len(ghReposPage)) 153 | for _, repo := range ghReposPage { 154 | if repo.Name == nil || repo.CreatedAt == nil { 155 | logger.Warn().Str("repo_id", repo.GetNodeID()).Msg("Skipping repository with nil name or creation date during ListBuckets") 156 | continue 157 | } 158 | s3ApiBuckets = append(s3ApiBuckets, s3.Bucket{ 159 | Name: strings.TrimPrefix(repo.GetName(), "ghs3-"), 160 | CreationDate: repo.GetCreatedAt().Time.UTC().Format(time.RFC3339), 161 | }) 162 | } 163 | 164 | owner := h.gh.GetOwner() 165 | 166 | s3Owner := s3.Owner{ 167 | ID: owner, 168 | DisplayName: owner, 169 | } 170 | 171 | result := s3.ListAllMyBucketsResult{ 172 | Owner: s3Owner, 173 | Buckets: s3ApiBuckets, 174 | } 175 | 176 | if incompleteResults { 177 | result.IsTruncated = true 178 | result.NextContinuationToken = nextPageFromGH 179 | } 180 | 181 | xmlBytes, err := xml.MarshalIndent(result, "", " ") 182 | if err != nil { 183 | logger.Error().Err(err).Msg("Failed to marshal ListBuckets response to XML") 184 | return h.s3ErrorResponse(c, http.StatusInternalServerError, "InternalError", "We encountered an internal error preparing the response.", "") 185 | } 186 | 187 | finalXML := xml.Header + string(xmlBytes) 188 | 189 | c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationXMLCharsetUTF8) 190 | return c.String(http.StatusOK, finalXML) 191 | } 192 | -------------------------------------------------------------------------------- /internal/server/catchall.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | 7 | "github.com/labstack/echo/v4" 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func (h *Handler) CatchAllHandler(c echo.Context) error { 12 | ctx := c.Request().Context() 13 | logger := log.Ctx(ctx).With().Str("path", c.Path()).Logger() 14 | logger.Info().Msg("CatchAllHandler invoked") 15 | 16 | logger.Info().Str("method", c.Request().Method).Msg("Method") 17 | logger.Info().Any("header", c.Request().Header).Msg("Request") 18 | logger.Info().Any("query", c.QueryParams()).Msg("Query") 19 | 20 | body, err := io.ReadAll(c.Request().Body) 21 | if err != nil { 22 | logger.Error().Err(err).Msg("Error reading request body") 23 | } else { 24 | logger.Info().Str("body", string(body)).Msg("Request Body") 25 | } 26 | 27 | // This is a catch-all handler for any unmatched routes. 28 | // It can be used to return a 404 Not Found or a custom error message. 29 | return c.String(http.StatusNotImplemented, "Not Implemented") 30 | } 31 | -------------------------------------------------------------------------------- /internal/server/handler.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github-as-s3/internal/git" 7 | "github-as-s3/internal/github" 8 | "github-as-s3/internal/s3" 9 | "mime" 10 | "net/http" 11 | "path" 12 | "strings" 13 | 14 | ghlib "github.com/google/go-github/v72/github" 15 | "github.com/labstack/echo/v4" 16 | "github.com/rs/zerolog/log" 17 | ) 18 | 19 | // Handler implements the S3API interface. 20 | type Handler struct { 21 | gh *github.GitHub 22 | git *git.Git 23 | gitasync *git.GitAsync 24 | async bool 25 | } 26 | 27 | // NewHandler returns a new Handler instance. 28 | func NewS3Handler(gh *github.GitHub, g *git.Git, async bool) Handler { 29 | return Handler{ 30 | gh: gh, 31 | git: g, 32 | gitasync: git.NewGitAsync(g), 33 | async: async, 34 | } 35 | } 36 | 37 | func (h *Handler) HeadBucket(c echo.Context) error { 38 | return c.String(http.StatusNotImplemented, "Not Implemented") 39 | } 40 | 41 | func (h *Handler) HeadObject(c echo.Context) error { 42 | bucketName := c.Param("bucket") 43 | objectKey := c.Param("*") 44 | versionID := c.QueryParam("versionId") 45 | 46 | logger := log.Ctx(c.Request().Context()).With(). 47 | Str("bucket", bucketName). 48 | Str("object", objectKey). 49 | Str("versionId", versionID). 50 | Str("command", "HeadObject"). 51 | Logger() 52 | 53 | logger.Debug().Msg("HeadObject request received") 54 | 55 | fileContent, _, lastModified, err := h.gh.Head(c.Request().Context(), bucketName, objectKey, versionID) 56 | 57 | if err != nil { 58 | var ghErrResp *ghlib.ErrorResponse 59 | if errors.As(err, &ghErrResp) && ghErrResp.Response != nil && ghErrResp.Response.StatusCode == http.StatusNotFound { 60 | return h.s3ErrorResponse(c, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.", objectKey) 61 | } 62 | 63 | return h.s3ErrorResponse(c, http.StatusInternalServerError, "InternalError", "An internal error occurred while trying to head the object.", objectKey) 64 | } 65 | 66 | if fileContent == nil || (fileContent.GetType() != "file" && fileContent.GetType() != "symlink") { 67 | return h.s3ErrorResponse(c, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.", objectKey) 68 | } 69 | 70 | logger.Debug(). 71 | Str("file_sha", *fileContent.SHA). 72 | Int("size", *fileContent.Size). 73 | Str("type", *fileContent.Type). 74 | Time("last_modified", *lastModified). 75 | Msg("Object found, setting headers") 76 | 77 | c.Response().Header().Set("ETag", fmt.Sprintf("\"%s\"", *fileContent.SHA)) 78 | c.Response().Header().Set("Content-Length", fmt.Sprintf("%d", *fileContent.Size)) 79 | 80 | c.Response().Header().Set("x-amz-version-id", *fileContent.SHA) 81 | 82 | contentType := mime.TypeByExtension(path.Ext(objectKey)) 83 | if contentType == "" { 84 | contentType = "application/octet-stream" // S3 default 85 | } 86 | c.Response().Header().Set("Content-Type", contentType) 87 | c.Response().Header().Set("Accept-Ranges", "bytes") // Common for S3 objects 88 | c.Response().Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) 89 | 90 | return c.NoContent(http.StatusOK) 91 | } 92 | 93 | func (h *Handler) s3ErrorResponse(c echo.Context, httpStatus int, s3ErrorCode, message, resourceName string) error { 94 | errResp := s3.S3Error{ 95 | Code: s3ErrorCode, 96 | Message: message, 97 | RequestID: c.Response().Header().Get(echo.HeaderXRequestID), 98 | HostID: "github-as-s3", 99 | } 100 | 101 | if strings.Contains(strings.ToLower(s3ErrorCode), "bucket") || (resourceName != "" && (s3ErrorCode == "NoSuchKey" || s3ErrorCode == "NoSuchBucket")) { 102 | errResp.BucketName = resourceName 103 | } else if resourceName != "" { 104 | errResp.Resource = resourceName 105 | } 106 | 107 | c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationXMLCharsetUTF8) 108 | return c.XML(httpStatus, errResp) 109 | } 110 | -------------------------------------------------------------------------------- /internal/server/object.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github-as-s3/internal/git" 7 | "github-as-s3/internal/s3" 8 | "net/http" 9 | "sort" 10 | "strconv" 11 | "strings" 12 | "time" 13 | 14 | gogit "github.com/go-git/go-git/v5" 15 | "github.com/labstack/echo/v4" 16 | "github.com/rs/zerolog/log" 17 | ) 18 | 19 | func (h *Handler) PutObject(c echo.Context) error { 20 | ctx := c.Request().Context() 21 | 22 | logger := log.Ctx(ctx).With().Str("command", "PutObject").Logger() 23 | 24 | logger.Debug().Msg("PutObject.Start") 25 | 26 | bucketName := c.Param("bucket") 27 | if bucketName == "" { 28 | logger.Warn().Msg("Bucket name is missing") 29 | return c.String(http.StatusBadRequest, "Bucket name is missing") 30 | } 31 | 32 | objectKey := c.Param("*") 33 | if objectKey == "" { 34 | logger.Warn().Msg("Object key is missing") 35 | return c.String(http.StatusBadRequest, "Object key is missing") 36 | } 37 | 38 | objectKey = strings.TrimPrefix(objectKey, "/") 39 | if objectKey == "" { 40 | logger.Warn().Msg("Object key is missing after trimming prefix") 41 | return c.String(http.StatusBadRequest, "Object key is missing") 42 | } 43 | 44 | logger = logger.With().Str("bucket", bucketName).Str("key", objectKey).Logger() 45 | logger.Debug().Msg("Parsed parameters") 46 | 47 | repo, err := h.git.Clone(ctx, bucketName) 48 | if err != nil { 49 | logger.Error().Err(err).Str("bucket", bucketName).Msg("Failed to clone repository") 50 | return c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to clone repository '%s': %v", bucketName, err)) 51 | } 52 | logger.Debug().Msg("Repository cloned successfully") 53 | 54 | err = h.git.PutRaw(ctx, repo, objectKey, c.Request().Body) 55 | if err != nil { 56 | logger.Error().Err(err).Str("key", objectKey).Msg("Failed to put object into repository") 57 | return c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to put object '%s': %v", objectKey, err)) 58 | } 59 | logger.Debug().Msg("Object put into repository successfully") 60 | 61 | var commitSHA string 62 | headRef, err := repo.Head() 63 | if err != nil { 64 | logger.Error().Err(err).Msg("Failed to get repository HEAD for versioning") 65 | // it's ok to not have a commit SHA 66 | } 67 | 68 | commitSHA = headRef.Hash().String() 69 | logger.Debug().Str("commitSHA", commitSHA).Msg("Got commit SHA for version ID") 70 | 71 | c.Response().Header().Set("ETag", fmt.Sprintf(`"%s"`, commitSHA)) // ETag should be quoted 72 | 73 | if commitSHA != "" { 74 | c.Response().Header().Set("x-amz-version-id", commitSHA) 75 | } 76 | 77 | logger.Info().Str("versionId", commitSHA).Msg("PutObject.OK") 78 | return c.String(http.StatusOK, "") 79 | } 80 | 81 | func (h *Handler) GetObject(c echo.Context) error { 82 | ctx := c.Request().Context() 83 | logger := log.Ctx(ctx).With().Str("command", "GetObject").Logger() 84 | 85 | logger.Debug().Msg("GetObject.Start") 86 | 87 | bucketName := c.Param("bucket") 88 | if bucketName == "" { 89 | logger.Warn().Msg("Bucket name is missing") 90 | return c.String(http.StatusBadRequest, "Bucket name is missing") 91 | } 92 | logger = logger.With().Str("bucket", bucketName).Logger() 93 | 94 | objectKey := c.Param("*") 95 | if objectKey == "" { 96 | logger.Warn().Msg("Object key is missing") 97 | return c.String(http.StatusBadRequest, "Object key is missing") 98 | } 99 | objectKey = strings.TrimPrefix(objectKey, "/") 100 | if objectKey == "" { 101 | logger.Warn().Msg("Object key is missing after trimming prefix") 102 | return c.String(http.StatusBadRequest, "Object key is missing") 103 | } 104 | logger = logger.With().Str("key", objectKey).Logger() 105 | 106 | logger.Debug().Msg("Parsed parameters") 107 | 108 | repo, err := h.git.Clone(ctx, bucketName) 109 | if err != nil { 110 | logger.Error().Err(err).Msg("Failed to clone repository") 111 | return c.String(http.StatusNotFound, fmt.Sprintf("Bucket (repository) '%s' not found: %v", bucketName, err)) 112 | } 113 | logger.Debug().Msg("Repository cloned successfully") 114 | 115 | objectData, fileInfo, err := h.git.Get(ctx, repo, objectKey) 116 | if err != nil { 117 | if errors.Is(err, git.ErrFileNotExists) { 118 | logger.Warn().Err(err).Msg("Object not found in repository") 119 | return c.String(http.StatusNotFound, fmt.Sprintf("Object '%s' not found in bucket '%s'", objectKey, bucketName)) 120 | } 121 | logger.Error().Err(err).Msg("Failed to get object from repository") 122 | return c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to get object '%s': %v", objectKey, err)) 123 | } 124 | logger.Debug().Msg("Object retrieved successfully") 125 | 126 | var commitSHA string 127 | headRef, err := repo.Head() 128 | if err != nil { 129 | logger.Warn().Err(err).Msg("Failed to get repository HEAD for versioning. ETag/versionId may be affected.") 130 | } else if headRef != nil { 131 | commitSHA = headRef.Hash().String() 132 | logger.Debug().Str("commitSHA", commitSHA).Msg("Got commit SHA for version ID/ETag") 133 | } else { 134 | logger.Warn().Msg("repo.Head() returned nil ref without error. ETag/versionId may be affected.") 135 | } 136 | 137 | if commitSHA != "" { 138 | c.Response().Header().Set("ETag", fmt.Sprintf(`"%s"`, commitSHA)) 139 | c.Response().Header().Set("x-amz-version-id", commitSHA) 140 | } 141 | 142 | if fileInfo != nil { 143 | c.Response().Header().Set("Last-Modified", fileInfo.ModTime().UTC().Format(http.TimeFormat)) 144 | c.Response().Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Size())) 145 | } else { 146 | // Fallback if fileInfo is somehow nil, though git.Get should provide it. 147 | c.Response().Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) 148 | c.Response().Header().Set("Content-Length", fmt.Sprintf("%d", len(objectData))) 149 | } 150 | 151 | contentType := http.DetectContentType(objectData) 152 | c.Response().Header().Set("Content-Type", contentType) 153 | 154 | logger.Info().Str("key", objectKey).Str("bucket", bucketName).Str("versionId", commitSHA).Msg("GetObject.OK") 155 | return c.Blob(http.StatusOK, contentType, objectData) 156 | } 157 | 158 | func (h *Handler) DeleteObject(c echo.Context) error { 159 | ctx := c.Request().Context() 160 | logger := log.Ctx(ctx).With().Str("command", "DeleteObject").Logger() 161 | 162 | logger.Debug().Msg("DeleteObject.Start") 163 | 164 | bucketName := c.Param("bucket") 165 | if bucketName == "" { 166 | logger.Warn().Msg("Bucket name is missing") 167 | return c.String(http.StatusBadRequest, "Bucket name is missing") 168 | } 169 | logger = logger.With().Str("bucket", bucketName).Logger() 170 | 171 | objectKey := c.Param("*") 172 | if objectKey == "" { 173 | logger.Warn().Msg("Object key is missing") 174 | return c.String(http.StatusBadRequest, "Object key is missing") 175 | } 176 | objectKey = strings.TrimPrefix(objectKey, "/") 177 | if objectKey == "" { 178 | logger.Warn().Msg("Object key is missing after trimming prefix") 179 | return c.String(http.StatusBadRequest, "Object key is missing") 180 | } 181 | logger = logger.With().Str("key", objectKey).Logger() 182 | 183 | logger.Debug().Msg("Parsed parameters") 184 | 185 | repo, err := h.git.Clone(ctx, bucketName) 186 | if err != nil { 187 | logger.Error().Err(err).Msg("Failed to clone repository") 188 | // could make this stricter if we wanted: return c.String(http.StatusNotFound, fmt.Sprintf("Bucket (repository) '%s' not found: %v", bucketName, err)) 189 | c.Response().Header().Set("x-amz-delete-marker", "true") 190 | logger.Info().Msg("Bucket not found, but DeleteObject is idempotent. Responding with 204 No Content.") 191 | return c.NoContent(http.StatusNoContent) 192 | } 193 | logger.Debug().Msg("Repository cloned successfully") 194 | 195 | err = h.git.Delete(ctx, repo, objectKey) 196 | if err != nil { 197 | if errors.Is(err, git.ErrFileNotExists) { 198 | logger.Info().Err(err).Msg("Object not found in repository, delete is idempotent.") 199 | // S3 returns 204 No Content if the object to be deleted is not found. 200 | } else { 201 | logger.Error().Err(err).Msg("Failed to delete object from repository") 202 | return c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to delete object '%s': %v", objectKey, err)) 203 | } 204 | } else { 205 | logger.Debug().Msg("Object deleted from repository successfully") 206 | } 207 | 208 | var deleteMarkerVersionID string 209 | headRef, headErr := repo.Head() 210 | if headErr != nil { 211 | logger.Warn().Err(headErr).Msg("Failed to get repository HEAD for versioning after delete.") 212 | } else if headRef != nil { 213 | deleteMarkerVersionID = headRef.Hash().String() 214 | logger.Debug().Str("deleteMarkerVersionID", deleteMarkerVersionID).Msg("Got commit SHA for delete marker version ID") 215 | } else { 216 | logger.Warn().Msg("repo.Head() returned nil ref without error after delete.") 217 | } 218 | 219 | c.Response().Header().Set("x-amz-delete-marker", "true") 220 | if deleteMarkerVersionID != "" { 221 | // For versioned buckets, S3 returns x-amz-version-id for the delete marker 222 | c.Response().Header().Set("x-amz-version-id", deleteMarkerVersionID) 223 | } 224 | 225 | logger.Info().Str("key", objectKey).Str("bucket", bucketName).Str("versionId", deleteMarkerVersionID).Msg("DeleteObject.OK") 226 | return c.NoContent(http.StatusNoContent) 227 | } 228 | 229 | func (h *Handler) ListObjectsV2(c echo.Context) error { 230 | ctx := c.Request().Context() 231 | logger := log.Ctx(ctx).With().Str("command", "ListObjectsV2").Logger() 232 | 233 | bucketName := c.Param("bucket") 234 | if bucketName == "" { 235 | logger.Warn().Msg("Bucket name is missing") 236 | return c.String(http.StatusBadRequest, "Bucket name is missing") 237 | } 238 | logger = logger.With().Str("bucket", bucketName).Logger() 239 | 240 | // Parse query parameters for echoing back, but not for limiting results 241 | requestPrefix := c.QueryParam("prefix") 242 | requestDelimiter := c.QueryParam("delimiter") 243 | requestMaxKeysStr := c.QueryParam("max-keys") 244 | requestContinuationToken := c.QueryParam("continuation-token") // Will be echoed 245 | requestStartAfter := c.QueryParam("start-after") // Will be echoed 246 | 247 | maxKeysForResponse := 1000 // Default MaxKeys for S3 response field 248 | if requestMaxKeysStr != "" { 249 | parsedMaxKeys, err := strconv.Atoi(requestMaxKeysStr) 250 | if err == nil && parsedMaxKeys >= 0 { // S3 allows 0 for MaxKeys 251 | maxKeysForResponse = parsedMaxKeys 252 | } else { 253 | logger.Warn().Str("max-keys", requestMaxKeysStr).Msg("Invalid max-keys value, using default for response field.") 254 | } 255 | } 256 | 257 | logger.Debug(). 258 | Str("prefix", requestPrefix). 259 | Str("delimiter", requestDelimiter). 260 | Int("maxKeys (for_response_echo)", maxKeysForResponse). 261 | Str("continuationToken (for_response_echo)", requestContinuationToken). 262 | Str("startAfter (for_response_echo)", requestStartAfter). 263 | Msg("ListObjectsV2.Start - returning all results") 264 | 265 | var repo *gogit.Repository 266 | var err error 267 | 268 | if !h.async { 269 | repo, err = h.git.Clone(ctx, bucketName) 270 | } else { 271 | repo, err = h.gitasync.Clone(ctx, bucketName) 272 | } 273 | 274 | if err != nil { 275 | logger.Error().Err(err).Msg("Failed to clone repository") 276 | return c.String(http.StatusNotFound, fmt.Sprintf("Bucket '%s' not found", bucketName)) 277 | } 278 | 279 | allFilesInfo, err := h.git.List(ctx, repo) 280 | if err != nil { 281 | logger.Error().Err(err).Msg("Failed to list objects from repository") 282 | return c.String(http.StatusInternalServerError, "Failed to list objects") 283 | } 284 | 285 | var sortedKeys []string 286 | for k := range allFilesInfo { 287 | sortedKeys = append(sortedKeys, k) 288 | } 289 | sort.Strings(sortedKeys) 290 | 291 | var contents []s3.ContentsType 292 | commonPrefixesMap := make(map[string]struct{}) 293 | 294 | for _, key := range sortedKeys { 295 | fileInfo := allFilesInfo[key] // Get the os.FileInfo for the key 296 | 297 | if requestPrefix != "" && !strings.HasPrefix(key, requestPrefix) { 298 | continue 299 | } 300 | 301 | if requestDelimiter != "" { 302 | keyRelativeToPrefix := strings.TrimPrefix(key, requestPrefix) 303 | delimiterIndex := strings.Index(keyRelativeToPrefix, requestDelimiter) 304 | 305 | if delimiterIndex != -1 { 306 | // This key contributes to a common prefix 307 | commonPrefix := requestPrefix + keyRelativeToPrefix[:delimiterIndex+len(requestDelimiter)] 308 | commonPrefixesMap[commonPrefix] = struct{}{} 309 | } else { 310 | // This key is an object 311 | contents = append(contents, s3.ContentsType{ 312 | Key: key, 313 | LastModified: fileInfo.ModTime().UTC().Format("2006-01-02T15:04:05.000Z"), 314 | Size: fileInfo.Size(), 315 | StorageClass: "STANDARD", 316 | // ETag is omitted as per simplification 317 | }) 318 | } 319 | } else { 320 | // No delimiter, so everything matching the prefix is an object 321 | contents = append(contents, s3.ContentsType{ 322 | Key: key, 323 | LastModified: fileInfo.ModTime().UTC().Format("2006-01-02T15:04:05.000Z"), 324 | Size: fileInfo.Size(), 325 | StorageClass: "STANDARD", 326 | // ETag is omitted 327 | }) 328 | } 329 | } 330 | 331 | var commonPrefixesList []s3.CommonPrefixType 332 | for cp := range commonPrefixesMap { 333 | commonPrefixesList = append(commonPrefixesList, s3.CommonPrefixType{Prefix: cp}) 334 | } 335 | sort.Slice(commonPrefixesList, func(i, j int) bool { 336 | return commonPrefixesList[i].Prefix < commonPrefixesList[j].Prefix 337 | }) 338 | 339 | result := s3.ListBucketResult{ 340 | Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", 341 | Name: bucketName, 342 | Prefix: requestPrefix, 343 | Delimiter: requestDelimiter, 344 | MaxKeys: maxKeysForResponse, // Echoing back the parsed or default MaxKeys 345 | IsTruncated: false, // Always false as we return all results 346 | Contents: contents, 347 | CommonPrefixes: commonPrefixesList, 348 | KeyCount: len(contents) + len(commonPrefixesList), 349 | ContinuationToken: requestContinuationToken, // Echo back if provided 350 | StartAfter: requestStartAfter, // Echo back if provided 351 | // NextContinuationToken is omitted (or empty string) as IsTruncated is false 352 | } 353 | 354 | logger.Info(). 355 | Int("returnedContents", len(result.Contents)). 356 | Int("returnedCommonPrefixes", len(result.CommonPrefixes)). 357 | Bool("isTruncated", result.IsTruncated). 358 | Msg("ListObjectsV2.OK - all results returned") 359 | 360 | c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationXMLCharsetUTF8) 361 | return c.XML(http.StatusOK, result) 362 | } 363 | -------------------------------------------------------------------------------- /internal/server/object_async.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github-as-s3/internal/git" 7 | "mime" 8 | "net/http" 9 | "path" 10 | "strings" 11 | 12 | "github.com/labstack/echo/v4" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | func (h *Handler) HeadObjectAsync(c echo.Context) error { 17 | bucketName := c.Param("bucket") 18 | objectKey := c.Param("*") 19 | versionID := c.QueryParam("versionId") 20 | 21 | logger := log.Ctx(c.Request().Context()).With(). 22 | Str("bucket", bucketName). 23 | Str("object", objectKey). 24 | Str("versionId", versionID). 25 | Str("command", "HeadObject"). 26 | Bool("async", true). 27 | Logger() 28 | 29 | logger.Debug().Msg("HeadObject request received") 30 | file, commit, err := h.gitasync.Head(c.Request().Context(), bucketName, objectKey, versionID) 31 | if err != nil || file == nil || commit == nil { 32 | return h.s3ErrorResponse(c, http.StatusNotFound, "NoSuchKey", "The specified key does not exist.", objectKey) 33 | } 34 | 35 | lastModified := commit.Committer.When 36 | 37 | logger.Debug(). 38 | Str("file_sha", file.Hash.String()). 39 | Int64("size", file.Size). 40 | Str("type", file.Type().String()). 41 | Time("last_modified", lastModified). 42 | Msg("Object found, setting headers") 43 | 44 | c.Response().Header().Set("ETag", fmt.Sprintf("\"%s\"", file.Hash.String())) 45 | c.Response().Header().Set("Content-Length", fmt.Sprintf("%d", file.Size)) 46 | 47 | c.Response().Header().Set("x-amz-version-id", commit.Hash.String()) 48 | 49 | contentType := mime.TypeByExtension(path.Ext(objectKey)) 50 | if contentType == "" { 51 | contentType = "application/octet-stream" // S3 default 52 | } 53 | c.Response().Header().Set("Content-Type", contentType) 54 | c.Response().Header().Set("Accept-Ranges", "bytes") // Common for S3 objects 55 | c.Response().Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) 56 | 57 | return c.NoContent(http.StatusOK) 58 | } 59 | 60 | func (h *Handler) PutObjectAsync(c echo.Context) error { 61 | ctx := c.Request().Context() 62 | 63 | logger := log.Ctx(ctx).With().Str("command", "PutObject").Bool("async", true).Logger() 64 | 65 | logger.Debug().Msg("PutObject.Start") 66 | 67 | bucketName := c.Param("bucket") 68 | if bucketName == "" { 69 | logger.Warn().Msg("Bucket name is missing") 70 | return c.String(http.StatusBadRequest, "Bucket name is missing") 71 | } 72 | 73 | logger = logger.With().Str("bucket", bucketName).Logger() 74 | 75 | objectKey := c.Param("*") 76 | if objectKey == "" { 77 | logger.Warn().Msg("Object key is missing") 78 | return c.String(http.StatusBadRequest, "Object key is missing") 79 | } 80 | 81 | objectKey = strings.TrimPrefix(objectKey, "/") 82 | if objectKey == "" { 83 | logger.Warn().Msg("Object key is missing after trimming prefix") 84 | return c.String(http.StatusBadRequest, "Object key is missing") 85 | } 86 | 87 | logger.Debug().Str("bucket", bucketName).Str("key", objectKey).Msg("Parsed parameters") 88 | 89 | // async difference starts 90 | repo, err := h.gitasync.Clone(ctx, bucketName) 91 | if err != nil { 92 | logger.Error().Err(err).Msg("Failed to clone repository") 93 | return c.String(http.StatusInternalServerError, "Failed to clone repository") 94 | } 95 | logger.Debug().Msg("Cloned repository") 96 | 97 | err = h.gitasync.PutRaw(ctx, repo, bucketName, objectKey, c.Request().Body) 98 | if err != nil { 99 | logger.Error().Err(err).Msg("Failed to put object") 100 | return c.String(http.StatusInternalServerError, "Failed to put object") 101 | } 102 | 103 | var commitSHA string 104 | headRef, err := repo.Head() 105 | if err != nil { 106 | logger.Error().Err(err).Msg("Failed to get repository HEAD for versioning") 107 | // it's ok to not have a commit SHA 108 | } 109 | 110 | commitSHA = headRef.Hash().String() 111 | logger.Debug().Str("commitSHA", commitSHA).Msg("Got commit SHA for version ID") 112 | 113 | c.Response().Header().Set("ETag", fmt.Sprintf(`"%s"`, commitSHA)) // ETag should be quoted 114 | 115 | if commitSHA != "" { 116 | c.Response().Header().Set("x-amz-version-id", commitSHA) 117 | } 118 | 119 | logger.Info().Str("versionId", commitSHA).Msg("PutObject.OK") 120 | return c.String(http.StatusOK, "") 121 | } 122 | 123 | func (h *Handler) DeleteObjectAsync(c echo.Context) error { 124 | ctx := c.Request().Context() 125 | logger := log.Ctx(ctx).With().Str("command", "DeleteObject").Logger() 126 | 127 | logger.Debug().Msg("DeleteObject.Start") 128 | 129 | bucketName := c.Param("bucket") 130 | if bucketName == "" { 131 | logger.Warn().Msg("Bucket name is missing") 132 | return c.String(http.StatusBadRequest, "Bucket name is missing") 133 | } 134 | logger = logger.With().Str("bucket", bucketName).Logger() 135 | 136 | objectKey := c.Param("*") 137 | if objectKey == "" { 138 | logger.Warn().Msg("Object key is missing") 139 | return c.String(http.StatusBadRequest, "Object key is missing") 140 | } 141 | objectKey = strings.TrimPrefix(objectKey, "/") 142 | if objectKey == "" { 143 | logger.Warn().Msg("Object key is missing after trimming prefix") 144 | return c.String(http.StatusBadRequest, "Object key is missing") 145 | } 146 | logger = logger.With().Str("key", objectKey).Logger() 147 | 148 | logger.Debug().Msg("Parsed parameters") 149 | 150 | repo, err := h.gitasync.Clone(ctx, bucketName) 151 | if err != nil { 152 | logger.Error().Err(err).Msg("Failed to clone repository") 153 | // could make this stricter if we wanted: return c.String(http.StatusNotFound, fmt.Sprintf("Bucket (repository) '%s' not found: %v", bucketName, err)) 154 | c.Response().Header().Set("x-amz-delete-marker", "true") 155 | logger.Info().Msg("Bucket not found, but DeleteObject is idempotent. Responding with 204 No Content.") 156 | return c.NoContent(http.StatusNoContent) 157 | } 158 | logger.Debug().Msg("Repository cloned successfully") 159 | 160 | err = h.gitasync.Delete(ctx, repo, bucketName, objectKey) 161 | if err != nil { 162 | if errors.Is(err, git.ErrFileNotExists) { 163 | logger.Info().Err(err).Msg("Object not found in repository, delete is idempotent.") 164 | // S3 returns 204 No Content if the object to be deleted is not found. 165 | } else { 166 | logger.Error().Err(err).Msg("Failed to delete object from repository") 167 | return c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to delete object '%s': %v", objectKey, err)) 168 | } 169 | } else { 170 | logger.Debug().Msg("Object deleted from repository successfully") 171 | } 172 | 173 | var deleteMarkerVersionID string 174 | headRef, headErr := repo.Head() 175 | if headErr != nil { 176 | logger.Warn().Err(headErr).Msg("Failed to get repository HEAD for versioning after delete.") 177 | } else if headRef != nil { 178 | deleteMarkerVersionID = headRef.Hash().String() 179 | logger.Debug().Str("deleteMarkerVersionID", deleteMarkerVersionID).Msg("Got commit SHA for delete marker version ID") 180 | } else { 181 | logger.Warn().Msg("repo.Head() returned nil ref without error after delete.") 182 | } 183 | 184 | c.Response().Header().Set("x-amz-delete-marker", "true") 185 | if deleteMarkerVersionID != "" { 186 | // For versioned buckets, S3 returns x-amz-version-id for the delete marker 187 | c.Response().Header().Set("x-amz-version-id", deleteMarkerVersionID) 188 | } 189 | 190 | logger.Info().Str("key", objectKey).Str("bucket", bucketName).Str("versionId", deleteMarkerVersionID).Msg("DeleteObject.OK") 191 | return c.NoContent(http.StatusNoContent) 192 | } 193 | -------------------------------------------------------------------------------- /internal/server/router.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import "github.com/labstack/echo/v4" 4 | 5 | // API defines the S3-compatible API interface. 6 | type S3API interface { 7 | CreateBucket(echo.Context) error 8 | DeleteBucket(echo.Context) error 9 | ListBuckets(echo.Context) error 10 | HeadBucket(echo.Context) error 11 | 12 | PutObject(echo.Context) error 13 | GetObject(echo.Context) error 14 | ListObjectsV2(echo.Context) error 15 | DeleteObject(echo.Context) error 16 | HeadObject(echo.Context) error 17 | } 18 | 19 | // RegisterRoutes registers S3-compatible routes with the Echo router. 20 | func RegisterRoutes(e *echo.Echo, api Handler) { 21 | // Bucket operations 22 | e.PUT("/:bucket", api.CreateBucket) 23 | e.DELETE("/:bucket", api.DeleteBucket) 24 | e.GET("/", api.ListBuckets) 25 | e.HEAD("/:bucket", api.HeadBucket) 26 | 27 | if api.async { 28 | e.PUT("/:bucket/*", api.PutObjectAsync) 29 | e.DELETE("/:bucket/*", api.DeleteObjectAsync) 30 | e.HEAD("/:bucket/*", api.HeadObjectAsync) 31 | } else { 32 | e.PUT("/:bucket/*", api.PutObject) 33 | e.DELETE("/:bucket/*", api.DeleteObject) 34 | e.HEAD("/:bucket/*", api.HeadObject) 35 | } 36 | e.GET("/:bucket/*", api.GetObject) 37 | e.GET("/:bucket", api.ListObjectsV2) 38 | e.GET("/:bucket/", api.ListObjectsV2) // Handles ?list-type=2 39 | 40 | e.Any("/*", api.CatchAllHandler) 41 | } 42 | -------------------------------------------------------------------------------- /internal/util/github.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "fmt" 4 | 5 | func GithubURL(owner, name string) string { 6 | return fmt.Sprintf("https://github.com/%s/%s", owner, *RepoName(name)) 7 | } 8 | -------------------------------------------------------------------------------- /internal/util/log.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/rs/zerolog" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | func LogCtx(ctx context.Context, contextParent string) *zerolog.Logger { 11 | slog := log.Ctx(ctx) 12 | if !slog.Info().Enabled() { 13 | v := log.With().Str("contextParent", contextParent).Logger() 14 | slog = &v 15 | } 16 | 17 | return slog 18 | } 19 | -------------------------------------------------------------------------------- /internal/util/name.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "fmt" 4 | 5 | func RepoName(name string) *string { 6 | s := fmt.Sprintf("ghs3-%s", name) 7 | 8 | return &s 9 | } 10 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # github-as-s3 2 | 3 | > This post is an educational exploration into the S3 protocol and a practical guide on how one might implement it using Git as the underlying storage mechanism. 4 | > Please be mindful of the terms of service and code of conduct for any Git hosting provider you choose to use with these concepts; I am not responsible for any actions taken against your accounts. 5 | 6 | ![a picture of git pretending to be s3](https://kristun.dev/_astro/s3-git-scooby-doo.D9vfHZvy_ZyPgXk.webp) 7 | 8 | This project contains the source code for the Github+S3 proxy. 9 | 10 | The intended use is to proxy object storage onto Git itself. You can read more about this [here](https://kristun.dev/posts/git-as-s3/) 11 | 12 | The following routes are implemented 13 | 14 | - [PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) 15 | - [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) 16 | - [HeadObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html) 17 | - [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html) 18 | - [ListObjectsV2](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) 19 | - [CreateBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html) 20 | - [DeleteBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html) 21 | - [ListBuckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html) 22 | 23 | # Getting started 24 | 25 | ## Run it with Go 26 | 27 | ```bash 28 | cp .env.example .env 29 | go run ./cmd/cli/ 30 | ``` 31 | 32 | The environment values require the following: 33 | 34 | ```bash 35 | # required - replace with your own username and token 36 | GITHUB_TOKEN= 37 | GITHUB_OWNER=ktunprasert 38 | 39 | # optional 40 | # hosting related 41 | GHS3_PORT= 42 | GHS3_ADDRESS= 43 | # committer's information 44 | # these are the defaults defined in application.go 45 | GIT_USERNAME=GHS3 46 | GIT_EMAIL=bot@ghs3.com 47 | ``` 48 | 49 | The token requires 2 permissions: `repo`, `delete_repo` 50 | 51 | ## Run it with Docker 52 | 53 | ```bash 54 | docker build . -t ghs3 55 | docker run --rm -t ghs3 --env "GITHUB_OWNER=ktunprasert" --env "GITHUB_TOKEN=$TOKEN" 56 | ``` 57 | 58 | ## Run it with docker-compose 59 | 60 | ```bash 61 | # configure your compose file with proper environment values 62 | docker-compose up -d 63 | ``` 64 | 65 | ## Development 66 | 67 | ```bash 68 | air 69 | ``` 70 | 71 | Hot reloading with [`air`](https://github.com/air-verse/air) is configured 72 | 73 | # Compatible & Tested applications 74 | 75 | | Tool | Tested | | 76 | | ------------------------------------- | ----------------------------------------------------------------- | --- | 77 | | [rclone](https://rclone.org/) | cp, sync, delete, mkdir, purge, ls, deletefile, touch, lsd, rmdir | ✅ | 78 | | [aws s3](https://aws.amazon.com/cli/) | mb, cp, ls, rm | ✅ | 79 | | [pocketbase](https://pocketbase.io/) | creating and restoring back up + deleting files | ✅ | 80 | | [pocketbase](https://pocketbase.io/) | using as file storage | ❓ | 81 | 82 | # FAQ 83 | 84 | ## How to set up my rclone to read to it? 85 | 86 | ```conf 87 | [ghs3] 88 | type = s3 89 | provider = Other 90 | endpoint = http://localhost:8080 91 | list_version = 2 92 | ``` 93 | 94 | This was all I needed to start moving files 95 | 96 | ```bash 97 | rclone mkdir ghs3:/test-repo/ 98 | rclone copy hello-world.txt ghs3:/test-repo/ 99 | ``` 100 | --------------------------------------------------------------------------------