├── .editorconfig ├── .github └── workflows │ └── go.yml ├── .gitignore ├── .goreleaser.yml ├── AUTHORS ├── CONTRIBUTORS ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── srpmproc │ ├── fetch.go │ └── main.go ├── gen.go ├── go.mod ├── go.sum ├── modulemd ├── modulemd.go └── v3.go ├── pb ├── cfg.pb.go └── response.pb.go ├── pkg ├── blob │ ├── blob.go │ ├── file │ │ └── file.go │ ├── gcs │ │ └── gcs.go │ └── s3 │ │ └── s3.go ├── data │ ├── import.go │ ├── process.go │ └── utils.go ├── directives │ ├── add.go │ ├── delete.go │ ├── directives.go │ ├── lookaside.go │ ├── patch.go │ ├── replace.go │ └── spec_change.go ├── misc │ └── regex.go ├── modes │ └── git.go ├── rpmutils │ └── regex.go └── srpmproc │ ├── fetch.go │ ├── patch.go │ └── process.go └── proto ├── cfg.proto └── response.proto /.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | end_of_line = lf 11 | 12 | [*.md] 13 | max_line_length = off 14 | trim_trailing_whitespace = false 15 | 16 | [*.go] 17 | indent_size = 4 18 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | - name: Set up Go 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version: 1.21 20 | - name: Run GoReleaser 21 | uses: goreleaser/goreleaser-action@v6 22 | with: 23 | distribution: goreleaser 24 | version: latest 25 | args: release --clean 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | testdata 17 | /srpmproc 18 | .idea 19 | /dist/ 20 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod tidy 4 | builds: 5 | - main: ./cmd/srpmproc 6 | binary: srpmproc 7 | ldflags: 8 | - -s -w 9 | env: 10 | - CGO_ENABLED=0 11 | goos: 12 | - linux 13 | - darwin 14 | goarch: 15 | - amd64 16 | - arm64 17 | - s390x 18 | - ppc64le 19 | - riscv64 20 | archives: 21 | - name_template: >- 22 | {{- .ProjectName }}_ 23 | {{- .Version }} 24 | {{- title .Os }}_ 25 | {{- if eq .Arch "amd64" }}x86_64 26 | {{- else if eq .Arch "arm64" }}aarch64 27 | {{- else }}{{ .Arch }}{{ end -}} 28 | checksum: 29 | name_template: 'checksums.txt' 30 | snapshot: 31 | name_template: "{{ .Tag }}-next" 32 | changelog: 33 | sort: asc 34 | filters: 35 | exclude: 36 | - '^docs:' 37 | - '^test:' 38 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Mustafa Gezen 2 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | Mustafa Gezen 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15.6-alpine 2 | COPY . /src 3 | WORKDIR /src 4 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ./cmd/srpmproc 5 | 6 | FROM centos:8.3.2011 7 | COPY --from=0 /src/srpmproc /usr/bin/srpmproc 8 | 9 | ENTRYPOINT ["/usr/bin/srpmproc"] 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 The Srpmproc Authors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # srpmproc 2 | Upstream package importer with auto patching. Reference implementation for OpenPatch 3 | 4 | ## Usage 5 | ``` 6 | Usage: 7 | srpmproc [flags] 8 | srpmproc [command] 9 | 10 | Available Commands: 11 | fetch 12 | help Help about any command 13 | 14 | Flags: 15 | --basic-password string Basic auth password 16 | --basic-username string Basic auth username 17 | --branch-prefix string Branch prefix (replaces import-branch-prefix) (default "r") 18 | --branch-suffix string Branch suffix to use for imported branches 19 | --cdn string CDN URL shortcuts for well-known distros, auto-assigns --cdn-url. Valid values: rocky8, rocky, fedora, centos, centos-stream. Setting this overrides --cdn-url 20 | --cdn-url string CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs) (default "https://git.centos.org/sources") 21 | --git-committer-email string Email of committer (default "rockyautomation@rockylinux.org") 22 | --git-committer-name string Name of committer (default "rockyautomation") 23 | -h, --help help for srpmproc 24 | --import-branch-prefix string Import branch prefix (default "c") 25 | --manual-commits string Comma separated branch and commit list for packages with broken release tags (Format: BRANCH:HASH) 26 | --module-fallback-stream string Override fallback stream. Some module packages are published as collections and mostly use the same stream name, some of them deviate from the main stream 27 | --module-mode If enabled, imports a module instead of a package 28 | --module-prefix string Where to retrieve modules if exists. Only used when source-rpm is a git repo (default "https://git.centos.org/modules") 29 | --no-dup-mode If enabled, skips already imported tags 30 | --no-storage-download If enabled, blobs are always downloaded from upstream 31 | --no-storage-upload If enabled, blobs are not uploaded to blob storage 32 | --package-release string Package release to fetch 33 | --package-version string Package version to fetch 34 | --rpm-prefix string Where to retrieve SRPM content. Only used when source-rpm is not a local file (default "https://git.centos.org/rpms") 35 | --single-tag string If set, only this tag is imported 36 | --source-rpm string Location of RPM to process 37 | --ssh-key-location string Location of the SSH key to use to authenticate against upstream 38 | --ssh-user string SSH User (default "git") 39 | --storage-addr string Bucket to use as blob storage 40 | --strict-branch-mode If enabled, only branches with the calculated name are imported and not prefix only 41 | --taglessmode Tagless mode: If set, pull the latest commit from the branch and determine version numbers from spec file. This is auto-tried if tags aren't found. 42 | --tmpfs-mode string If set, packages are imported to path and patched but not pushed 43 | --upstream-prefix string Upstream git repository prefix 44 | --version int Upstream version 45 | 46 | Use "srpmproc [command] --help" for more information about a command. 47 | ``` 48 | 49 |
50 | 51 | ## Examples: 52 | 53 | 1. Import the kernel package from git.centos.org/rpms/, to local folder /opt/gitroot/rpms/kernel.git/ . Download the lookaside source tarballs from the default CentOS file server location to local folder `/opt/fake_s3/` . We want to grab branch "c8" (import prefix plus RHEL version), and it will be committed as branch "r8" (branch prefix plus RHEL version). This assumes that `/opt/fake_s3` exists, and `/opt/gitroot/rpms/kernel.git` exists and is a git repository of some kind (even an empty one). 54 | 55 | ``` 56 | srpmproc --branch-prefix "r" --import-branch-prefix "c" --rpm-prefix "https://git.centos.org/rpms" --version 8 --storage-addr file:///opt/fake_s3 --upstream-prefix file:///opt/gitroot --cdn centos --strict-branch-mode --source-rpm kernel 57 | ``` 58 | 59 |
60 | 61 | ## CDN and --cdn-url 62 | The --cdn-url option allows for Go-style templates to craft complex URL patterns. These templates are: `{{.Name}}` (package name), `{{.Hash}}` (hash of lookaside file), `{{.Hashtype}}` (hash type of file, like "sha256" or "sha512"), `{{.Branch}}` (the branch we are importing), and `{{.Filename}}` (the lookaside file's name as it appears in SOURCES/). You can add these values as part of --cdn-url to craft your lookaside pattern. 63 | 64 | 65 | For example, if I wanted my lookaside downloads to come from CentOS 9 Stream, I would use as part of my command: 66 | ``` 67 | --cdn-url "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}" 68 | ``` 69 | 70 | 71 | **Default Behavior:** If these templates are not used, the default behavior of `--cdn-url` is to fall back on the traditional RHEL import pattern: `///` . If that fails, a further fallback is attempted, the simple: `/`. These cover the common Rocky Linux and RHEL/CentOS imports if the base lookaside URL is the only thing given. If no `--cdn-url` is specified, it defaults to "https://git.centos.org/sources" (for RHEL imports into Rocky Linux) 72 | 73 | 74 | **CDN Shorthand:** For convenience, some lookaside patterns for popular distros are provided via the `--cdn` option. You can specify this without needing to use the longer `--cdn-url`. For example, when importing from CentOS 9 Stream, you could use `--cdn centos-stream` 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /cmd/srpmproc/fetch.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package main 22 | 23 | import ( 24 | "log" 25 | "os" 26 | 27 | "github.com/go-git/go-billy/v5/osfs" 28 | "github.com/rocky-linux/srpmproc/pkg/srpmproc" 29 | "github.com/spf13/cobra" 30 | ) 31 | 32 | var fetch = &cobra.Command{ 33 | Use: "fetch", 34 | Run: runFetch, 35 | } 36 | 37 | var cdnUrl string 38 | 39 | func init() { 40 | fetch.Flags().StringVar(&cdnUrl, "cdn-url", "", "Path to CDN") 41 | _ = fetch.MarkFlagRequired("cdn-url") 42 | 43 | root.AddCommand(fetch) 44 | } 45 | 46 | func runFetch(_ *cobra.Command, _ []string) { 47 | wd, err := os.Getwd() 48 | if err != nil { 49 | log.Fatalf("could not get working directory: %v", err) 50 | } 51 | 52 | err = srpmproc.Fetch(os.Stdout, cdnUrl, wd, osfs.New("/"), nil) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /cmd/srpmproc/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package main 22 | 23 | import ( 24 | "encoding/json" 25 | "log" 26 | "os" 27 | 28 | "github.com/rocky-linux/srpmproc/pkg/srpmproc" 29 | 30 | "github.com/spf13/cobra" 31 | ) 32 | 33 | var ( 34 | sourceRpm string 35 | sourceRpmGitName string 36 | sshKeyLocation string 37 | sshUser string 38 | sshAskKeyPassword bool 39 | upstreamPrefix string 40 | version int 41 | storageAddr string 42 | gitCommitterName string 43 | gitCommitterEmail string 44 | modulePrefix string 45 | rpmPrefix string 46 | importBranchPrefix string 47 | branchPrefix string 48 | singleTag string 49 | noDupMode bool 50 | moduleMode bool 51 | tmpFsMode string 52 | noStorageDownload bool 53 | noStorageUpload bool 54 | manualCommits string 55 | moduleFallbackStream string 56 | branchSuffix string 57 | strictBranchMode bool 58 | basicUsername string 59 | basicPassword string 60 | packageVersion string 61 | packageRelease string 62 | taglessMode bool 63 | cdn string 64 | moduleBranchNames bool 65 | ) 66 | 67 | var root = &cobra.Command{ 68 | Use: "srpmproc", 69 | Run: mn, 70 | } 71 | 72 | func mn(_ *cobra.Command, _ []string) { 73 | pd, err := srpmproc.NewProcessData(&srpmproc.ProcessDataRequest{ 74 | Version: version, 75 | StorageAddr: storageAddr, 76 | Package: sourceRpm, 77 | PackageGitName: sourceRpmGitName, 78 | ModuleMode: moduleMode, 79 | TmpFsMode: tmpFsMode, 80 | ModulePrefix: modulePrefix, 81 | RpmPrefix: rpmPrefix, 82 | SshKeyLocation: sshKeyLocation, 83 | SshUser: sshUser, 84 | SshKeyPassword: sshAskKeyPassword, 85 | ManualCommits: manualCommits, 86 | UpstreamPrefix: upstreamPrefix, 87 | GitCommitterName: gitCommitterName, 88 | GitCommitterEmail: gitCommitterEmail, 89 | ImportBranchPrefix: importBranchPrefix, 90 | BranchPrefix: branchPrefix, 91 | NoDupMode: noDupMode, 92 | BranchSuffix: branchSuffix, 93 | StrictBranchMode: strictBranchMode, 94 | ModuleFallbackStream: moduleFallbackStream, 95 | NoStorageUpload: noStorageUpload, 96 | NoStorageDownload: noStorageDownload, 97 | SingleTag: singleTag, 98 | CdnUrl: cdnUrl, 99 | HttpUsername: basicUsername, 100 | HttpPassword: basicPassword, 101 | PackageVersion: packageVersion, 102 | PackageRelease: packageRelease, 103 | TaglessMode: taglessMode, 104 | Cdn: cdn, 105 | ModuleBranchNames: moduleBranchNames, 106 | }) 107 | if err != nil { 108 | log.Fatal(err) 109 | } 110 | 111 | res, err := srpmproc.ProcessRPM(pd) 112 | if err != nil { 113 | log.Fatal(err) 114 | } 115 | 116 | err = json.NewEncoder(os.Stdout).Encode(res) 117 | if err != nil { 118 | log.Fatal(err) 119 | } 120 | } 121 | 122 | func main() { 123 | root.Flags().StringVar(&sourceRpm, "source-rpm", "", "Location of RPM to process") 124 | _ = root.MarkFlagRequired("source-rpm") 125 | root.Flags().StringVar(&upstreamPrefix, "upstream-prefix", "", "Upstream git repository prefix") 126 | _ = root.MarkFlagRequired("upstream-prefix") 127 | root.Flags().IntVar(&version, "version", 0, "Upstream version") 128 | _ = root.MarkFlagRequired("version") 129 | root.Flags().StringVar(&storageAddr, "storage-addr", "", "Bucket to use as blob storage") 130 | _ = root.MarkFlagRequired("storage-addr") 131 | 132 | root.Flags().StringVar(&sourceRpmGitName, "source-rpm-git-name", "", "Actual git repo name of package if name is different from source-rpm value") 133 | root.Flags().StringVar(&sshKeyLocation, "ssh-key-location", "", "Location of the SSH key to use to authenticate against upstream") 134 | root.Flags().StringVar(&sshUser, "ssh-user", "git", "SSH User") 135 | root.Flags().BoolVar(&sshAskKeyPassword, "ssh-key-password", false, "If enabled, prompt for ssh key password") 136 | root.Flags().StringVar(&gitCommitterName, "git-committer-name", "rockyautomation", "Name of committer") 137 | root.Flags().StringVar(&gitCommitterEmail, "git-committer-email", "rockyautomation@rockylinux.org", "Email of committer") 138 | root.Flags().StringVar(&modulePrefix, "module-prefix", "https://git.centos.org/modules", "Where to retrieve modules if exists. Only used when source-rpm is a git repo") 139 | root.Flags().StringVar(&rpmPrefix, "rpm-prefix", "https://git.centos.org/rpms", "Where to retrieve SRPM content. Only used when source-rpm is not a local file") 140 | root.Flags().StringVar(&importBranchPrefix, "import-branch-prefix", "c", "Import branch prefix") 141 | root.Flags().StringVar(&branchPrefix, "branch-prefix", "r", "Branch prefix (replaces import-branch-prefix)") 142 | root.Flags().StringVar(&cdnUrl, "cdn-url", "https://git.centos.org/sources", "CDN URL to download blobs from. Simple URL follows default rocky/centos patterns. Can be customized using macros (see docs)") 143 | root.Flags().StringVar(&singleTag, "single-tag", "", "If set, only this tag is imported") 144 | root.Flags().BoolVar(&noDupMode, "no-dup-mode", false, "If enabled, skips already imported tags") 145 | root.Flags().BoolVar(&moduleMode, "module-mode", false, "If enabled, imports a module instead of a package") 146 | root.Flags().StringVar(&tmpFsMode, "tmpfs-mode", "", "If set, packages are imported to path and patched but not pushed") 147 | root.Flags().BoolVar(&noStorageDownload, "no-storage-download", false, "If enabled, blobs are always downloaded from upstream") 148 | root.Flags().BoolVar(&noStorageUpload, "no-storage-upload", false, "If enabled, blobs are not uploaded to blob storage") 149 | root.Flags().StringVar(&manualCommits, "manual-commits", "", "Comma separated branch and commit list for packages with broken release tags (Format: BRANCH:HASH)") 150 | root.Flags().StringVar(&moduleFallbackStream, "module-fallback-stream", "", "Override fallback stream. Some module packages are published as collections and mostly use the same stream name, some of them deviate from the main stream") 151 | root.Flags().StringVar(&branchSuffix, "branch-suffix", "", "Branch suffix to use for imported branches") 152 | root.Flags().BoolVar(&strictBranchMode, "strict-branch-mode", false, "If enabled, only branches with the calculated name are imported and not prefix only") 153 | root.Flags().StringVar(&basicUsername, "basic-username", "", "Basic auth username") 154 | root.Flags().StringVar(&basicPassword, "basic-password", "", "Basic auth password") 155 | root.Flags().StringVar(&packageVersion, "package-version", "", "Package version to fetch") 156 | root.Flags().StringVar(&packageRelease, "package-release", "", "Package release to fetch") 157 | root.Flags().BoolVar(&taglessMode, "taglessmode", false, "Tagless mode: If set, pull the latest commit from the branch and determine version numbers from spec file. This is auto-tried if tags aren't found.") 158 | root.Flags().StringVar(&cdn, "cdn", "", "CDN URL shortcuts for well-known distros, auto-assigns --cdn-url. Valid values: rocky8, rocky, fedora, centos, centos-stream. Setting this overrides --cdn-url") 159 | root.Flags().BoolVar(&moduleBranchNames, "module-branch-names-only", false, "If enabled, module imports will use the branch name that is being imported, rather than use the commit hash.") 160 | 161 | if err := root.Execute(); err != nil { 162 | log.Fatal(err) 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /gen.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | //go:generate protoc -Iproto --go_opt=paths=source_relative --go_out=pb proto/cfg.proto proto/response.proto 22 | package srpmproc 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rocky-linux/srpmproc 2 | 3 | go 1.21 4 | 5 | require ( 6 | cloud.google.com/go/storage v1.43.0 7 | github.com/aws/aws-sdk-go v1.54.19 8 | github.com/bluekeyes/go-gitdiff v0.7.3 9 | github.com/go-git/go-billy/v5 v5.5.0 10 | github.com/go-git/go-git/v5 v5.12.0 11 | github.com/spf13/cobra v1.8.1 12 | github.com/spf13/viper v1.19.0 13 | google.golang.org/protobuf v1.34.2 14 | gopkg.in/yaml.v3 v3.0.1 15 | ) 16 | 17 | require ( 18 | cloud.google.com/go v0.115.0 // indirect 19 | cloud.google.com/go/auth v0.7.1 // indirect 20 | cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect 21 | cloud.google.com/go/compute/metadata v0.5.0 // indirect 22 | cloud.google.com/go/iam v1.1.11 // indirect 23 | dario.cat/mergo v1.0.0 // indirect 24 | github.com/Microsoft/go-winio v0.6.2 // indirect 25 | github.com/ProtonMail/go-crypto v1.0.0 // indirect 26 | github.com/cloudflare/circl v1.3.9 // indirect 27 | github.com/cyphar/filepath-securejoin v0.3.0 // indirect 28 | github.com/emirpasic/gods v1.18.1 // indirect 29 | github.com/felixge/httpsnoop v1.0.4 // indirect 30 | github.com/fsnotify/fsnotify v1.7.0 // indirect 31 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect 32 | github.com/go-logr/logr v1.4.2 // indirect 33 | github.com/go-logr/stdr v1.2.2 // indirect 34 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 35 | github.com/golang/protobuf v1.5.4 // indirect 36 | github.com/google/s2a-go v0.1.7 // indirect 37 | github.com/google/uuid v1.6.0 // indirect 38 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 39 | github.com/googleapis/gax-go/v2 v2.12.5 // indirect 40 | github.com/hashicorp/hcl v1.0.0 // indirect 41 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 42 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 43 | github.com/jmespath/go-jmespath v0.4.0 // indirect 44 | github.com/kevinburke/ssh_config v1.2.0 // indirect 45 | github.com/magiconair/properties v1.8.7 // indirect 46 | github.com/mitchellh/mapstructure v1.5.0 // indirect 47 | github.com/pelletier/go-toml v1.9.5 // indirect 48 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 49 | github.com/pjbgf/sha1cd v0.3.0 // indirect 50 | github.com/sagikazarmark/locafero v0.6.0 // indirect 51 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 52 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect 53 | github.com/skeema/knownhosts v1.2.2 // indirect 54 | github.com/sourcegraph/conc v0.3.0 // indirect 55 | github.com/spf13/afero v1.11.0 // indirect 56 | github.com/spf13/cast v1.6.0 // indirect 57 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 58 | github.com/spf13/pflag v1.0.5 // indirect 59 | github.com/subosito/gotenv v1.6.0 // indirect 60 | github.com/xanzy/ssh-agent v0.3.3 // indirect 61 | go.opencensus.io v0.24.0 // indirect 62 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect 63 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect 64 | go.opentelemetry.io/otel v1.28.0 // indirect 65 | go.opentelemetry.io/otel/metric v1.28.0 // indirect 66 | go.opentelemetry.io/otel/trace v1.28.0 // indirect 67 | go.uber.org/atomic v1.11.0 // indirect 68 | go.uber.org/multierr v1.11.0 // indirect 69 | golang.org/x/crypto v0.25.0 // indirect 70 | golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 // indirect 71 | golang.org/x/mod v0.19.0 // indirect 72 | golang.org/x/net v0.27.0 // indirect 73 | golang.org/x/oauth2 v0.21.0 // indirect 74 | golang.org/x/sync v0.7.0 // indirect 75 | golang.org/x/sys v0.27.0 // indirect 76 | golang.org/x/term v0.26.0 77 | golang.org/x/text v0.16.0 // indirect 78 | golang.org/x/time v0.5.0 // indirect 79 | golang.org/x/tools v0.23.0 // indirect 80 | google.golang.org/api v0.188.0 // indirect 81 | google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d // indirect 82 | google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect 83 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect 84 | google.golang.org/grpc v1.65.0 // indirect 85 | gopkg.in/ini.v1 v1.67.0 // indirect 86 | gopkg.in/warnings.v0 v0.1.2 // indirect 87 | gopkg.in/yaml.v2 v2.4.0 // indirect 88 | ) 89 | -------------------------------------------------------------------------------- /modulemd/modulemd.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package modulemd 22 | 23 | import ( 24 | "fmt" 25 | 26 | "github.com/go-git/go-billy/v5" 27 | "gopkg.in/yaml.v3" 28 | ) 29 | 30 | type ServiceLevelType string 31 | 32 | const ( 33 | ServiceLevelRawhide ServiceLevelType = "rawhide" 34 | ServiceLevelStableAPI ServiceLevelType = "stable_api" 35 | ServiceLevelBugFixes ServiceLevelType = "bug_fixes" 36 | ServiceLevelSecurityFixes ServiceLevelType = "security_fixes" 37 | ) 38 | 39 | type ServiceLevel struct { 40 | Eol string `yaml:"eol,omitempty"` 41 | } 42 | 43 | type License struct { 44 | Module []string `yaml:"module,omitempty"` 45 | Content []string `yaml:"content,omitempty"` 46 | } 47 | 48 | type Dependencies struct { 49 | BuildRequires map[string][]string `yaml:"buildrequires,omitempty,omitempty"` 50 | Requires map[string][]string `yaml:"requires,omitempty,omitempty"` 51 | } 52 | 53 | type References struct { 54 | Community string `yaml:"community,omitempty"` 55 | Documentation string `yaml:"documentation,omitempty"` 56 | Tracker string `yaml:"tracker,omitempty"` 57 | } 58 | 59 | type Profile struct { 60 | Description string `yaml:"description,omitempty"` 61 | Rpms []string `yaml:"rpms,omitempty"` 62 | } 63 | 64 | type API struct { 65 | Rpms []string `yaml:"rpms,omitempty"` 66 | } 67 | 68 | type BuildOptsRPM struct { 69 | Macros string `yaml:"macros,omitempty"` 70 | Whitelist []string `yaml:"whitelist,omitempty"` 71 | } 72 | 73 | type BuildOpts struct { 74 | Rpms *BuildOptsRPM `yaml:"rpms,omitempty"` 75 | Arches []string `yaml:"arches,omitempty"` 76 | } 77 | 78 | type ComponentRPM struct { 79 | Name string `yaml:"name,omitempty"` 80 | Rationale string `yaml:"rationale,omitempty"` 81 | Repository string `yaml:"repository,omitempty"` 82 | Cache string `yaml:"cache,omitempty"` 83 | Ref string `yaml:"ref,omitempty"` 84 | Buildonly bool `yaml:"buildonly,omitempty"` 85 | Buildroot bool `yaml:"buildroot,omitempty"` 86 | SrpmBuildroot bool `yaml:"srpm-buildroot,omitempty"` 87 | Buildorder int `yaml:"buildorder,omitempty"` 88 | Arches []string `yaml:"arches,omitempty"` 89 | Multilib []string `yaml:"multilib,omitempty"` 90 | } 91 | 92 | type ComponentModule struct { 93 | Rationale string `yaml:"rationale,omitempty"` 94 | Repository string `yaml:"repository,omitempty"` 95 | Ref string `yaml:"ref,omitempty"` 96 | Buildorder int `yaml:"buildorder,omitempty"` 97 | } 98 | 99 | type Components struct { 100 | Rpms map[string]*ComponentRPM `yaml:"rpms,omitempty"` 101 | Modules map[string]*ComponentModule `yaml:"modules,omitempty"` 102 | } 103 | 104 | type ArtifactsRPMMap struct { 105 | Name string `yaml:"name,omitempty"` 106 | Epoch int `yaml:"epoch,omitempty"` 107 | Version float64 `yaml:"version,omitempty"` 108 | Release string `yaml:"release,omitempty"` 109 | Arch string `yaml:"arch,omitempty"` 110 | Nevra string `yaml:"nevra,omitempty"` 111 | } 112 | 113 | type Artifacts struct { 114 | Rpms []string `yaml:"rpms,omitempty"` 115 | RpmMap map[string]map[string]*ArtifactsRPMMap `yaml:"rpm-map,omitempty"` 116 | } 117 | 118 | type Data struct { 119 | Name string `yaml:"name,omitempty"` 120 | Stream string `yaml:"stream,omitempty"` 121 | Version string `yaml:"version,omitempty"` 122 | StaticContext bool `yaml:"static_context,omitempty"` 123 | Context string `yaml:"context,omitempty"` 124 | Arch string `yaml:"arch,omitempty"` 125 | Summary string `yaml:"summary,omitempty"` 126 | Description string `yaml:"description,omitempty"` 127 | ServiceLevels map[ServiceLevelType]*ServiceLevel `yaml:"servicelevels,omitempty"` 128 | License *License `yaml:"license,omitempty"` 129 | Xmd map[string]map[string]string `yaml:"xmd,omitempty"` 130 | Dependencies []*Dependencies `yaml:"dependencies,omitempty"` 131 | References *References `yaml:"references,omitempty"` 132 | Profiles map[string]*Profile `yaml:"profiles,omitempty"` 133 | Profile map[string]*Profile `yaml:"profile,omitempty"` 134 | API *API `yaml:"api,omitempty"` 135 | Filter *API `yaml:"filter,omitempty"` 136 | BuildOpts *BuildOpts `yaml:"buildopts,omitempty"` 137 | Components *Components `yaml:"components,omitempty"` 138 | Artifacts *Artifacts `yaml:"artifacts,omitempty"` 139 | } 140 | 141 | type ModuleMd struct { 142 | Document string `yaml:"document,omitempty"` 143 | Version int `yaml:"version,omitempty"` 144 | Data *Data `yaml:"data,omitempty"` 145 | } 146 | 147 | type DetectVersionDocument struct { 148 | Document string `yaml:"document,omitempty"` 149 | Version int `yaml:"version,omitempty"` 150 | } 151 | 152 | type DefaultsData struct { 153 | Module string `yaml:"module,omitempty"` 154 | Stream string `yaml:"stream,omitempty"` 155 | Profiles map[string][]string `yaml:"profiles,omitempty"` 156 | } 157 | 158 | type Defaults struct { 159 | Document string `yaml:"document,omitempty"` 160 | Version int `yaml:"version,omitempty"` 161 | Data *DefaultsData `yaml:"data,omitempty"` 162 | } 163 | 164 | type NotBackwardsCompatibleModuleMd struct { 165 | V2 *ModuleMd 166 | V3 *V3 167 | } 168 | 169 | func Parse(input []byte) (*NotBackwardsCompatibleModuleMd, error) { 170 | var detect DetectVersionDocument 171 | err := yaml.Unmarshal(input, &detect) 172 | if err != nil { 173 | return nil, fmt.Errorf("error detecting document version: %s", err) 174 | } 175 | 176 | var ret NotBackwardsCompatibleModuleMd 177 | 178 | if detect.Version == 2 { 179 | var v2 ModuleMd 180 | err = yaml.Unmarshal(input, &v2) 181 | if err != nil { 182 | return nil, fmt.Errorf("error parsing modulemd: %s", err) 183 | } 184 | ret.V2 = &v2 185 | } else if detect.Version == 3 { 186 | var v3 V3 187 | err = yaml.Unmarshal(input, &v3) 188 | if err != nil { 189 | return nil, fmt.Errorf("error parsing modulemd: %s", err) 190 | } 191 | ret.V3 = &v3 192 | } 193 | 194 | return &ret, nil 195 | } 196 | 197 | func (m *NotBackwardsCompatibleModuleMd) Marshal(fs billy.Filesystem, path string) error { 198 | var bts []byte 199 | 200 | var err error 201 | if m.V2 != nil { 202 | bts, err = yaml.Marshal(m.V2) 203 | } 204 | if m.V3 != nil { 205 | bts, err = yaml.Marshal(m.V3) 206 | } 207 | if err != nil { 208 | return err 209 | } 210 | 211 | _ = fs.Remove(path) 212 | f, err := fs.Create(path) 213 | if err != nil { 214 | return err 215 | } 216 | _, err = f.Write(bts) 217 | if err != nil { 218 | return err 219 | } 220 | _ = f.Close() 221 | 222 | return nil 223 | } 224 | -------------------------------------------------------------------------------- /modulemd/v3.go: -------------------------------------------------------------------------------- 1 | package modulemd 2 | 3 | type V3 struct { 4 | Document string `yaml:"document,omitempty"` 5 | Version int `yaml:"version,omitempty"` 6 | Data *V3Data `yaml:"data,omitempty"` 7 | } 8 | 9 | type Configurations struct { 10 | Context string `yaml:"context,omitempty"` 11 | Platform string `yaml:"platform,omitempty"` 12 | BuildRequires map[string][]string `yaml:"buildrequires,omitempty"` 13 | Requires map[string][]string `yaml:"requires,omitempty"` 14 | BuildOpts *BuildOpts `yaml:"buildopts,omitempty"` 15 | } 16 | 17 | type V3Data struct { 18 | Name string `yaml:"name,omitempty"` 19 | Stream string `yaml:"stream,omitempty"` 20 | Summary string `yaml:"summary,omitempty"` 21 | Description string `yaml:"description,omitempty"` 22 | License []string `yaml:"license,omitempty"` 23 | Xmd map[string]map[string]string `yaml:"xmd,omitempty"` 24 | Configurations []*Configurations `yaml:"configurations,omitempty"` 25 | References *References `yaml:"references,omitempty"` 26 | Profiles map[string]*Profile `yaml:"profiles,omitempty"` 27 | Profile map[string]*Profile `yaml:"profile,omitempty"` 28 | API *API `yaml:"api,omitempty"` 29 | Filter *API `yaml:"filter,omitempty"` 30 | Demodularized *API `yaml:"demodularized,omitempty"` 31 | Components *Components `yaml:"components,omitempty"` 32 | } 33 | -------------------------------------------------------------------------------- /pb/response.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.27.1 4 | // protoc v3.19.3 5 | // source: response.proto 6 | 7 | package srpmprocpb 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type VersionRelease struct { 24 | state protoimpl.MessageState 25 | sizeCache protoimpl.SizeCache 26 | unknownFields protoimpl.UnknownFields 27 | 28 | Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` 29 | Release string `protobuf:"bytes,2,opt,name=release,proto3" json:"release,omitempty"` 30 | } 31 | 32 | func (x *VersionRelease) Reset() { 33 | *x = VersionRelease{} 34 | if protoimpl.UnsafeEnabled { 35 | mi := &file_response_proto_msgTypes[0] 36 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 37 | ms.StoreMessageInfo(mi) 38 | } 39 | } 40 | 41 | func (x *VersionRelease) String() string { 42 | return protoimpl.X.MessageStringOf(x) 43 | } 44 | 45 | func (*VersionRelease) ProtoMessage() {} 46 | 47 | func (x *VersionRelease) ProtoReflect() protoreflect.Message { 48 | mi := &file_response_proto_msgTypes[0] 49 | if protoimpl.UnsafeEnabled && x != nil { 50 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 51 | if ms.LoadMessageInfo() == nil { 52 | ms.StoreMessageInfo(mi) 53 | } 54 | return ms 55 | } 56 | return mi.MessageOf(x) 57 | } 58 | 59 | // Deprecated: Use VersionRelease.ProtoReflect.Descriptor instead. 60 | func (*VersionRelease) Descriptor() ([]byte, []int) { 61 | return file_response_proto_rawDescGZIP(), []int{0} 62 | } 63 | 64 | func (x *VersionRelease) GetVersion() string { 65 | if x != nil { 66 | return x.Version 67 | } 68 | return "" 69 | } 70 | 71 | func (x *VersionRelease) GetRelease() string { 72 | if x != nil { 73 | return x.Release 74 | } 75 | return "" 76 | } 77 | 78 | type ProcessResponse struct { 79 | state protoimpl.MessageState 80 | sizeCache protoimpl.SizeCache 81 | unknownFields protoimpl.UnknownFields 82 | 83 | BranchCommits map[string]string `protobuf:"bytes,1,rep,name=branch_commits,json=branchCommits,proto3" json:"branch_commits,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` 84 | BranchVersions map[string]*VersionRelease `protobuf:"bytes,2,rep,name=branch_versions,json=branchVersions,proto3" json:"branch_versions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` 85 | } 86 | 87 | func (x *ProcessResponse) Reset() { 88 | *x = ProcessResponse{} 89 | if protoimpl.UnsafeEnabled { 90 | mi := &file_response_proto_msgTypes[1] 91 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 92 | ms.StoreMessageInfo(mi) 93 | } 94 | } 95 | 96 | func (x *ProcessResponse) String() string { 97 | return protoimpl.X.MessageStringOf(x) 98 | } 99 | 100 | func (*ProcessResponse) ProtoMessage() {} 101 | 102 | func (x *ProcessResponse) ProtoReflect() protoreflect.Message { 103 | mi := &file_response_proto_msgTypes[1] 104 | if protoimpl.UnsafeEnabled && x != nil { 105 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 106 | if ms.LoadMessageInfo() == nil { 107 | ms.StoreMessageInfo(mi) 108 | } 109 | return ms 110 | } 111 | return mi.MessageOf(x) 112 | } 113 | 114 | // Deprecated: Use ProcessResponse.ProtoReflect.Descriptor instead. 115 | func (*ProcessResponse) Descriptor() ([]byte, []int) { 116 | return file_response_proto_rawDescGZIP(), []int{1} 117 | } 118 | 119 | func (x *ProcessResponse) GetBranchCommits() map[string]string { 120 | if x != nil { 121 | return x.BranchCommits 122 | } 123 | return nil 124 | } 125 | 126 | func (x *ProcessResponse) GetBranchVersions() map[string]*VersionRelease { 127 | if x != nil { 128 | return x.BranchVersions 129 | } 130 | return nil 131 | } 132 | 133 | var File_response_proto protoreflect.FileDescriptor 134 | 135 | var file_response_proto_rawDesc = []byte{ 136 | 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 137 | 0x12, 0x08, 0x73, 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x22, 0x44, 0x0a, 0x0e, 0x56, 0x65, 138 | 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 139 | 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 140 | 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 141 | 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 142 | 0x22, 0xdd, 0x02, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 143 | 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x0e, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x5f, 0x63, 144 | 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x73, 145 | 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 146 | 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x43, 0x6f, 147 | 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x62, 0x72, 0x61, 0x6e, 148 | 0x63, 0x68, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x56, 0x0a, 0x0f, 0x62, 0x72, 0x61, 149 | 0x6e, 0x63, 0x68, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 150 | 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x73, 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x2e, 0x50, 0x72, 151 | 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x72, 152 | 0x61, 0x6e, 0x63, 0x68, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 153 | 0x79, 0x52, 0x0e, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 154 | 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 155 | 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 156 | 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 157 | 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 158 | 0x02, 0x38, 0x01, 0x1a, 0x5b, 0x0a, 0x13, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x56, 0x65, 0x72, 159 | 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 160 | 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, 161 | 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x73, 0x72, 162 | 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 163 | 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 164 | 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 165 | 0x6f, 0x63, 0x6b, 0x79, 0x2d, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x2f, 0x73, 0x72, 0x70, 0x6d, 0x70, 166 | 0x72, 0x6f, 0x63, 0x2f, 0x70, 0x62, 0x3b, 0x73, 0x72, 0x70, 0x6d, 0x70, 0x72, 0x6f, 0x63, 0x70, 167 | 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 168 | } 169 | 170 | var ( 171 | file_response_proto_rawDescOnce sync.Once 172 | file_response_proto_rawDescData = file_response_proto_rawDesc 173 | ) 174 | 175 | func file_response_proto_rawDescGZIP() []byte { 176 | file_response_proto_rawDescOnce.Do(func() { 177 | file_response_proto_rawDescData = protoimpl.X.CompressGZIP(file_response_proto_rawDescData) 178 | }) 179 | return file_response_proto_rawDescData 180 | } 181 | 182 | var file_response_proto_msgTypes = make([]protoimpl.MessageInfo, 4) 183 | var file_response_proto_goTypes = []interface{}{ 184 | (*VersionRelease)(nil), // 0: srpmproc.VersionRelease 185 | (*ProcessResponse)(nil), // 1: srpmproc.ProcessResponse 186 | nil, // 2: srpmproc.ProcessResponse.BranchCommitsEntry 187 | nil, // 3: srpmproc.ProcessResponse.BranchVersionsEntry 188 | } 189 | var file_response_proto_depIdxs = []int32{ 190 | 2, // 0: srpmproc.ProcessResponse.branch_commits:type_name -> srpmproc.ProcessResponse.BranchCommitsEntry 191 | 3, // 1: srpmproc.ProcessResponse.branch_versions:type_name -> srpmproc.ProcessResponse.BranchVersionsEntry 192 | 0, // 2: srpmproc.ProcessResponse.BranchVersionsEntry.value:type_name -> srpmproc.VersionRelease 193 | 3, // [3:3] is the sub-list for method output_type 194 | 3, // [3:3] is the sub-list for method input_type 195 | 3, // [3:3] is the sub-list for extension type_name 196 | 3, // [3:3] is the sub-list for extension extendee 197 | 0, // [0:3] is the sub-list for field type_name 198 | } 199 | 200 | func init() { file_response_proto_init() } 201 | func file_response_proto_init() { 202 | if File_response_proto != nil { 203 | return 204 | } 205 | if !protoimpl.UnsafeEnabled { 206 | file_response_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 207 | switch v := v.(*VersionRelease); i { 208 | case 0: 209 | return &v.state 210 | case 1: 211 | return &v.sizeCache 212 | case 2: 213 | return &v.unknownFields 214 | default: 215 | return nil 216 | } 217 | } 218 | file_response_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 219 | switch v := v.(*ProcessResponse); i { 220 | case 0: 221 | return &v.state 222 | case 1: 223 | return &v.sizeCache 224 | case 2: 225 | return &v.unknownFields 226 | default: 227 | return nil 228 | } 229 | } 230 | } 231 | type x struct{} 232 | out := protoimpl.TypeBuilder{ 233 | File: protoimpl.DescBuilder{ 234 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 235 | RawDescriptor: file_response_proto_rawDesc, 236 | NumEnums: 0, 237 | NumMessages: 4, 238 | NumExtensions: 0, 239 | NumServices: 0, 240 | }, 241 | GoTypes: file_response_proto_goTypes, 242 | DependencyIndexes: file_response_proto_depIdxs, 243 | MessageInfos: file_response_proto_msgTypes, 244 | }.Build() 245 | File_response_proto = out.File 246 | file_response_proto_rawDesc = nil 247 | file_response_proto_goTypes = nil 248 | file_response_proto_depIdxs = nil 249 | } 250 | -------------------------------------------------------------------------------- /pkg/blob/blob.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package blob 22 | 23 | type Storage interface { 24 | Write(path string, content []byte) error 25 | Read(path string) ([]byte, error) 26 | Exists(path string) (bool, error) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/blob/file/file.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package file 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "os" 27 | "path/filepath" 28 | ) 29 | 30 | type File struct { 31 | path string 32 | } 33 | 34 | func New(path string) *File { 35 | return &File{ 36 | path: path, 37 | } 38 | } 39 | 40 | func (f *File) Write(path string, content []byte) error { 41 | w, err := os.OpenFile(filepath.Join(f.path, path), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o644) 42 | if err != nil { 43 | return fmt.Errorf("could not open file: %v", err) 44 | } 45 | 46 | _, err = w.Write(content) 47 | if err != nil { 48 | return fmt.Errorf("could not write file to file: %v", err) 49 | } 50 | 51 | // Close, just like writing a file. 52 | if err := w.Close(); err != nil { 53 | return fmt.Errorf("could not close file writer to source: %v", err) 54 | } 55 | 56 | return nil 57 | } 58 | 59 | func (f *File) Read(path string) ([]byte, error) { 60 | r, err := os.OpenFile(filepath.Join(f.path, path), os.O_RDONLY, 0o644) 61 | if err != nil { 62 | if os.IsNotExist(err) { 63 | return nil, nil 64 | } 65 | return nil, err 66 | } 67 | 68 | body, err := io.ReadAll(r) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | return body, nil 74 | } 75 | 76 | func (f *File) Exists(path string) (bool, error) { 77 | _, err := os.Stat(filepath.Join(f.path, path)) 78 | if !os.IsNotExist(err) { 79 | if !os.IsExist(err) { 80 | return false, err 81 | } 82 | return true, nil 83 | } 84 | 85 | return false, nil 86 | } 87 | -------------------------------------------------------------------------------- /pkg/blob/gcs/gcs.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package gcs 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "io" 27 | 28 | "cloud.google.com/go/storage" 29 | ) 30 | 31 | type GCS struct { 32 | bucket *storage.BucketHandle 33 | } 34 | 35 | func New(name string) (*GCS, error) { 36 | ctx := context.Background() 37 | client, err := storage.NewClient(ctx) 38 | if err != nil { 39 | return nil, fmt.Errorf("could not create gcloud client: %v", err) 40 | } 41 | 42 | return &GCS{ 43 | bucket: client.Bucket(name), 44 | }, nil 45 | } 46 | 47 | func (g *GCS) Write(path string, content []byte) error { 48 | ctx := context.Background() 49 | obj := g.bucket.Object(path) 50 | w := obj.NewWriter(ctx) 51 | 52 | _, err := w.Write(content) 53 | if err != nil { 54 | return fmt.Errorf("could not write file to gcs: %v", err) 55 | } 56 | 57 | // Close, just like writing a file. 58 | if err := w.Close(); err != nil { 59 | return fmt.Errorf("could not close gcs writer to source: %v", err) 60 | } 61 | 62 | return nil 63 | } 64 | 65 | func (g *GCS) Read(path string) ([]byte, error) { 66 | ctx := context.Background() 67 | obj := g.bucket.Object(path) 68 | 69 | r, err := obj.NewReader(ctx) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | body, err := io.ReadAll(r) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | return body, nil 80 | } 81 | 82 | func (g *GCS) Exists(path string) (bool, error) { 83 | ctx := context.Background() 84 | obj := g.bucket.Object(path) 85 | _, err := obj.NewReader(ctx) 86 | return err == nil, nil 87 | } 88 | -------------------------------------------------------------------------------- /pkg/blob/s3/s3.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package s3 22 | 23 | import ( 24 | "bytes" 25 | "io" 26 | 27 | "github.com/aws/aws-sdk-go/aws" 28 | "github.com/aws/aws-sdk-go/aws/awserr" 29 | "github.com/aws/aws-sdk-go/aws/credentials" 30 | "github.com/aws/aws-sdk-go/aws/session" 31 | "github.com/aws/aws-sdk-go/service/s3" 32 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 33 | "github.com/spf13/viper" 34 | ) 35 | 36 | type S3 struct { 37 | bucket string 38 | uploader *s3manager.Uploader 39 | } 40 | 41 | func New(name string) *S3 { 42 | awsCfg := &aws.Config{} 43 | 44 | if accessKey := viper.GetString("s3-access-key"); accessKey != "" { 45 | awsCfg.Credentials = credentials.NewStaticCredentials(accessKey, viper.GetString("s3-secret-key"), "") 46 | } 47 | 48 | if endpoint := viper.GetString("s3-endpoint"); endpoint != "" { 49 | awsCfg.Endpoint = aws.String(endpoint) 50 | } 51 | 52 | if region := viper.GetString("s3-region"); region != "" { 53 | awsCfg.Region = aws.String(region) 54 | } 55 | 56 | if disableSsl := viper.GetBool("s3-disable-ssl"); disableSsl { 57 | awsCfg.DisableSSL = aws.Bool(true) 58 | } 59 | 60 | if forcePathStyle := viper.GetBool("s3-force-path-style"); forcePathStyle { 61 | awsCfg.S3ForcePathStyle = aws.Bool(true) 62 | } 63 | 64 | sess := session.Must(session.NewSession(awsCfg)) 65 | uploader := s3manager.NewUploader(sess) 66 | 67 | return &S3{ 68 | bucket: name, 69 | uploader: uploader, 70 | } 71 | } 72 | 73 | func (s *S3) Write(path string, content []byte) error { 74 | buf := bytes.NewBuffer(content) 75 | 76 | _, err := s.uploader.Upload(&s3manager.UploadInput{ 77 | Bucket: aws.String(s.bucket), 78 | Key: aws.String(path), 79 | Body: buf, 80 | }) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func (s *S3) Read(path string) ([]byte, error) { 89 | obj, err := s.uploader.S3.GetObject(&s3.GetObjectInput{ 90 | Bucket: aws.String(s.bucket), 91 | Key: aws.String(path), 92 | }) 93 | if err != nil { 94 | s3err, ok := err.(awserr.Error) 95 | if !ok || s3err.Code() != s3.ErrCodeNoSuchKey { 96 | return nil, err 97 | } 98 | 99 | return nil, nil 100 | } 101 | 102 | body, err := io.ReadAll(obj.Body) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | return body, nil 108 | } 109 | 110 | func (s *S3) Exists(path string) (bool, error) { 111 | _, err := s.uploader.S3.GetObject(&s3.GetObjectInput{ 112 | Bucket: aws.String(s.bucket), 113 | Key: aws.String(path), 114 | }) 115 | return err == nil, nil 116 | } 117 | -------------------------------------------------------------------------------- /pkg/data/import.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package data 22 | 23 | import ( 24 | "hash" 25 | 26 | "github.com/go-git/go-git/v5" 27 | ) 28 | 29 | type ImportMode interface { 30 | RetrieveSource(pd *ProcessData) (*ModeData, error) 31 | WriteSource(pd *ProcessData, md *ModeData) error 32 | PostProcess(md *ModeData) error 33 | ImportName(pd *ProcessData, md *ModeData) string 34 | } 35 | 36 | type ModeData struct { 37 | Name string 38 | Repo *git.Repository 39 | Worktree *git.Worktree 40 | FileWrites map[string][]byte 41 | TagBranch string 42 | PushBranch string 43 | Branches []string 44 | SourcesToIgnore []*IgnoredSource 45 | BlobCache map[string][]byte 46 | } 47 | 48 | type IgnoredSource struct { 49 | Name string 50 | HashFunction hash.Hash 51 | Expired bool 52 | } 53 | -------------------------------------------------------------------------------- /pkg/data/process.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package data 22 | 23 | import ( 24 | "log" 25 | 26 | "github.com/go-git/go-billy/v5" 27 | "github.com/go-git/go-git/v5/plumbing/transport" 28 | "github.com/rocky-linux/srpmproc/pkg/blob" 29 | ) 30 | 31 | type FsCreatorFunc func(branch string) (billy.Filesystem, error) 32 | 33 | type ProcessData struct { 34 | RpmLocation string 35 | UpstreamPrefix string 36 | Version int 37 | GitCommitterName string 38 | GitCommitterEmail string 39 | Mode int 40 | ModulePrefix string 41 | ImportBranchPrefix string 42 | BranchPrefix string 43 | SingleTag string 44 | Authenticator transport.AuthMethod 45 | Importer ImportMode 46 | BlobStorage blob.Storage 47 | NoDupMode bool 48 | ModuleMode bool 49 | TmpFsMode string 50 | NoStorageDownload bool 51 | NoStorageUpload bool 52 | ManualCommits []string 53 | ModuleFallbackStream string 54 | BranchSuffix string 55 | StrictBranchMode bool 56 | FsCreator FsCreatorFunc 57 | CdnUrl string 58 | Log *log.Logger 59 | PackageVersion string 60 | PackageRelease string 61 | TaglessMode bool 62 | Cdn string 63 | ModuleBranchNames bool 64 | } 65 | -------------------------------------------------------------------------------- /pkg/data/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package data 22 | 23 | import ( 24 | "crypto/md5" 25 | "crypto/sha1" 26 | "crypto/sha256" 27 | "crypto/sha512" 28 | "encoding/hex" 29 | "fmt" 30 | "hash" 31 | "io" 32 | "os" 33 | "path/filepath" 34 | 35 | "github.com/go-git/go-billy/v5" 36 | ) 37 | 38 | func CopyFromFs(from billy.Filesystem, to billy.Filesystem, path string) error { 39 | read, err := from.ReadDir(path) 40 | if err != nil { 41 | return fmt.Errorf("could not read dir: %v", err) 42 | } 43 | 44 | for _, fi := range read { 45 | fullPath := filepath.Join(path, fi.Name()) 46 | 47 | if fi.IsDir() { 48 | _ = to.MkdirAll(fullPath, 0o755) 49 | err := CopyFromFs(from, to, fullPath) 50 | if err != nil { 51 | return err 52 | } 53 | } else { 54 | _ = to.Remove(fullPath) 55 | 56 | f, err := to.OpenFile(fullPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fi.Mode()) 57 | if err != nil { 58 | return fmt.Errorf("could not open file: %v", err) 59 | } 60 | 61 | oldFile, err := from.Open(fullPath) 62 | if err != nil { 63 | return fmt.Errorf("could not open from file: %v", err) 64 | } 65 | 66 | _, err = io.Copy(f, oldFile) 67 | if err != nil { 68 | return fmt.Errorf("could not copy from oldFile to new: %v", err) 69 | } 70 | } 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func IgnoredContains(a []*IgnoredSource, b string) bool { 77 | for _, val := range a { 78 | if val.Name == b { 79 | return true 80 | } 81 | } 82 | 83 | return false 84 | } 85 | 86 | func StrContains(a []string, b string) bool { 87 | for _, val := range a { 88 | if val == b { 89 | return true 90 | } 91 | } 92 | 93 | return false 94 | } 95 | 96 | // CompareHash checks if content and checksum matches 97 | // returns the hash type if success else nil 98 | func (pd *ProcessData) CompareHash(content []byte, checksum string) hash.Hash { 99 | var hashType hash.Hash 100 | 101 | switch len(checksum) { 102 | case 128: 103 | hashType = sha512.New() 104 | break 105 | case 64: 106 | hashType = sha256.New() 107 | break 108 | case 40: 109 | hashType = sha1.New() 110 | break 111 | case 32: 112 | hashType = md5.New() 113 | break 114 | default: 115 | return nil 116 | } 117 | 118 | hashType.Reset() 119 | _, err := hashType.Write(content) 120 | if err != nil { 121 | return nil 122 | } 123 | 124 | calculated := hex.EncodeToString(hashType.Sum(nil)) 125 | if calculated != checksum { 126 | pd.Log.Printf("wanted checksum %s, but got %s", checksum, calculated) 127 | return nil 128 | } 129 | 130 | return hashType 131 | } 132 | -------------------------------------------------------------------------------- /pkg/directives/add.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package directives 22 | 23 | import ( 24 | "errors" 25 | "fmt" 26 | "io" 27 | "os" 28 | "path/filepath" 29 | 30 | "github.com/go-git/go-git/v5" 31 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 32 | "github.com/rocky-linux/srpmproc/pkg/data" 33 | ) 34 | 35 | // returns right if not empty, else left 36 | func eitherString(left string, right string) string { 37 | if right != "" { 38 | return right 39 | } 40 | 41 | return left 42 | } 43 | 44 | func add(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error { 45 | for _, add := range cfg.Add { 46 | var replacingBytes []byte 47 | var filePath string 48 | 49 | switch addType := add.Source.(type) { 50 | case *srpmprocpb.Add_File: 51 | filePath = checkAddPrefix(eitherString(filepath.Base(addType.File), add.Name)) 52 | 53 | fPatch, err := patchTree.Filesystem.OpenFile(addType.File, os.O_RDONLY, 0o644) 54 | if err != nil { 55 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_FROM:%s", addType.File)) 56 | } 57 | 58 | replacingBytes, err = io.ReadAll(fPatch) 59 | if err != nil { 60 | return errors.New(fmt.Sprintf("COULD_NOT_READ_FROM:%s", addType.File)) 61 | } 62 | break 63 | case *srpmprocpb.Add_Lookaside: 64 | filePath = checkAddPrefix(eitherString(filepath.Base(addType.Lookaside), add.Name)) 65 | var err error 66 | replacingBytes, err = pd.BlobStorage.Read(addType.Lookaside) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | hashFunction := pd.CompareHash(replacingBytes, addType.Lookaside) 72 | if hashFunction == nil { 73 | return errors.New(fmt.Sprintf("LOOKASIDE_HASH_DOES_NOT_MATCH:%s", addType.Lookaside)) 74 | } 75 | 76 | md.SourcesToIgnore = append(md.SourcesToIgnore, &data.IgnoredSource{ 77 | Name: filePath, 78 | HashFunction: hashFunction, 79 | }) 80 | break 81 | } 82 | 83 | f, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) 84 | if err != nil { 85 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_DESTINATION:%s", filePath)) 86 | } 87 | 88 | _, err = f.Write(replacingBytes) 89 | if err != nil { 90 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_DESTIONATION:%s", filePath)) 91 | } 92 | } 93 | 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/directives/delete.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package directives 22 | 23 | import ( 24 | "errors" 25 | "fmt" 26 | 27 | "github.com/go-git/go-git/v5" 28 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 29 | "github.com/rocky-linux/srpmproc/pkg/data" 30 | ) 31 | 32 | func del(cfg *srpmprocpb.Cfg, _ *data.ProcessData, _ *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error { 33 | for _, del := range cfg.Delete { 34 | filePath := del.File 35 | _, err := pushTree.Filesystem.Stat(filePath) 36 | if err != nil { 37 | return errors.New(fmt.Sprintf("FILE_DOES_NOT_EXIST:%s", filePath)) 38 | } 39 | 40 | err = pushTree.Filesystem.Remove(filePath) 41 | if err != nil { 42 | return errors.New(fmt.Sprintf("COULD_NOT_DELETE_FILE:%s", filePath)) 43 | } 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/directives/directives.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package directives 22 | 23 | import ( 24 | "path/filepath" 25 | "strings" 26 | 27 | "github.com/go-git/go-git/v5" 28 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 29 | "github.com/rocky-linux/srpmproc/pkg/data" 30 | ) 31 | 32 | func checkAddPrefix(file string) string { 33 | if strings.HasPrefix(file, "SOURCES/") || 34 | strings.HasPrefix(file, "SPECS/") { 35 | return file 36 | } 37 | 38 | return filepath.Join("SOURCES", file) 39 | } 40 | 41 | func Apply(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) []error { 42 | var errs []error 43 | 44 | directives := []func(*srpmprocpb.Cfg, *data.ProcessData, *data.ModeData, *git.Worktree, *git.Worktree) error{ 45 | replace, 46 | del, 47 | add, 48 | patch, 49 | lookaside, 50 | specChange, 51 | } 52 | 53 | for _, directive := range directives { 54 | err := directive(cfg, pd, md, patchTree, pushTree) 55 | if err != nil { 56 | errs = append(errs, err) 57 | } 58 | } 59 | 60 | if len(errs) > 0 { 61 | return errs 62 | } 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/directives/lookaside.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package directives 22 | 23 | import ( 24 | "archive/tar" 25 | "bytes" 26 | "compress/gzip" 27 | "crypto/sha256" 28 | "errors" 29 | "fmt" 30 | "io" 31 | "os" 32 | "path/filepath" 33 | "time" 34 | 35 | "github.com/go-git/go-git/v5" 36 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 37 | "github.com/rocky-linux/srpmproc/pkg/data" 38 | ) 39 | 40 | func lookaside(cfg *srpmprocpb.Cfg, _ *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error { 41 | for _, directive := range cfg.Lookaside { 42 | var buf bytes.Buffer 43 | writer := tar.NewWriter(&buf) 44 | w := pushTree 45 | if directive.FromPatchTree { 46 | w = patchTree 47 | } 48 | 49 | for _, file := range directive.File { 50 | if directive.Tar && directive.ArchiveName == "" { 51 | return errors.New("TAR_NO_ARCHIVE_NAME") 52 | } 53 | 54 | path := filepath.Join("SOURCES", file) 55 | if directive.FromPatchTree { 56 | path = file 57 | } 58 | 59 | stat, err := w.Filesystem.Stat(path) 60 | if err != nil { 61 | return errors.New(fmt.Sprintf("COULD_NOT_STAT_FILE:%s", path)) 62 | } 63 | 64 | f, err := w.Filesystem.Open(path) 65 | if err != nil { 66 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_FILE:%s", path)) 67 | } 68 | 69 | bts, err := io.ReadAll(f) 70 | if err != nil { 71 | return errors.New(fmt.Sprintf("COULD_NOT_READ_FILE:%s", path)) 72 | } 73 | 74 | if directive.Tar { 75 | hdr := &tar.Header{ 76 | Name: file, 77 | Mode: int64(stat.Mode()), 78 | Size: stat.Size(), 79 | } 80 | 81 | err = writer.WriteHeader(hdr) 82 | if err != nil { 83 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_TAR_HEADER:%s", file)) 84 | } 85 | 86 | _, err = writer.Write(bts) 87 | if err != nil { 88 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_TAR_FILE:%s", file)) 89 | } 90 | } else { 91 | if directive.FromPatchTree { 92 | pushF, err := pushTree.Filesystem.OpenFile(filepath.Join("SOURCES", filepath.Base(file)), os.O_CREATE|os.O_TRUNC|os.O_RDWR, stat.Mode()) 93 | if err != nil { 94 | return errors.New(fmt.Sprintf("COULD_NOT_CREATE_FILE_IN_PUSH_TREE:%s", file)) 95 | } 96 | 97 | _, err = pushF.Write(bts) 98 | if err != nil { 99 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_FILE_IN_PUSH_TREE:%s", file)) 100 | } 101 | } 102 | 103 | md.SourcesToIgnore = append(md.SourcesToIgnore, &data.IgnoredSource{ 104 | Name: filepath.Join("SOURCES", file), 105 | HashFunction: sha256.New(), 106 | }) 107 | } 108 | } 109 | 110 | if directive.Tar { 111 | err := writer.Close() 112 | if err != nil { 113 | return errors.New(fmt.Sprintf("COULD_NOT_CLOSE_TAR:%s", directive.ArchiveName)) 114 | } 115 | 116 | var gbuf bytes.Buffer 117 | gw := gzip.NewWriter(&gbuf) 118 | gw.Name = fmt.Sprintf("%s.tar.gz", directive.ArchiveName) 119 | gw.ModTime = time.Now() 120 | 121 | _, err = gw.Write(buf.Bytes()) 122 | if err != nil { 123 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_GZIP:%s", directive.ArchiveName)) 124 | } 125 | err = gw.Close() 126 | if err != nil { 127 | return errors.New(fmt.Sprintf("COULD_NOT_CLOSE_GZIP:%s", directive.ArchiveName)) 128 | } 129 | 130 | path := filepath.Join("SOURCES", fmt.Sprintf("%s.tar.gz", directive.ArchiveName)) 131 | pushF, err := pushTree.Filesystem.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o644) 132 | if err != nil { 133 | return errors.New(fmt.Sprintf("COULD_NOT_CREATE_TAR_FILE:%s", path)) 134 | } 135 | 136 | _, err = pushF.Write(gbuf.Bytes()) 137 | if err != nil { 138 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_TAR_FILE:%s", path)) 139 | } 140 | 141 | md.SourcesToIgnore = append(md.SourcesToIgnore, &data.IgnoredSource{ 142 | Name: path, 143 | HashFunction: sha256.New(), 144 | }) 145 | } 146 | } 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /pkg/directives/patch.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package directives 22 | 23 | import ( 24 | "bytes" 25 | "errors" 26 | "fmt" 27 | 28 | "github.com/bluekeyes/go-gitdiff/gitdiff" 29 | "github.com/go-git/go-git/v5" 30 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 31 | "github.com/rocky-linux/srpmproc/pkg/data" 32 | ) 33 | 34 | func patch(cfg *srpmprocpb.Cfg, pd *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error { 35 | for _, patch := range cfg.Patch { 36 | patchFile, err := patchTree.Filesystem.Open(patch.File) 37 | pd.Log.Printf("[directives.patch] Parsing File: %s", patchFile.Name()) 38 | if err != nil { 39 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_PATCH_FILE:%s", patch.File)) 40 | } 41 | files, _, err := gitdiff.Parse(patchFile) 42 | 43 | if err != nil { 44 | pd.Log.Printf("could not parse patch file: %v", err) 45 | return errors.New(fmt.Sprintf("COULD_NOT_PARSE_PATCH_FILE:%s", patch.File)) 46 | } 47 | 48 | for _, patchedFile := range files { 49 | srcPath := patchedFile.NewName 50 | if !patch.Strict { 51 | srcPath = checkAddPrefix(patchedFile.NewName) 52 | } 53 | var output bytes.Buffer 54 | if !patchedFile.IsDelete && !patchedFile.IsNew { 55 | patchSubjectFile, err := pushTree.Filesystem.Open(srcPath) 56 | if err != nil { 57 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_PATCH_SUBJECT:%s", srcPath)) 58 | } 59 | 60 | err = gitdiff.Apply(&output, patchSubjectFile, patchedFile) 61 | if err != nil { 62 | pd.Log.Printf("[directives.patch] could not apply patch: \"%v\" on \"%s\" from \"%s\"", err, srcPath, patchSubjectFile.Name()) 63 | return errors.New(fmt.Sprintf("COULD_NOT_APPLY_PATCH_WITH_SUBJECT:%s", srcPath)) 64 | } 65 | } 66 | 67 | oldName := patchedFile.OldName 68 | if !patch.Strict { 69 | oldName = checkAddPrefix(patchedFile.OldName) 70 | } 71 | _ = pushTree.Filesystem.Remove(oldName) 72 | _ = pushTree.Filesystem.Remove(srcPath) 73 | 74 | if patchedFile.IsNew { 75 | newFile, err := pushTree.Filesystem.Create(srcPath) 76 | if err != nil { 77 | return errors.New(fmt.Sprintf("COULD_NOT_CREATE_NEW_FILE:%s", srcPath)) 78 | } 79 | err = gitdiff.Apply(&output, newFile, patchedFile) 80 | if err != nil { 81 | return errors.New(fmt.Sprintf("COULD_NOT_APPLY_PATCH_TO_NEW_FILE:%s", srcPath)) 82 | } 83 | _, err = newFile.Write(output.Bytes()) 84 | if err != nil { 85 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_TO_NEW_FILE:%s", srcPath)) 86 | } 87 | _, err = pushTree.Add(srcPath) 88 | if err != nil { 89 | return errors.New(fmt.Sprintf("COULD_NOT_ADD_NEW_FILE_TO_GIT:%s", srcPath)) 90 | } 91 | } else if !patchedFile.IsDelete { 92 | newFile, err := pushTree.Filesystem.Create(srcPath) 93 | if err != nil { 94 | return errors.New(fmt.Sprintf("COULD_NOT_CREATE_POST_PATCH_FILE:%s", srcPath)) 95 | } 96 | _, err = newFile.Write(output.Bytes()) 97 | if err != nil { 98 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_POST_PATCH_FILE:%s", srcPath)) 99 | } 100 | _, err = pushTree.Add(srcPath) 101 | if err != nil { 102 | return errors.New(fmt.Sprintf("COULD_NOT_ADD_POST_PATCH_FILE_TO_GIT:%s", srcPath)) 103 | } 104 | } else { 105 | _, err = pushTree.Remove(oldName) 106 | if err != nil { 107 | return errors.New(fmt.Sprintf("COULD_NOT_REMOVE_FILE_FROM_GIT:%s", oldName)) 108 | } 109 | } 110 | } 111 | } 112 | 113 | return nil 114 | } 115 | -------------------------------------------------------------------------------- /pkg/directives/replace.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package directives 22 | 23 | import ( 24 | "errors" 25 | "fmt" 26 | "io" 27 | "os" 28 | 29 | "github.com/go-git/go-git/v5" 30 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 31 | "github.com/rocky-linux/srpmproc/pkg/data" 32 | ) 33 | 34 | func replace(cfg *srpmprocpb.Cfg, pd *data.ProcessData, _ *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error { 35 | for _, replace := range cfg.Replace { 36 | filePath := checkAddPrefix(replace.File) 37 | stat, err := pushTree.Filesystem.Stat(filePath) 38 | if replace.File == "" || err != nil { 39 | return errors.New(fmt.Sprintf("INVALID_FILE:%s", filePath)) 40 | } 41 | 42 | err = pushTree.Filesystem.Remove(filePath) 43 | if err != nil { 44 | return errors.New(fmt.Sprintf("COULD_NOT_REMOVE_OLD_FILE:%s", filePath)) 45 | } 46 | 47 | f, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, stat.Mode()) 48 | if err != nil { 49 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_REPLACEMENT:%s", filePath)) 50 | } 51 | 52 | switch replacing := replace.Replacing.(type) { 53 | case *srpmprocpb.Replace_WithFile: 54 | fPatch, err := patchTree.Filesystem.OpenFile(replacing.WithFile, os.O_RDONLY, 0o644) 55 | if err != nil { 56 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_REPLACING:%s", replacing.WithFile)) 57 | } 58 | 59 | replacingBytes, err := io.ReadAll(fPatch) 60 | if err != nil { 61 | return errors.New(fmt.Sprintf("COULD_NOT_READ_REPLACING:%s", replacing.WithFile)) 62 | } 63 | 64 | _, err = f.Write(replacingBytes) 65 | if err != nil { 66 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_REPLACING:%s", replacing.WithFile)) 67 | } 68 | break 69 | case *srpmprocpb.Replace_WithInline: 70 | _, err := f.Write([]byte(replacing.WithInline)) 71 | if err != nil { 72 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_INLINE:%s", filePath)) 73 | } 74 | break 75 | case *srpmprocpb.Replace_WithLookaside: 76 | bts, err := pd.BlobStorage.Read(replacing.WithLookaside) 77 | if err != nil { 78 | return err 79 | } 80 | hasher := pd.CompareHash(bts, replacing.WithLookaside) 81 | if hasher == nil { 82 | return errors.New("LOOKASIDE_FILE_AND_HASH_NOT_MATCHING") 83 | } 84 | 85 | _, err = f.Write(bts) 86 | if err != nil { 87 | return errors.New(fmt.Sprintf("COULD_NOT_WRITE_LOOKASIDE:%s", filePath)) 88 | } 89 | break 90 | } 91 | } 92 | 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /pkg/directives/spec_change.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package directives 22 | 23 | import ( 24 | "errors" 25 | "fmt" 26 | "io" 27 | "math" 28 | "os" 29 | "path/filepath" 30 | "regexp" 31 | "strconv" 32 | "strings" 33 | "time" 34 | 35 | "github.com/go-git/go-git/v5" 36 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 37 | "github.com/rocky-linux/srpmproc/pkg/data" 38 | ) 39 | 40 | const ( 41 | sectionChangelog = "%changelog" 42 | ) 43 | 44 | var sections = []string{"%description", "%prep", "%build", "%install", "%files", "%changelog"} 45 | 46 | type sourcePatchOperationInLoopRequest struct { 47 | cfg *srpmprocpb.Cfg 48 | field string 49 | value *string 50 | longestField int 51 | lastNum *int 52 | in *string 53 | expectedField string 54 | operation srpmprocpb.SpecChange_FileOperation_Type 55 | } 56 | 57 | type sourcePatchOperationAfterLoopRequest struct { 58 | cfg *srpmprocpb.Cfg 59 | inLoopNum int 60 | lastNum *int 61 | longestField int 62 | newLines *[]string 63 | in *string 64 | expectedField string 65 | operation srpmprocpb.SpecChange_FileOperation_Type 66 | } 67 | 68 | func sourcePatchOperationInLoop(req *sourcePatchOperationInLoopRequest) error { 69 | if strings.HasPrefix(req.field, req.expectedField) { 70 | for _, file := range req.cfg.SpecChange.File { 71 | if file.Type != req.operation { 72 | continue 73 | } 74 | 75 | switch file.Mode.(type) { 76 | case *srpmprocpb.SpecChange_FileOperation_Delete: 77 | if file.Name == *req.value { 78 | *req.value = "" 79 | } 80 | break 81 | } 82 | } 83 | if req.field != req.expectedField { 84 | sourceNum, err := strconv.Atoi(strings.Split(req.field, req.expectedField)[1]) 85 | if err != nil { 86 | return errors.New(fmt.Sprintf("INVALID_%s_NUM:%s", strings.ToUpper(req.expectedField), req.field)) 87 | } 88 | *req.lastNum = sourceNum 89 | } 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func sourcePatchOperationAfterLoop(req *sourcePatchOperationAfterLoopRequest) (bool, error) { 96 | if req.inLoopNum == *req.lastNum && *req.in == req.expectedField { 97 | for _, file := range req.cfg.SpecChange.File { 98 | if file.Type != req.operation { 99 | continue 100 | } 101 | 102 | switch file.Mode.(type) { 103 | case *srpmprocpb.SpecChange_FileOperation_Add: 104 | fieldNum := *req.lastNum + 1 105 | field := fmt.Sprintf("%s%d", req.expectedField, fieldNum) 106 | spaces := calculateSpaces(req.longestField, len(field), req.cfg.SpecChange.DisableAutoAlign) 107 | *req.newLines = append(*req.newLines, fmt.Sprintf("%s:%s%s", field, spaces, file.Name)) 108 | 109 | if req.expectedField == "Patch" && file.AddToPrep { 110 | val := fmt.Sprintf("%%patch -P%d", fieldNum) 111 | if file.NPath > 0 { 112 | val = fmt.Sprintf("%s -p%d", val, file.NPath) 113 | } 114 | 115 | req.cfg.SpecChange.Append = append(req.cfg.SpecChange.Append, &srpmprocpb.SpecChange_AppendOperation{ 116 | Field: "%prep", 117 | Value: val, 118 | }) 119 | } 120 | 121 | *req.lastNum++ 122 | break 123 | } 124 | } 125 | *req.in = "" 126 | 127 | return true, nil 128 | } 129 | 130 | return false, nil 131 | } 132 | 133 | func calculateSpaces(longestField int, fieldLength int, disableAutoAlign bool) string { 134 | if disableAutoAlign { 135 | return " " 136 | } 137 | return strings.Repeat(" ", longestField+8-fieldLength) 138 | } 139 | 140 | func searchAndReplaceLine(line string, sar []*srpmprocpb.SpecChange_SearchAndReplaceOperation) string { 141 | for _, searchAndReplace := range sar { 142 | switch searchAndReplace.Identifier.(type) { 143 | case *srpmprocpb.SpecChange_SearchAndReplaceOperation_Any: 144 | line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N)) 145 | break 146 | case *srpmprocpb.SpecChange_SearchAndReplaceOperation_StartsWith: 147 | if strings.HasPrefix(strings.TrimSpace(line), searchAndReplace.Find) { 148 | line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N)) 149 | } 150 | break 151 | case *srpmprocpb.SpecChange_SearchAndReplaceOperation_EndsWith: 152 | if strings.HasSuffix(strings.TrimSpace(line), searchAndReplace.Find) { 153 | line = strings.Replace(line, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N)) 154 | } 155 | break 156 | } 157 | } 158 | 159 | return line 160 | } 161 | 162 | func isNextLineSection(lineNum int, lines []string) bool { 163 | if len(lines)-1 > lineNum { 164 | if strings.HasPrefix(strings.TrimSpace(lines[lineNum+1]), "%") { 165 | return true 166 | } 167 | 168 | return false 169 | } 170 | 171 | return true 172 | } 173 | 174 | func setFASlice(futureAdditions map[int][]string, key int, addition string) { 175 | if futureAdditions[key] == nil { 176 | futureAdditions[key] = []string{} 177 | } 178 | futureAdditions[key] = append(futureAdditions[key], addition) 179 | } 180 | 181 | func strSliceContains(slice []string, str string) bool { 182 | for _, x := range slice { 183 | if str == x { 184 | return true 185 | } 186 | } 187 | 188 | return false 189 | } 190 | 191 | func specChange(cfg *srpmprocpb.Cfg, pd *data.ProcessData, md *data.ModeData, _ *git.Worktree, pushTree *git.Worktree) error { 192 | // no spec change operations present 193 | // skip parsing spec 194 | if cfg.SpecChange == nil { 195 | return nil 196 | } 197 | 198 | specFiles, err := pushTree.Filesystem.ReadDir("SPECS") 199 | if err != nil { 200 | return errors.New("COULD_NOT_READ_SPECS_DIR") 201 | } 202 | 203 | if len(specFiles) != 1 { 204 | return errors.New("ONLY_ONE_SPEC_FILE_IS_SUPPORTED") 205 | } 206 | 207 | filePath := filepath.Join("SPECS", specFiles[0].Name()) 208 | stat, err := pushTree.Filesystem.Stat(filePath) 209 | if err != nil { 210 | return errors.New("COULD_NOT_STAT_SPEC_FILE") 211 | } 212 | 213 | specFile, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDONLY, 0o644) 214 | if err != nil { 215 | return errors.New("COULD_NOT_READ_SPEC_FILE") 216 | } 217 | 218 | specBts, err := io.ReadAll(specFile) 219 | if err != nil { 220 | return errors.New("COULD_NOT_READ_ALL_BYTES") 221 | } 222 | 223 | specStr := string(specBts) 224 | lines := strings.Split(specStr, "\n") 225 | 226 | var newLines []string 227 | futureAdditions := map[int][]string{} 228 | newFieldMemory := map[string]map[string]int{} 229 | lastSourceNum := 0 230 | lastPatchNum := 0 231 | inSection := "" 232 | inField := "" 233 | lastSource := "" 234 | lastPatch := "" 235 | hasPatch := false 236 | 237 | version := "" 238 | importName := strings.Replace(pd.Importer.ImportName(pd, md), md.Name, "1", 1) 239 | importNameSplit := strings.SplitN(importName, "-", 2) 240 | if len(importNameSplit) == 2 { 241 | versionSplit := strings.SplitN(importNameSplit[1], ".el", 2) 242 | if len(versionSplit) == 2 { 243 | version = versionSplit[0] 244 | } else { 245 | versionSplit := strings.SplitN(importNameSplit[1], ".module", 2) 246 | if len(versionSplit) == 2 { 247 | version = versionSplit[0] 248 | } 249 | } 250 | } 251 | 252 | fieldValueRegex := regexp.MustCompile("^[a-zA-Z0-9]+:") 253 | 254 | longestField := 0 255 | for lineNum, line := range lines { 256 | if fieldValueRegex.MatchString(line) { 257 | fieldValue := strings.SplitN(line, ":", 2) 258 | field := strings.TrimSpace(fieldValue[0]) 259 | longestField = int(math.Max(float64(len(field)), float64(longestField))) 260 | 261 | if strings.HasPrefix(field, "Source") { 262 | lastSource = field 263 | } else if strings.HasPrefix(field, "Patch") { 264 | lastPatch = field 265 | hasPatch = true 266 | } else { 267 | for _, nf := range cfg.SpecChange.NewField { 268 | if field == nf.Key { 269 | if newFieldMemory[field] == nil { 270 | newFieldMemory[field] = map[string]int{} 271 | } 272 | newFieldMemory[field][nf.Value] = lineNum 273 | } 274 | } 275 | } 276 | } 277 | } 278 | for _, nf := range cfg.SpecChange.NewField { 279 | if newFieldMemory[nf.Key] == nil { 280 | newFieldMemory[nf.Key] = map[string]int{} 281 | newFieldMemory[nf.Key][nf.Value] = 0 282 | } 283 | } 284 | 285 | for field, nfm := range newFieldMemory { 286 | for value, lineNum := range nfm { 287 | if lineNum != 0 { 288 | newLine := fmt.Sprintf("%s:%s%s", field, calculateSpaces(longestField, len(field), cfg.SpecChange.DisableAutoAlign), value) 289 | setFASlice(futureAdditions, lineNum+1, newLine) 290 | } 291 | } 292 | } 293 | 294 | for lineNum, line := range lines { 295 | inLoopSourceNum := lastSourceNum 296 | inLoopPatchNum := lastPatchNum 297 | prefixLine := strings.TrimSpace(line) 298 | 299 | for i, additions := range futureAdditions { 300 | if lineNum == i { 301 | for _, addition := range additions { 302 | newLines = append(newLines, addition) 303 | } 304 | } 305 | } 306 | 307 | if fieldValueRegex.MatchString(line) { 308 | line = searchAndReplaceLine(line, cfg.SpecChange.SearchAndReplace) 309 | fieldValue := strings.SplitN(line, ":", 2) 310 | field := strings.TrimSpace(fieldValue[0]) 311 | value := strings.TrimSpace(fieldValue[1]) 312 | 313 | if field == lastSource { 314 | inField = "Source" 315 | } else if field == lastPatch { 316 | inField = "Patch" 317 | } 318 | 319 | if field == "Version" && version == "" { 320 | version = value 321 | } 322 | 323 | for _, searchAndReplace := range cfg.SpecChange.SearchAndReplace { 324 | switch identifier := searchAndReplace.Identifier.(type) { 325 | case *srpmprocpb.SpecChange_SearchAndReplaceOperation_Field: 326 | if field == identifier.Field { 327 | value = strings.Replace(value, searchAndReplace.Find, searchAndReplace.Replace, int(searchAndReplace.N)) 328 | } 329 | break 330 | } 331 | } 332 | 333 | for _, appendOp := range cfg.SpecChange.Append { 334 | if field == appendOp.Field { 335 | value = value + appendOp.Value 336 | 337 | if field == "Release" { 338 | version = version + appendOp.Value 339 | } 340 | } 341 | } 342 | 343 | spaces := calculateSpaces(longestField, len(field), cfg.SpecChange.DisableAutoAlign) 344 | 345 | err := sourcePatchOperationInLoop(&sourcePatchOperationInLoopRequest{ 346 | cfg: cfg, 347 | field: field, 348 | value: &value, 349 | lastNum: &lastSourceNum, 350 | longestField: longestField, 351 | in: &inField, 352 | expectedField: "Source", 353 | operation: srpmprocpb.SpecChange_FileOperation_Source, 354 | }) 355 | if err != nil { 356 | return err 357 | } 358 | 359 | err = sourcePatchOperationInLoop(&sourcePatchOperationInLoopRequest{ 360 | cfg: cfg, 361 | field: field, 362 | value: &value, 363 | longestField: longestField, 364 | lastNum: &lastPatchNum, 365 | in: &inField, 366 | expectedField: "Patch", 367 | operation: srpmprocpb.SpecChange_FileOperation_Patch, 368 | }) 369 | if err != nil { 370 | return err 371 | } 372 | 373 | if value != "" { 374 | newLines = append(newLines, fmt.Sprintf("%s:%s%s", field, spaces, value)) 375 | } 376 | } else { 377 | executed, err := sourcePatchOperationAfterLoop(&sourcePatchOperationAfterLoopRequest{ 378 | cfg: cfg, 379 | inLoopNum: inLoopSourceNum, 380 | lastNum: &lastSourceNum, 381 | longestField: longestField, 382 | newLines: &newLines, 383 | expectedField: "Source", 384 | in: &inField, 385 | operation: srpmprocpb.SpecChange_FileOperation_Source, 386 | }) 387 | if err != nil { 388 | return err 389 | } 390 | 391 | if executed && !hasPatch { 392 | newLines = append(newLines, "") 393 | inField = "Patch" 394 | } 395 | 396 | executed, err = sourcePatchOperationAfterLoop(&sourcePatchOperationAfterLoopRequest{ 397 | cfg: cfg, 398 | inLoopNum: inLoopPatchNum, 399 | lastNum: &lastPatchNum, 400 | longestField: longestField, 401 | newLines: &newLines, 402 | expectedField: "Patch", 403 | in: &inField, 404 | operation: srpmprocpb.SpecChange_FileOperation_Patch, 405 | }) 406 | if err != nil { 407 | return err 408 | } 409 | 410 | if executed { 411 | var innerNewLines []string 412 | for field, nfm := range newFieldMemory { 413 | for value, ln := range nfm { 414 | newLine := fmt.Sprintf("%s:%s%s", field, calculateSpaces(longestField, len(field), cfg.SpecChange.DisableAutoAlign), value) 415 | if ln == 0 { 416 | if isNextLineSection(lineNum, lines) { 417 | innerNewLines = append(innerNewLines, newLine) 418 | } 419 | } 420 | } 421 | } 422 | if len(innerNewLines) > 0 { 423 | newLines = append(newLines, "") 424 | for _, il := range innerNewLines { 425 | newLines = append(newLines, il) 426 | } 427 | } 428 | } 429 | 430 | if executed && !strings.Contains(specStr, "%changelog") { 431 | newLines = append(newLines, "") 432 | newLines = append(newLines, "%changelog") 433 | inSection = sectionChangelog 434 | } 435 | 436 | if inSection == sectionChangelog { 437 | now := time.Now().Format("Mon Jan 02 2006") 438 | for _, changelog := range cfg.SpecChange.Changelog { 439 | newLines = append(newLines, fmt.Sprintf("* %s %s <%s> - %s", now, changelog.AuthorName, changelog.AuthorEmail, version)) 440 | for _, msg := range changelog.Message { 441 | newLines = append(newLines, fmt.Sprintf("- %s", msg)) 442 | } 443 | newLines = append(newLines, "") 444 | } 445 | inSection = "" 446 | } else { 447 | line = searchAndReplaceLine(line, cfg.SpecChange.SearchAndReplace) 448 | } 449 | 450 | if strings.HasPrefix(prefixLine, "%") { 451 | inSection = prefixLine 452 | 453 | for _, appendOp := range cfg.SpecChange.Append { 454 | if inSection == appendOp.Field { 455 | insertedLine := 0 456 | for i, x := range lines[lineNum+1:] { 457 | if strSliceContains(sections, strings.TrimSpace(x)) { 458 | insertedLine = lineNum + i 459 | setFASlice(futureAdditions, insertedLine, appendOp.Value) 460 | break 461 | } 462 | } 463 | if insertedLine == 0 { 464 | for i, x := range lines[lineNum+1:] { 465 | if strings.TrimSpace(x) == "" { 466 | insertedLine = lineNum + i + 2 467 | setFASlice(futureAdditions, insertedLine, appendOp.Value) 468 | break 469 | } 470 | } 471 | } 472 | } 473 | } 474 | } 475 | 476 | newLines = append(newLines, line) 477 | } 478 | } 479 | 480 | err = pushTree.Filesystem.Remove(filePath) 481 | if err != nil { 482 | return errors.New(fmt.Sprintf("COULD_NOT_REMOVE_OLD_SPEC_FILE:%s", filePath)) 483 | } 484 | 485 | f, err := pushTree.Filesystem.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, stat.Mode()) 486 | if err != nil { 487 | return errors.New(fmt.Sprintf("COULD_NOT_OPEN_REPLACEMENT_SPEC_FILE:%s", filePath)) 488 | } 489 | 490 | _, err = f.Write([]byte(strings.Join(newLines, "\n"))) 491 | if err != nil { 492 | return errors.New("COULD_NOT_WRITE_NEW_SPEC_FILE") 493 | } 494 | 495 | return nil 496 | } 497 | -------------------------------------------------------------------------------- /pkg/misc/regex.go: -------------------------------------------------------------------------------- 1 | package misc 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/rocky-linux/srpmproc/pkg/data" 10 | ) 11 | 12 | func GetTagImportRegex(pd *data.ProcessData) *regexp.Regexp { 13 | branchRegex := regexp.QuoteMeta(fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix)) 14 | if !pd.StrictBranchMode { 15 | branchRegex += "(?:.+|)" 16 | } else { 17 | branchRegex += "(?:-stream-.+|)" 18 | } 19 | 20 | initialVerRegex := regexp.QuoteMeta(filepath.Base(pd.RpmLocation)) + "-" 21 | if pd.PackageVersion != "" { 22 | initialVerRegex += regexp.QuoteMeta(pd.PackageVersion) + "-" 23 | } else { 24 | initialVerRegex += ".+-" 25 | } 26 | if pd.PackageRelease != "" { 27 | initialVerRegex += regexp.QuoteMeta(pd.PackageRelease) 28 | } else { 29 | initialVerRegex += ".+" 30 | } 31 | 32 | regex := fmt.Sprintf("(?i)refs/tags/(imports/(%s)/(%s))", branchRegex, initialVerRegex) 33 | 34 | return regexp.MustCompile(regex) 35 | } 36 | 37 | // Given a git reference in tagless mode (like "refs/heads/c9s", or "refs/heads/stream-httpd-2.4-rhel-9.1.0"), determine 38 | // if we are ok with importing that reference. We are looking for the traditional pattern, like "c9s", and also the 39 | // modular "stream---rhel- branch pattern as well 40 | func TaglessRefOk(tag string, pd *data.ProcessData) bool { 41 | // First case is very easy: if we are exactly "refs/heads/" , then this is def. a branch we should import 42 | if tag == fmt.Sprintf("refs/heads/%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) { 43 | return true 44 | } 45 | 46 | // Less easy: if a modular branch is present (starts w/ "stream-"), we need to check if it's part of our major version, and return true if it is 47 | // (major version means we look for the text "rhel-X." in the branch name, like "rhel-9.1.0") 48 | if strings.HasPrefix(tag, "refs/heads/stream-") && strings.Contains(tag, fmt.Sprintf("rhel-%d.", pd.Version)) { 49 | return true 50 | } 51 | 52 | return false 53 | } 54 | -------------------------------------------------------------------------------- /pkg/modes/git.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package modes 22 | 23 | import ( 24 | "bytes" 25 | "fmt" 26 | "io" 27 | "log" 28 | "net/http" 29 | "path/filepath" 30 | "sort" 31 | "strings" 32 | "text/template" 33 | "time" 34 | 35 | "github.com/go-git/go-git/v5/plumbing/transport" 36 | "github.com/rocky-linux/srpmproc/pkg/misc" 37 | 38 | "github.com/go-git/go-billy/v5/memfs" 39 | "github.com/go-git/go-git/v5" 40 | "github.com/go-git/go-git/v5/config" 41 | "github.com/go-git/go-git/v5/plumbing" 42 | "github.com/go-git/go-git/v5/plumbing/object" 43 | "github.com/go-git/go-git/v5/storage/memory" 44 | "github.com/rocky-linux/srpmproc/pkg/data" 45 | ) 46 | 47 | type remoteTarget struct { 48 | remote string 49 | when time.Time 50 | } 51 | 52 | // Struct to define the possible template values ( {{.Value}} in CDN URL strings: 53 | type Lookaside struct { 54 | Name string 55 | Branch string 56 | Hash string 57 | Hashtype string 58 | Filename string 59 | } 60 | 61 | type remoteTargetSlice []remoteTarget 62 | 63 | func (p remoteTargetSlice) Len() int { 64 | return len(p) 65 | } 66 | 67 | func (p remoteTargetSlice) Less(i, j int) bool { 68 | return p[i].when.Before(p[j].when) 69 | } 70 | 71 | func (p remoteTargetSlice) Swap(i, j int) { 72 | p[i], p[j] = p[j], p[i] 73 | } 74 | 75 | type GitMode struct{} 76 | 77 | func (g *GitMode) RetrieveSource(pd *data.ProcessData) (*data.ModeData, error) { 78 | repo, err := git.Init(memory.NewStorage(), memfs.New()) 79 | if err != nil { 80 | return nil, fmt.Errorf("could not init git Repo: %v", err) 81 | } 82 | 83 | w, err := repo.Worktree() 84 | if err != nil { 85 | return nil, fmt.Errorf("could not get Worktree: %v", err) 86 | } 87 | 88 | refspec := config.RefSpec("+refs/heads/*:refs/remotes/*") 89 | remote, err := repo.CreateRemote(&config.RemoteConfig{ 90 | Name: "upstream", 91 | URLs: []string{fmt.Sprintf("%s.git", pd.RpmLocation)}, 92 | Fetch: []config.RefSpec{refspec}, 93 | }) 94 | if err != nil { 95 | return nil, fmt.Errorf("could not create remote: %v", err) 96 | } 97 | 98 | fetchOpts := &git.FetchOptions{ 99 | Auth: pd.Authenticator, 100 | RefSpecs: []config.RefSpec{refspec}, 101 | Tags: git.AllTags, 102 | Force: true, 103 | } 104 | 105 | err = remote.Fetch(fetchOpts) 106 | if err != nil { 107 | if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { 108 | fetchOpts.Auth = nil 109 | err = remote.Fetch(fetchOpts) 110 | if err != nil { 111 | return nil, fmt.Errorf("could not fetch upstream: %v", err) 112 | } 113 | } else { 114 | return nil, fmt.Errorf("could not fetch upstream: %v", err) 115 | } 116 | } 117 | 118 | var branches remoteTargetSlice 119 | 120 | latestTags := map[string]*remoteTarget{} 121 | 122 | tagAdd := func(tag *object.Tag) error { 123 | if strings.HasPrefix(tag.Name, fmt.Sprintf("imports/%s%d", pd.ImportBranchPrefix, pd.Version)) { 124 | refSpec := fmt.Sprintf("refs/tags/%s", tag.Name) 125 | if misc.GetTagImportRegex(pd).MatchString(refSpec) { 126 | match := misc.GetTagImportRegex(pd).FindStringSubmatch(refSpec) 127 | 128 | exists := latestTags[match[2]] 129 | if exists != nil && exists.when.After(tag.Tagger.When) { 130 | return nil 131 | } 132 | latestTags[match[2]] = &remoteTarget{ 133 | remote: refSpec, 134 | when: tag.Tagger.When, 135 | } 136 | } 137 | } 138 | return nil 139 | } 140 | 141 | // In case of "tagless mode", we need to get the head ref of the branch instead 142 | // This is a kind of alternative implementation of the above tagAdd assignment 143 | refAdd := func(tag *object.Tag) error { 144 | if misc.TaglessRefOk(tag.Name, pd) { 145 | pd.Log.Printf("Tagless mode: Identified tagless commit for import: %s\n", tag.Name) 146 | refSpec := fmt.Sprintf(tag.Name) 147 | 148 | // We split the string by "/", the branch name we're looking for to pass to latestTags is always last 149 | // (ex: "refs/heads/c9s" ---> we want latestTags[c9s] 150 | tmpRef := strings.Split(refSpec, "/") 151 | tmpBranchName := tmpRef[(len(tmpRef) - 1)] 152 | 153 | latestTags[tmpBranchName] = &remoteTarget{ 154 | remote: refSpec, 155 | when: tag.Tagger.When, 156 | } 157 | } 158 | return nil 159 | } 160 | 161 | tagIter, err := repo.TagObjects() 162 | if err != nil { 163 | return nil, fmt.Errorf("could not get tag objects: %v", err) 164 | } 165 | 166 | // tagless mode means we use "refAdd" (add commit by reference) 167 | // normal mode means we can rely on "tagAdd" (the tag should be present for us in the source repo) 168 | if pd.TaglessMode { 169 | _ = tagIter.ForEach(refAdd) 170 | } else { 171 | _ = tagIter.ForEach(tagAdd) 172 | } 173 | 174 | listOpts := &git.ListOptions{ 175 | Auth: pd.Authenticator, 176 | } 177 | list, err := remote.List(listOpts) 178 | if err != nil { 179 | if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { 180 | listOpts.Auth = nil 181 | list, err = remote.List(listOpts) 182 | if err != nil { 183 | return nil, fmt.Errorf("could not list upstream: %v", err) 184 | } 185 | } else { 186 | return nil, fmt.Errorf("could not list upstream: %v", err) 187 | } 188 | } 189 | 190 | for _, ref := range list { 191 | if ref.Hash().IsZero() { 192 | continue 193 | } 194 | 195 | commit, err := repo.CommitObject(ref.Hash()) 196 | if err != nil { 197 | continue 198 | } 199 | 200 | // Call refAdd instead of tagAdd in the case of TaglessMode enabled 201 | if pd.TaglessMode { 202 | _ = refAdd(&object.Tag{ 203 | Name: string(ref.Name()), 204 | Tagger: commit.Committer, 205 | }) 206 | } else { 207 | _ = tagAdd(&object.Tag{ 208 | Name: strings.TrimPrefix(string(ref.Name()), "refs/tags/"), 209 | Tagger: commit.Committer, 210 | }) 211 | } 212 | 213 | } 214 | 215 | for _, branch := range latestTags { 216 | pd.Log.Printf("tag: %s", strings.TrimPrefix(branch.remote, "refs/tags/")) 217 | branches = append(branches, *branch) 218 | } 219 | sort.Sort(branches) 220 | 221 | var sortedBranches []string 222 | for _, branch := range branches { 223 | sortedBranches = append(sortedBranches, branch.remote) 224 | } 225 | 226 | return &data.ModeData{ 227 | Name: filepath.Base(pd.RpmLocation), 228 | Repo: repo, 229 | Worktree: w, 230 | FileWrites: nil, 231 | Branches: sortedBranches, 232 | }, nil 233 | } 234 | 235 | func (g *GitMode) WriteSource(pd *data.ProcessData, md *data.ModeData) error { 236 | remote, err := md.Repo.Remote("upstream") 237 | 238 | if err != nil && !pd.TaglessMode { 239 | return fmt.Errorf("could not get upstream remote: %v", err) 240 | } 241 | 242 | var refspec config.RefSpec 243 | var branchName string 244 | 245 | // In the case of tagless mode, we already have the transformed repo sitting in the worktree, 246 | // and don't need to perform any checkout or fetch operations 247 | if !pd.TaglessMode { 248 | if strings.HasPrefix(md.TagBranch, "refs/heads") { 249 | refspec = config.RefSpec(fmt.Sprintf("+%s:%s", md.TagBranch, md.TagBranch)) 250 | branchName = strings.TrimPrefix(md.TagBranch, "refs/heads/") 251 | } else { 252 | match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) 253 | branchName = match[2] 254 | refspec = config.RefSpec(fmt.Sprintf("+refs/heads/%s:%s", branchName, md.TagBranch)) 255 | fmt.Println("Found branchname that does not start w/ refs/heads :: ", branchName) 256 | } 257 | pd.Log.Printf("checking out upstream refspec %s", refspec) 258 | 259 | fetchOpts := &git.FetchOptions{ 260 | Auth: pd.Authenticator, 261 | RemoteName: "upstream", 262 | RefSpecs: []config.RefSpec{refspec}, 263 | Tags: git.AllTags, 264 | Force: true, 265 | } 266 | err = remote.Fetch(fetchOpts) 267 | if err != nil && err != git.NoErrAlreadyUpToDate { 268 | if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { 269 | fetchOpts.Auth = nil 270 | err = remote.Fetch(fetchOpts) 271 | if err != nil && err != git.NoErrAlreadyUpToDate { 272 | return fmt.Errorf("could not fetch upstream: %v", err) 273 | } 274 | } else { 275 | return fmt.Errorf("could not fetch upstream: %v", err) 276 | } 277 | } 278 | 279 | err = md.Worktree.Checkout(&git.CheckoutOptions{ 280 | Branch: plumbing.ReferenceName(md.TagBranch), 281 | Force: true, 282 | }) 283 | if err != nil { 284 | return fmt.Errorf("could not checkout source from git: %v", err) 285 | } 286 | 287 | _, err = md.Worktree.Add(".") 288 | if err != nil { 289 | return fmt.Errorf("could not add Worktree: %v", err) 290 | } 291 | } 292 | 293 | if pd.TaglessMode { 294 | branchName = fmt.Sprintf("%s%d%s", pd.ImportBranchPrefix, pd.Version, pd.BranchSuffix) 295 | } 296 | 297 | metadataPath := "" 298 | ls, err := md.Worktree.Filesystem.ReadDir(".") 299 | if err != nil { 300 | return fmt.Errorf("could not read directory: %v", err) 301 | } 302 | for _, f := range ls { 303 | if strings.HasSuffix(f.Name(), ".metadata") { 304 | if metadataPath != "" { 305 | return fmt.Errorf("multiple metadata files found") 306 | } 307 | metadataPath = f.Name() 308 | } 309 | } 310 | if metadataPath == "" { 311 | metadataPath = fmt.Sprintf(".%s.metadata", md.Name) 312 | } 313 | 314 | metadataFile, err := md.Worktree.Filesystem.Open(metadataPath) 315 | if err != nil { 316 | pd.Log.Printf("warn: could not open metadata file, so skipping: %v", err) 317 | return nil 318 | } 319 | 320 | fileBytes, err := io.ReadAll(metadataFile) 321 | if err != nil { 322 | return fmt.Errorf("could not read metadata file: %v", err) 323 | } 324 | 325 | client := &http.Client{ 326 | Transport: &http.Transport{ 327 | DisableCompression: false, 328 | }, 329 | } 330 | fileContent := strings.Split(string(fileBytes), "\n") 331 | for _, line := range fileContent { 332 | if strings.TrimSpace(line) == "" { 333 | continue 334 | } 335 | 336 | lineInfo := strings.SplitN(line, " ", 2) 337 | hash := strings.TrimSpace(lineInfo[0]) 338 | path := strings.TrimSpace(lineInfo[1]) 339 | 340 | var body []byte 341 | 342 | if md.BlobCache[hash] != nil { 343 | body = md.BlobCache[hash] 344 | pd.Log.Printf("retrieving %s from cache", hash) 345 | } else { 346 | fromBlobStorage, err := pd.BlobStorage.Read(hash) 347 | if err != nil { 348 | return err 349 | } 350 | if fromBlobStorage != nil && !pd.NoStorageDownload { 351 | body = fromBlobStorage 352 | pd.Log.Printf("downloading %s from blob storage", hash) 353 | } else { 354 | 355 | url := "" 356 | 357 | // We need to figure out the hashtype for templating purposes: 358 | hashType := "sha512" 359 | switch len(hash) { 360 | case 128: 361 | hashType = "sha512" 362 | case 64: 363 | hashType = "sha256" 364 | case 40: 365 | hashType = "sha1" 366 | case 32: 367 | hashType = "md5" 368 | } 369 | 370 | // need the name of the file without "SOURCES/": 371 | fileName := strings.Split(path, "/")[1] 372 | 373 | // Feed our template info to ProcessUrl and transform to the real values: ( {{.Name}}, {{.Branch}}, {{.Hash}}, {{.Hashtype}}, {{.Filename}} ) 374 | url, hasTemplate := ProcessUrl(pd.CdnUrl, md.Name, branchName, hash, hashType, fileName) 375 | 376 | var req *http.Request 377 | var resp *http.Response 378 | 379 | // Download the --cdn-url given, but *only* if it contains template strings ( {{.Name}} , {{.Hash}} , etc. ) 380 | // Otherwise we need to fall back to the traditional cdn-url patterns 381 | if hasTemplate { 382 | pd.Log.Printf("downloading %s", url) 383 | 384 | req, err := http.NewRequest("GET", url, nil) 385 | if err != nil { 386 | return fmt.Errorf("could not create new http request: %v", err) 387 | } 388 | req.Header.Set("Accept-Encoding", "*") 389 | 390 | resp, err = client.Do(req) 391 | if err != nil { 392 | return fmt.Errorf("could not download dist-git file: %v", err) 393 | } 394 | } 395 | 396 | // Default cdn-url: If we don't have a templated download string, try the default /// pattern: 397 | if resp == nil || resp.StatusCode != http.StatusOK { 398 | url = fmt.Sprintf("%s/%s/%s/%s", pd.CdnUrl, md.Name, branchName, hash) 399 | pd.Log.Printf("Attempting default URL: %s", url) 400 | req, err = http.NewRequest("GET", url, nil) 401 | if err != nil { 402 | return fmt.Errorf("could not create new http request: %v", err) 403 | } 404 | req.Header.Set("Accept-Encoding", "*") 405 | resp, err = client.Do(req) 406 | if err != nil { 407 | return fmt.Errorf("could not download dist-git file: %v", err) 408 | } 409 | } 410 | 411 | // If the default URL fails, we have one more pattern to try. The simple / pattern 412 | // If this one fails, we are truly lost, and have to bail out w/ an error: 413 | if resp == nil || resp.StatusCode != http.StatusOK { 414 | url = fmt.Sprintf("%s/%s", pd.CdnUrl, hash) 415 | pd.Log.Printf("Attempting 2nd fallback URL: %s", url) 416 | req, err = http.NewRequest("GET", url, nil) 417 | if err != nil { 418 | return fmt.Errorf("could not create new http request: %v", err) 419 | } 420 | req.Header.Set("Accept-Encoding", "*") 421 | resp, err = client.Do(req) 422 | if err != nil { 423 | return fmt.Errorf("could not download dist-git file: %v", err) 424 | } 425 | if resp.StatusCode != http.StatusOK { 426 | return fmt.Errorf("could not download dist-git file (status code %d): %v", resp.StatusCode, err) 427 | } 428 | } 429 | 430 | body, err = io.ReadAll(resp.Body) 431 | if err != nil { 432 | return fmt.Errorf("could not read the whole dist-git file: %v", err) 433 | } 434 | err = resp.Body.Close() 435 | if err != nil { 436 | return fmt.Errorf("could not close body handle: %v", err) 437 | } 438 | } 439 | 440 | md.BlobCache[hash] = body 441 | } 442 | 443 | f, err := md.Worktree.Filesystem.Create(path) 444 | if err != nil { 445 | return fmt.Errorf("could not open file pointer: %v", err) 446 | } 447 | 448 | hasher := pd.CompareHash(body, hash) 449 | if hasher == nil { 450 | return fmt.Errorf("checksum in metadata does not match dist-git file") 451 | } 452 | 453 | md.SourcesToIgnore = append(md.SourcesToIgnore, &data.IgnoredSource{ 454 | Name: path, 455 | HashFunction: hasher, 456 | }) 457 | 458 | _, err = f.Write(body) 459 | if err != nil { 460 | return fmt.Errorf("could not copy dist-git file to in-tree: %v", err) 461 | } 462 | _ = f.Close() 463 | } 464 | 465 | return nil 466 | } 467 | 468 | func (g *GitMode) PostProcess(md *data.ModeData) error { 469 | for _, source := range md.SourcesToIgnore { 470 | _, err := md.Worktree.Filesystem.Stat(source.Name) 471 | if err == nil { 472 | err := md.Worktree.Filesystem.Remove(source.Name) 473 | if err != nil { 474 | return fmt.Errorf("could not remove dist-git file: %v", err) 475 | } 476 | } 477 | } 478 | 479 | _, err := md.Worktree.Add(".") 480 | if err != nil { 481 | return fmt.Errorf("could not add git sources: %v", err) 482 | } 483 | 484 | return nil 485 | } 486 | 487 | func (g *GitMode) ImportName(pd *data.ProcessData, md *data.ModeData) string { 488 | if misc.GetTagImportRegex(pd).MatchString(md.TagBranch) { 489 | match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) 490 | return match[3] 491 | } 492 | 493 | return strings.Replace(strings.TrimPrefix(md.TagBranch, "refs/heads/"), "%", "_", -1) 494 | } 495 | 496 | // Given a cdnUrl string as input, return same string, but with substituted 497 | // template values ( {{.Name}} , {{.Hash}}, {{.Filename}}, etc. ) 498 | func ProcessUrl(cdnUrl string, name string, branch string, hash string, hashtype string, filename string) (string, bool) { 499 | tmpUrl := Lookaside{name, branch, hash, hashtype, filename} 500 | 501 | // Return cdnUrl as-is if we don't have any templates ("{{ .Variable }}") to process: 502 | if !(strings.Contains(cdnUrl, "{{") && strings.Contains(cdnUrl, "}}")) { 503 | return cdnUrl, false 504 | } 505 | 506 | // If we run into trouble with our template parsing, we'll just return the cdnUrl, exactly as we found it 507 | tmpl, err := template.New("").Parse(cdnUrl) 508 | if err != nil { 509 | return cdnUrl, false 510 | } 511 | 512 | var result bytes.Buffer 513 | err = tmpl.Execute(&result, tmpUrl) 514 | if err != nil { 515 | log.Fatalf("ERROR: Could not process CDN URL template(s) from URL string: %s\n", cdnUrl) 516 | } 517 | 518 | return result.String(), true 519 | 520 | } 521 | -------------------------------------------------------------------------------- /pkg/rpmutils/regex.go: -------------------------------------------------------------------------------- 1 | package rpmutils 2 | 3 | import "regexp" 4 | 5 | // Nvr is a regular expression that matches a NVR. 6 | var Nvr = regexp.MustCompile("^(\\S+)-([\\w~%.+]+)-(\\w+(?:\\.[\\w~%+]+)+?)(?:\\.rpm)?$") 7 | -------------------------------------------------------------------------------- /pkg/srpmproc/fetch.go: -------------------------------------------------------------------------------- 1 | package srpmproc 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/go-git/go-billy/v5" 13 | "github.com/rocky-linux/srpmproc/pkg/blob" 14 | "github.com/rocky-linux/srpmproc/pkg/data" 15 | ) 16 | 17 | func Fetch(logger io.Writer, cdnUrl string, dir string, fs billy.Filesystem, storage blob.Storage) error { 18 | pd := &data.ProcessData{ 19 | Log: log.New(logger, "", log.LstdFlags), 20 | } 21 | 22 | metadataPath := "" 23 | ls, err := fs.ReadDir(dir) 24 | if err != nil { 25 | return err 26 | } 27 | for _, f := range ls { 28 | if strings.HasSuffix(f.Name(), ".metadata") { 29 | if metadataPath != "" { 30 | return errors.New("multiple metadata files found") 31 | } 32 | metadataPath = filepath.Join(dir, f.Name()) 33 | } 34 | } 35 | if metadataPath == "" { 36 | return errors.New("no metadata file found") 37 | } 38 | 39 | metadataFile, err := fs.Open(metadataPath) 40 | if err != nil { 41 | return fmt.Errorf("could not open metadata file: %v", err) 42 | } 43 | 44 | fileBytes, err := io.ReadAll(metadataFile) 45 | if err != nil { 46 | return fmt.Errorf("could not read metadata file: %v", err) 47 | } 48 | 49 | client := &http.Client{ 50 | Transport: &http.Transport{ 51 | DisableCompression: false, 52 | }, 53 | } 54 | fileContent := strings.Split(string(fileBytes), "\n") 55 | for _, line := range fileContent { 56 | if strings.TrimSpace(line) == "" { 57 | continue 58 | } 59 | 60 | lineInfo := strings.SplitN(line, " ", 2) 61 | hash := strings.TrimSpace(lineInfo[0]) 62 | path := strings.TrimSpace(lineInfo[1]) 63 | 64 | url := fmt.Sprintf("%s/%s", cdnUrl, hash) 65 | if storage != nil { 66 | url = hash 67 | } 68 | pd.Log.Printf("downloading %s", url) 69 | 70 | var body []byte 71 | 72 | if storage != nil { 73 | body, err = storage.Read(hash) 74 | if err != nil { 75 | return fmt.Errorf("could not read blob: %v", err) 76 | } 77 | } else { 78 | req, err := http.NewRequest("GET", url, nil) 79 | if err != nil { 80 | return fmt.Errorf("could not create new http request: %v", err) 81 | } 82 | req.Header.Set("Accept-Encoding", "*") 83 | 84 | resp, err := client.Do(req) 85 | if err != nil { 86 | return fmt.Errorf("could not download dist-git file: %v", err) 87 | } 88 | 89 | body, err = io.ReadAll(resp.Body) 90 | if err != nil { 91 | return fmt.Errorf("could not read the whole dist-git file: %v", err) 92 | } 93 | err = resp.Body.Close() 94 | if err != nil { 95 | return fmt.Errorf("could not close body handle: %v", err) 96 | } 97 | } 98 | 99 | hasher := pd.CompareHash(body, hash) 100 | if hasher == nil { 101 | return fmt.Errorf("checksum in metadata does not match dist-git file") 102 | } 103 | 104 | err = fs.MkdirAll(filepath.Join(dir, filepath.Dir(path)), 0o755) 105 | if err != nil { 106 | return fmt.Errorf("could not create all directories") 107 | } 108 | 109 | f, err := fs.Create(filepath.Join(dir, path)) 110 | if err != nil { 111 | return fmt.Errorf("could not open file pointer: %v", err) 112 | } 113 | 114 | _, err = f.Write(body) 115 | if err != nil { 116 | return fmt.Errorf("could not copy dist-git file to in-tree: %v", err) 117 | } 118 | _ = f.Close() 119 | } 120 | 121 | return nil 122 | } 123 | -------------------------------------------------------------------------------- /pkg/srpmproc/patch.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package srpmproc 22 | 23 | import ( 24 | "fmt" 25 | "io" 26 | "log" 27 | "path/filepath" 28 | "strings" 29 | "time" 30 | 31 | "github.com/go-git/go-git/v5/plumbing/transport" 32 | "github.com/rocky-linux/srpmproc/pkg/misc" 33 | 34 | "github.com/go-git/go-billy/v5" 35 | "github.com/go-git/go-billy/v5/memfs" 36 | "github.com/go-git/go-git/v5" 37 | "github.com/go-git/go-git/v5/config" 38 | "github.com/go-git/go-git/v5/plumbing" 39 | "github.com/go-git/go-git/v5/storage/memory" 40 | "github.com/rocky-linux/srpmproc/modulemd" 41 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 42 | "github.com/rocky-linux/srpmproc/pkg/data" 43 | "github.com/rocky-linux/srpmproc/pkg/directives" 44 | "google.golang.org/protobuf/encoding/prototext" 45 | ) 46 | 47 | func cfgPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error { 48 | // check CFG patches 49 | // use PATCHES directory if it exists otherwise ROCKY/CFG 50 | cfgdir := "PATCHES" 51 | _, err := patchTree.Filesystem.Stat(cfgdir) 52 | if err != nil { 53 | cfgdir = "ROCKY/CFG" 54 | } 55 | _, err = patchTree.Filesystem.Stat(cfgdir) 56 | if err == nil { 57 | // iterate through patches 58 | infos, err := patchTree.Filesystem.ReadDir(cfgdir) 59 | if err != nil { 60 | return fmt.Errorf("could not walk patches: %v", err) 61 | } 62 | 63 | for _, info := range infos { 64 | // can only process .cfg files 65 | if !strings.HasSuffix(info.Name(), ".cfg") { 66 | continue 67 | } 68 | 69 | pd.Log.Printf("applying directive %s", info.Name()) 70 | filePath := filepath.Join(cfgdir, info.Name()) 71 | directive, err := patchTree.Filesystem.Open(filePath) 72 | if err != nil { 73 | return fmt.Errorf("could not open directive file %s: %v", info.Name(), err) 74 | } 75 | directiveBytes, err := io.ReadAll(directive) 76 | if err != nil { 77 | return fmt.Errorf("could not read directive file: %v", err) 78 | } 79 | 80 | var cfg srpmprocpb.Cfg 81 | err = prototext.Unmarshal(directiveBytes, &cfg) 82 | if err != nil { 83 | return fmt.Errorf("could not unmarshal cfg file: %v", err) 84 | } 85 | 86 | errs := directives.Apply(&cfg, pd, md, patchTree, pushTree) 87 | if errs != nil { 88 | fmt.Printf("errors: %v\n", errs) 89 | return fmt.Errorf("directives could not be applied") 90 | } 91 | } 92 | } 93 | 94 | return nil 95 | } 96 | 97 | func applyPatches(pd *data.ProcessData, md *data.ModeData, patchTree *git.Worktree, pushTree *git.Worktree) error { 98 | // check if patches exist 99 | cfgdir := "PATCHES" 100 | _, err := patchTree.Filesystem.Stat(cfgdir) 101 | if err != nil { 102 | cfgdir = "ROCKY" 103 | } 104 | _, err = patchTree.Filesystem.Stat(cfgdir) 105 | if err == nil { 106 | err := cfgPatches(pd, md, patchTree, pushTree) 107 | if err != nil { 108 | return err 109 | } 110 | } 111 | 112 | return nil 113 | } 114 | 115 | func executePatchesRpm(pd *data.ProcessData, md *data.ModeData) error { 116 | // fetch patch repository 117 | repo, err := git.Init(memory.NewStorage(), memfs.New()) 118 | if err != nil { 119 | return fmt.Errorf("could not create new dist Repo: %v", err) 120 | } 121 | w, err := repo.Worktree() 122 | if err != nil { 123 | return fmt.Errorf("could not get dist Worktree: %v", err) 124 | } 125 | 126 | remoteUrl := fmt.Sprintf("%s/patch/%s.git", pd.UpstreamPrefix, gitlabify(md.Name)) 127 | refspec := config.RefSpec(fmt.Sprintf("+refs/heads/*:refs/remotes/origin/*")) 128 | 129 | _, err = repo.CreateRemote(&config.RemoteConfig{ 130 | Name: "origin", 131 | URLs: []string{remoteUrl}, 132 | Fetch: []config.RefSpec{refspec}, 133 | }) 134 | if err != nil { 135 | return fmt.Errorf("could not create remote: %v", err) 136 | } 137 | 138 | fetchOptions := &git.FetchOptions{ 139 | Auth: pd.Authenticator, 140 | RemoteName: "origin", 141 | RefSpecs: []config.RefSpec{refspec}, 142 | } 143 | if !strings.HasPrefix(pd.UpstreamPrefix, "http") { 144 | fetchOptions.Auth = pd.Authenticator 145 | } 146 | err = repo.Fetch(fetchOptions) 147 | 148 | refName := plumbing.NewBranchReferenceName(md.PushBranch) 149 | pd.Log.Printf("set reference to ref: %s", refName) 150 | 151 | if err != nil { 152 | if err == transport.ErrInvalidAuthMethod || err == transport.ErrAuthenticationRequired { 153 | fetchOptions.Auth = nil 154 | err = repo.Fetch(fetchOptions) 155 | if err != nil { 156 | // no patches active 157 | log.Println("info: patch repo not found") 158 | return nil 159 | } 160 | } else { 161 | // no patches active 162 | log.Println("info: patch repo not found") 163 | return nil 164 | } 165 | } 166 | 167 | err = w.Checkout(&git.CheckoutOptions{ 168 | Branch: plumbing.NewRemoteReferenceName("origin", "main"), 169 | Force: true, 170 | }) 171 | // common patches found, apply them 172 | if err == nil { 173 | err := applyPatches(pd, md, w, md.Worktree) 174 | if err != nil { 175 | return err 176 | } 177 | } else { 178 | log.Println("info: no common patches found") 179 | } 180 | 181 | err = w.Checkout(&git.CheckoutOptions{ 182 | Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch), 183 | Force: true, 184 | }) 185 | // branch specific patches found, apply them 186 | if err == nil { 187 | err := applyPatches(pd, md, w, md.Worktree) 188 | if err != nil { 189 | return err 190 | } 191 | } else { 192 | log.Println("info: no branch specific patches found") 193 | } 194 | 195 | return nil 196 | } 197 | 198 | func getTipStream(pd *data.ProcessData, module string, pushBranch string, origPushBranch string, tries int) (string, error) { 199 | repo, err := git.Init(memory.NewStorage(), memfs.New()) 200 | if err != nil { 201 | return "", fmt.Errorf("could not init git Repo: %v", err) 202 | } 203 | 204 | remoteUrl := fmt.Sprintf("%s/rpms/%s.git", pd.UpstreamPrefix, gitlabify(module)) 205 | refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") 206 | remote, err := repo.CreateRemote(&config.RemoteConfig{ 207 | Name: "origin", 208 | URLs: []string{remoteUrl}, 209 | Fetch: []config.RefSpec{refspec}, 210 | }) 211 | if err != nil { 212 | return "", fmt.Errorf("could not create remote: %v", err) 213 | } 214 | 215 | list, err := remote.List(&git.ListOptions{ 216 | Auth: pd.Authenticator, 217 | }) 218 | if err != nil { 219 | pd.Log.Printf("could not import module: %s", module) 220 | if tries < 3 { 221 | pd.Log.Printf("could not get rpm refs. will retry in 3s. %v", err) 222 | time.Sleep(3 * time.Second) 223 | return getTipStream(pd, module, pushBranch, origPushBranch, tries+1) 224 | } 225 | 226 | return "", fmt.Errorf("could not get rpm refs. import the rpm before the module: %v", err) 227 | } 228 | 229 | var tipHash string 230 | 231 | for _, ref := range list { 232 | if strings.Contains(ref.Name().String(), "-bootstrap") { 233 | continue 234 | } 235 | 236 | prefix := fmt.Sprintf("refs/heads/%s", pushBranch) 237 | 238 | branchVersion := strings.Split(pushBranch, "-") 239 | if len(branchVersion) == 3 { 240 | version := branchVersion[2] 241 | // incompatible pattern: v1,v2 etc. 242 | // example can be found in refs/tags/imports/c8-stream-1.1/subversion-1.10-8030020200519083055.9ce6d490 243 | if strings.HasPrefix(version, "v") { 244 | prefix = strings.Replace(prefix, version, strings.TrimPrefix(version, "v"), 1) 245 | } 246 | } 247 | 248 | if strings.HasPrefix(ref.Name().String(), prefix) { 249 | tipHash = ref.Hash().String() 250 | } 251 | } 252 | 253 | if tipHash == "" { 254 | for _, ref := range list { 255 | if strings.Contains(ref.Name().String(), "-bootstrap") { 256 | continue 257 | } 258 | 259 | prefix := fmt.Sprintf("refs/heads/%s", origPushBranch) 260 | 261 | if strings.HasPrefix(ref.Name().String(), prefix) { 262 | tipHash = ref.Hash().String() 263 | } 264 | } 265 | } 266 | 267 | if tipHash == "" { 268 | for _, ref := range list { 269 | if strings.Contains(ref.Name().String(), "-bootstrap") { 270 | continue 271 | } 272 | 273 | if !strings.Contains(ref.Name().String(), "stream") { 274 | tipHash = ref.Hash().String() 275 | } 276 | } 277 | } 278 | 279 | if tipHash == "" { 280 | for _, ref := range list { 281 | log.Println(pushBranch, ref.Name()) 282 | } 283 | return "", fmt.Errorf("could not find tip hash") 284 | } 285 | 286 | return strings.TrimSpace(tipHash), nil 287 | } 288 | 289 | func patchModuleYaml(pd *data.ProcessData, md *data.ModeData) error { 290 | // special case for platform.yaml 291 | _, err := md.Worktree.Filesystem.Open("platform.yaml") 292 | if err == nil { 293 | return nil 294 | } 295 | 296 | mdTxtPath := "" 297 | var f billy.File 298 | 299 | // tagless mode implies we're looking for CentOS Stream modules, which are generally "SOURCES/NAME.yaml" (copied to SOURCES/ from import) 300 | // if not tagless mode, proceed as usual with SOURCES/modulemd.*.txt 301 | if pd.TaglessMode { 302 | mdTxtPath = fmt.Sprintf("SOURCES/%s.yaml", md.Name) 303 | f, err = md.Worktree.Filesystem.Open(mdTxtPath) 304 | if err != nil { 305 | mdTxtPath = fmt.Sprintf("SOURCES/%s.yml", md.Name) 306 | f, err = md.Worktree.Filesystem.Open(mdTxtPath) 307 | if err != nil { 308 | return fmt.Errorf("could not open modulemd file: %v", err) 309 | } 310 | } 311 | } else { 312 | mdTxtPath = "SOURCES/modulemd.src.txt" 313 | f, err = md.Worktree.Filesystem.Open(mdTxtPath) 314 | if err != nil { 315 | mdTxtPath = "SOURCES/modulemd.txt" 316 | f, err = md.Worktree.Filesystem.Open(mdTxtPath) 317 | if err != nil { 318 | return fmt.Errorf("could not open modulemd file: %v", err) 319 | } 320 | } 321 | } 322 | 323 | content, err := io.ReadAll(f) 324 | if err != nil { 325 | return fmt.Errorf("could not read modulemd file: %v", err) 326 | } 327 | 328 | module, err := modulemd.Parse(content) 329 | if err != nil { 330 | return fmt.Errorf("could not parse modulemd file: %v", err) 331 | } 332 | 333 | // Get stream branch from tag 334 | // (in tagless mode we are trusting the "Stream: " text in the source YAML to be accurate) 335 | if !pd.TaglessMode { 336 | match := misc.GetTagImportRegex(pd).FindStringSubmatch(md.TagBranch) 337 | streamBranch := strings.Split(match[2], "-") 338 | 339 | // Force stream to be the same as stream name in branch 340 | if module.V2 != nil { 341 | module.V2.Data.Stream = streamBranch[len(streamBranch)-1] 342 | } 343 | if module.V3 != nil { 344 | module.V3.Data.Stream = streamBranch[len(streamBranch)-1] 345 | } 346 | } 347 | 348 | var components *modulemd.Components 349 | if module.V2 != nil { 350 | components = module.V2.Data.Components 351 | } 352 | if module.V3 != nil { 353 | components = module.V3.Data.Components 354 | } 355 | 356 | log.Println("This module contains the following rpms:") 357 | for name := range components.Rpms { 358 | pd.Log.Printf("\t- %s", name) 359 | } 360 | 361 | defaultBranch := md.PushBranch 362 | if pd.ModuleFallbackStream != "" { 363 | defaultBranch = fmt.Sprintf("%s%d-stream-%s", pd.BranchPrefix, pd.Version, pd.ModuleFallbackStream) 364 | } 365 | 366 | for name, rpm := range components.Rpms { 367 | var tipHash string 368 | var pushBranch string 369 | 370 | split := strings.Split(rpm.Ref, "-") 371 | // TODO: maybe point to correct release tag? but refer to latest for now, 372 | // we're bootstrapping a new distro for latest RHEL8 anyways. So earlier 373 | // versions are not that important 374 | if strings.HasPrefix(rpm.Ref, "stream-rhel-rhel-") { 375 | pushBranch = defaultBranch 376 | } else if strings.HasPrefix(rpm.Ref, "stream-rhel-") && len(split) > 4 { 377 | repString := fmt.Sprintf("%s%ss-", pd.BranchPrefix, string(split[4][0])) 378 | newString := fmt.Sprintf("%s%s-", pd.BranchPrefix, string(split[4][0])) 379 | pushBranch = strings.Replace(md.PushBranch, repString, newString, 1) 380 | } else if strings.HasPrefix(rpm.Ref, "stream-") && len(split) == 2 { 381 | pushBranch = defaultBranch 382 | } else if strings.HasPrefix(rpm.Ref, "stream-") && len(split) == 3 { 383 | // example: ant 384 | pushBranch = fmt.Sprintf("%s%d-stream-%s", pd.BranchPrefix, pd.Version, split[2]) 385 | } else if strings.HasPrefix(rpm.Ref, "stream-") { 386 | pushBranch = fmt.Sprintf("%s%s-stream-%s", pd.BranchPrefix, string(split[3][0]), split[1]) 387 | } else if strings.HasPrefix(rpm.Ref, "rhel-") { 388 | pushBranch = defaultBranch 389 | } else { 390 | return fmt.Errorf("could not recognize modulemd ref") 391 | } 392 | 393 | tipHash, err = getTipStream(pd, name, pushBranch, md.PushBranch, 0) 394 | if err != nil { 395 | return err 396 | } 397 | if tipHash == "0000000000000000000000000000000000000000" { 398 | pushBranch = defaultBranch 399 | tipHash, err = getTipStream(pd, name, pushBranch, md.PushBranch, 0) 400 | if err != nil { 401 | return err 402 | } 403 | } 404 | 405 | // If we got this far, we're good. This means the repos, branches, and commits exist. 406 | // If this option is on, just use the branch name. 407 | if pd.ModuleBranchNames { 408 | tipHash = md.PushBranch 409 | } 410 | 411 | rpm.Ref = tipHash 412 | } 413 | 414 | rootModule := fmt.Sprintf("%s.yaml", md.Name) 415 | err = module.Marshal(md.Worktree.Filesystem, rootModule) 416 | if err != nil { 417 | return fmt.Errorf("could not marshal root modulemd: %v", err) 418 | } 419 | 420 | _, err = md.Worktree.Add(rootModule) 421 | if err != nil { 422 | return fmt.Errorf("could not add root modulemd: %v", err) 423 | } 424 | 425 | return nil 426 | } 427 | -------------------------------------------------------------------------------- /pkg/srpmproc/process.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 The Srpmproc Authors 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in all 11 | // copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | // SOFTWARE. 20 | 21 | package srpmproc 22 | 23 | import ( 24 | "bufio" 25 | "encoding/hex" 26 | "fmt" 27 | "io" 28 | "log" 29 | "os" 30 | "os/exec" 31 | "os/user" 32 | "path/filepath" 33 | "strings" 34 | "syscall" 35 | "time" 36 | 37 | "github.com/go-git/go-billy/v5" 38 | "github.com/go-git/go-billy/v5/osfs" 39 | "github.com/go-git/go-git/v5/plumbing/format/gitignore" 40 | "github.com/go-git/go-git/v5/plumbing/transport" 41 | "github.com/go-git/go-git/v5/plumbing/transport/http" 42 | "github.com/go-git/go-git/v5/plumbing/transport/ssh" 43 | srpmprocpb "github.com/rocky-linux/srpmproc/pb" 44 | "github.com/rocky-linux/srpmproc/pkg/blob" 45 | "github.com/rocky-linux/srpmproc/pkg/blob/file" 46 | "github.com/rocky-linux/srpmproc/pkg/blob/gcs" 47 | "github.com/rocky-linux/srpmproc/pkg/blob/s3" 48 | "github.com/rocky-linux/srpmproc/pkg/misc" 49 | "github.com/rocky-linux/srpmproc/pkg/modes" 50 | "github.com/rocky-linux/srpmproc/pkg/rpmutils" 51 | 52 | "github.com/go-git/go-billy/v5/memfs" 53 | "github.com/go-git/go-git/v5" 54 | "github.com/go-git/go-git/v5/config" 55 | "github.com/go-git/go-git/v5/plumbing" 56 | "github.com/go-git/go-git/v5/plumbing/object" 57 | "github.com/go-git/go-git/v5/storage/memory" 58 | "github.com/rocky-linux/srpmproc/pkg/data" 59 | "golang.org/x/term" 60 | ) 61 | 62 | const ( 63 | RpmPrefixCentOS = "https://git.centos.org/rpms" 64 | ModulePrefixCentOS = "https://git.centos.org/modules" 65 | RpmPrefixRocky = "https://git.rockylinux.org/staging/rpms" 66 | ModulePrefixRocky = "https://git.rockylinux.org/staging/modules" 67 | UpstreamPrefixRocky = "https://git.rockylinux.org/staging" 68 | ) 69 | 70 | type ProcessDataRequest struct { 71 | // Required 72 | Version int 73 | StorageAddr string 74 | Package string 75 | PackageGitName string 76 | 77 | // Optional 78 | ModuleMode bool 79 | TmpFsMode string 80 | ModulePrefix string 81 | RpmPrefix string 82 | SshKeyLocation string 83 | SshUser string 84 | SshKeyPassword bool 85 | HttpUsername string 86 | HttpPassword string 87 | ManualCommits string 88 | UpstreamPrefix string 89 | GitCommitterName string 90 | GitCommitterEmail string 91 | ImportBranchPrefix string 92 | BranchPrefix string 93 | FsCreator data.FsCreatorFunc 94 | NoDupMode bool 95 | BranchSuffix string 96 | StrictBranchMode bool 97 | ModuleFallbackStream string 98 | NoStorageUpload bool 99 | NoStorageDownload bool 100 | SingleTag string 101 | CdnUrl string 102 | LogWriter io.Writer 103 | 104 | PackageVersion string 105 | PackageRelease string 106 | 107 | TaglessMode bool 108 | Cdn string 109 | 110 | ModuleBranchNames bool 111 | } 112 | 113 | type LookasidePath struct { 114 | Distro string 115 | Url string 116 | } 117 | 118 | func gitlabify(str string) string { 119 | if str == "tree" { 120 | return "treepkg" 121 | } 122 | 123 | return strings.Replace(str, "+", "plus", -1) 124 | } 125 | 126 | // List of distros and their lookaside patterns 127 | // If we find one of these passed as --cdn (ex: "--cdn fedora"), then we override, and assign this URL to be our --cdn-url 128 | func StaticLookasides() []LookasidePath { 129 | centos := LookasidePath{ 130 | Distro: "centos", 131 | Url: "https://git.centos.org/sources/{{.Name}}/{{.Branch}}/{{.Hash}}", 132 | } 133 | centosStream := LookasidePath{ 134 | Distro: "centos-stream", 135 | Url: "https://sources.stream.centos.org/sources/rpms/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}", 136 | } 137 | rocky8 := LookasidePath{ 138 | Distro: "rocky8", 139 | Url: "https://rocky-linux-sources-staging.a1.rockylinux.org/{{.Hash}}", 140 | } 141 | rocky := LookasidePath{ 142 | Distro: "rocky", 143 | Url: "https://sources.build.resf.org/{{.Hash}}", 144 | } 145 | fedora := LookasidePath{ 146 | Distro: "fedora", 147 | Url: "https://src.fedoraproject.org/repo/pkgs/{{.Name}}/{{.Filename}}/{{.Hashtype}}/{{.Hash}}/{{.Filename}}", 148 | } 149 | 150 | return []LookasidePath{centos, centosStream, rocky8, rocky, fedora} 151 | 152 | } 153 | 154 | // Given a "--cdn" entry like "centos", we can search through our struct list of distros, and return the proper lookaside URL 155 | // If we can't find it, we return false and the calling function will error out 156 | func FindDistro(cdn string) (string, bool) { 157 | var cdnUrl = "" 158 | 159 | // Loop through each distro in the static list defined, try to find a match with "--cdn": 160 | for _, distro := range StaticLookasides() { 161 | if distro.Distro == strings.ToLower(cdn) { 162 | cdnUrl = distro.Url 163 | return cdnUrl, true 164 | } 165 | } 166 | return "", false 167 | } 168 | 169 | func NewProcessData(req *ProcessDataRequest) (*data.ProcessData, error) { 170 | // Build the logger to use for the data import 171 | var writer io.Writer = os.Stdout 172 | if req.LogWriter != nil { 173 | writer = req.LogWriter 174 | } 175 | logger := log.New(writer, "", log.LstdFlags) 176 | 177 | // Set defaults 178 | if req.ModulePrefix == "" { 179 | req.ModulePrefix = ModulePrefixCentOS 180 | } 181 | if req.RpmPrefix == "" { 182 | req.RpmPrefix = RpmPrefixCentOS 183 | } 184 | if req.SshUser == "" { 185 | req.SshUser = "git" 186 | } 187 | if req.UpstreamPrefix == "" { 188 | req.UpstreamPrefix = UpstreamPrefixRocky 189 | } 190 | if req.GitCommitterName == "" { 191 | req.GitCommitterName = "rockyautomation" 192 | } 193 | if req.GitCommitterEmail == "" { 194 | req.GitCommitterEmail = "rockyautomation@rockylinux.org" 195 | } 196 | if req.ImportBranchPrefix == "" { 197 | req.ImportBranchPrefix = "c" 198 | } 199 | if req.BranchPrefix == "" { 200 | req.BranchPrefix = "r" 201 | } 202 | if req.CdnUrl == "" { 203 | req.CdnUrl = "https://git.centos.org/sources" 204 | } 205 | 206 | // If a Cdn distro is defined, we try to find a match from StaticLookasides() array of structs 207 | // see if we have a match to --cdn (matching values are things like fedora, centos, rocky8, etc.) 208 | // If we match, then we want to short-circuit the CdnUrl to the assigned distro's one 209 | if req.Cdn != "" { 210 | newCdn, foundDistro := FindDistro(req.Cdn) 211 | 212 | if !foundDistro { 213 | return nil, fmt.Errorf("Error, distro name given as --cdn argument is not valid.") 214 | } 215 | 216 | req.CdnUrl = newCdn 217 | logger.Printf("Discovered --cdn distro: %s . Using override CDN URL Pattern: %s", req.Cdn, req.CdnUrl) 218 | } 219 | 220 | // Validate required 221 | if req.Package == "" { 222 | return nil, fmt.Errorf("package cannot be empty") 223 | } 224 | 225 | // tells srpmproc what the source name actually is 226 | if req.PackageGitName == "" { 227 | req.PackageGitName = req.Package 228 | } 229 | 230 | var importer data.ImportMode 231 | var blobStorage blob.Storage 232 | 233 | if strings.HasPrefix(req.StorageAddr, "gs://") { 234 | var err error 235 | blobStorage, err = gcs.New(strings.Replace(req.StorageAddr, "gs://", "", 1)) 236 | if err != nil { 237 | return nil, err 238 | } 239 | } else if strings.HasPrefix(req.StorageAddr, "s3://") { 240 | blobStorage = s3.New(strings.Replace(req.StorageAddr, "s3://", "", 1)) 241 | } else if strings.HasPrefix(req.StorageAddr, "file://") { 242 | blobStorage = file.New(strings.Replace(req.StorageAddr, "file://", "", 1)) 243 | } else { 244 | return nil, fmt.Errorf("invalid blob storage") 245 | } 246 | 247 | sourceRpmLocation := "" 248 | if req.ModuleMode { 249 | sourceRpmLocation = fmt.Sprintf("%s/%s", req.ModulePrefix, req.PackageGitName) 250 | } else { 251 | sourceRpmLocation = fmt.Sprintf("%s/%s", req.RpmPrefix, req.PackageGitName) 252 | } 253 | importer = &modes.GitMode{} 254 | 255 | lastKeyLocation := req.SshKeyLocation 256 | if lastKeyLocation == "" { 257 | usr, err := user.Current() 258 | if err != nil { 259 | return nil, fmt.Errorf("could not get user: %v", err) 260 | } 261 | lastKeyLocation = filepath.Join(usr.HomeDir, ".ssh/id_rsa") 262 | } 263 | 264 | var authenticator transport.AuthMethod 265 | 266 | var err error 267 | if req.HttpUsername != "" { 268 | authenticator = &http.BasicAuth{ 269 | Username: req.HttpUsername, 270 | Password: req.HttpPassword, 271 | } 272 | } else { 273 | var sshPassword string = "" 274 | if req.SshKeyPassword { 275 | 276 | fmt.Print("Enter SSH key password: ") 277 | sshBytePassword, err := term.ReadPassword(int(syscall.Stdin)) 278 | if err != nil { 279 | return nil, fmt.Errorf("could not read password for ssh key: %v", err) 280 | } 281 | 282 | sshPassword = string(sshBytePassword) 283 | } 284 | 285 | // create ssh key authenticator 286 | authenticator, err = ssh.NewPublicKeysFromFile(req.SshUser, lastKeyLocation, sshPassword) 287 | } 288 | if err != nil { 289 | return nil, fmt.Errorf("could not get git authenticator: %v", err) 290 | } 291 | 292 | fsCreator := func(branch string) (billy.Filesystem, error) { 293 | if req.TmpFsMode != "" { 294 | return osfs.New(""), nil 295 | } 296 | return memfs.New(), nil 297 | } 298 | reqFsCreator := fsCreator 299 | if req.FsCreator != nil { 300 | reqFsCreator = req.FsCreator 301 | } 302 | 303 | if req.TmpFsMode != "" { 304 | logger.Printf("using tmpfs dir: %s", req.TmpFsMode) 305 | fsCreator = func(branch string) (billy.Filesystem, error) { 306 | fs, err := reqFsCreator(branch) 307 | if err != nil { 308 | return nil, err 309 | } 310 | tmpDir := filepath.Join(req.TmpFsMode, branch) 311 | err = fs.MkdirAll(tmpDir, 0o755) 312 | if err != nil { 313 | return nil, fmt.Errorf("could not create tmpfs dir: %v", err) 314 | } 315 | nFs, err := fs.Chroot(tmpDir) 316 | if err != nil { 317 | return nil, err 318 | } 319 | 320 | return nFs, nil 321 | } 322 | } else { 323 | fsCreator = reqFsCreator 324 | } 325 | 326 | var manualCs []string 327 | if strings.TrimSpace(req.ManualCommits) != "" { 328 | manualCs = strings.Split(req.ManualCommits, ",") 329 | } 330 | 331 | return &data.ProcessData{ 332 | Importer: importer, 333 | RpmLocation: sourceRpmLocation, 334 | UpstreamPrefix: req.UpstreamPrefix, 335 | Version: req.Version, 336 | BlobStorage: blobStorage, 337 | GitCommitterName: req.GitCommitterName, 338 | GitCommitterEmail: req.GitCommitterEmail, 339 | ModulePrefix: req.ModulePrefix, 340 | ImportBranchPrefix: req.ImportBranchPrefix, 341 | BranchPrefix: req.BranchPrefix, 342 | SingleTag: req.SingleTag, 343 | Authenticator: authenticator, 344 | NoDupMode: req.NoDupMode, 345 | ModuleMode: req.ModuleMode, 346 | TmpFsMode: req.TmpFsMode, 347 | NoStorageDownload: req.NoStorageDownload, 348 | NoStorageUpload: req.NoStorageUpload, 349 | ManualCommits: manualCs, 350 | ModuleFallbackStream: req.ModuleFallbackStream, 351 | BranchSuffix: req.BranchSuffix, 352 | StrictBranchMode: req.StrictBranchMode, 353 | FsCreator: fsCreator, 354 | CdnUrl: req.CdnUrl, 355 | Log: logger, 356 | PackageVersion: req.PackageVersion, 357 | PackageRelease: req.PackageRelease, 358 | TaglessMode: req.TaglessMode, 359 | Cdn: req.Cdn, 360 | ModuleBranchNames: req.ModuleBranchNames, 361 | }, nil 362 | } 363 | 364 | // ProcessRPM checks the RPM specs and discards any remote files 365 | // This functions also sorts files into directories 366 | // .spec files goes into -> SPECS 367 | // metadata files goes to root 368 | // source files goes into -> SOURCES 369 | // all files that are remote goes into .gitignore 370 | // all ignored files' hash goes into .{Name}.metadata 371 | func ProcessRPM(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { 372 | // if we are using "tagless mode", then we need to jump to a completely different import process: 373 | // Version info needs to be derived from rpmbuild + spec file, not tags 374 | if pd.TaglessMode { 375 | result, err := processRPMTagless(pd) 376 | return result, err 377 | } 378 | 379 | md, err := pd.Importer.RetrieveSource(pd) 380 | if err != nil { 381 | return nil, err 382 | } 383 | md.BlobCache = map[string][]byte{} 384 | 385 | remotePrefix := "rpms" 386 | if pd.ModuleMode { 387 | remotePrefix = "modules" 388 | } 389 | 390 | latestHashForBranch := map[string]string{} 391 | versionForBranch := map[string]*srpmprocpb.VersionRelease{} 392 | 393 | // already uploaded blobs are skipped 394 | var alreadyUploadedBlobs []string 395 | 396 | // if no-dup-mode is enabled then skip already imported versions 397 | var tagIgnoreList []string 398 | if pd.NoDupMode { 399 | repo, err := git.Init(memory.NewStorage(), memfs.New()) 400 | if err != nil { 401 | return nil, fmt.Errorf("could not init git repo: %v", err) 402 | } 403 | remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) 404 | refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") 405 | 406 | remote, err := repo.CreateRemote(&config.RemoteConfig{ 407 | Name: "origin", 408 | URLs: []string{remoteUrl}, 409 | Fetch: []config.RefSpec{refspec}, 410 | }) 411 | if err != nil { 412 | return nil, fmt.Errorf("could not create remote: %v", err) 413 | } 414 | 415 | list, err := remote.List(&git.ListOptions{ 416 | Auth: pd.Authenticator, 417 | }) 418 | 419 | if err != nil { 420 | log.Println("ignoring no-dup-mode") 421 | } else { 422 | for _, ref := range list { 423 | if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") { 424 | continue 425 | } 426 | tagIgnoreList = append(tagIgnoreList, string(ref.Name())) 427 | } 428 | } 429 | } 430 | 431 | sourceRepo := *md.Repo 432 | sourceWorktree := *md.Worktree 433 | 434 | commitPin := map[string]string{} 435 | 436 | if pd.SingleTag != "" { 437 | md.Branches = []string{fmt.Sprintf("refs/tags/%s", pd.SingleTag)} 438 | } else if len(pd.ManualCommits) > 0 { 439 | log.Println("Manual commits were listed for import. Switching to perform a tagless import of these commit(s).") 440 | pd.TaglessMode = true 441 | return processRPMTagless(pd) 442 | } 443 | 444 | // If we have no valid branches to consider, then we'll automatically switch to attempt a tagless import: 445 | if len(md.Branches) == 0 { 446 | log.Println("No valid tags (refs/tags/imports/*) found in repository! Switching to perform a tagless import.") 447 | pd.TaglessMode = true 448 | result, err := processRPMTagless(pd) 449 | return result, err 450 | } 451 | 452 | for _, branch := range md.Branches { 453 | md.Repo = &sourceRepo 454 | md.Worktree = &sourceWorktree 455 | md.TagBranch = branch 456 | for _, source := range md.SourcesToIgnore { 457 | source.Expired = true 458 | } 459 | 460 | var matchString string 461 | if !misc.GetTagImportRegex(pd).MatchString(md.TagBranch) { 462 | if pd.ModuleMode { 463 | prefix := fmt.Sprintf("refs/heads/%s%d", pd.ImportBranchPrefix, pd.Version) 464 | if strings.HasPrefix(md.TagBranch, prefix) { 465 | replace := strings.Replace(md.TagBranch, "refs/heads/", "", 1) 466 | matchString = fmt.Sprintf("refs/tags/imports/%s/%s", replace, filepath.Base(pd.RpmLocation)) 467 | pd.Log.Printf("using match string: %s", matchString) 468 | } 469 | } 470 | if !misc.GetTagImportRegex(pd).MatchString(matchString) { 471 | continue 472 | } 473 | } else { 474 | matchString = md.TagBranch 475 | } 476 | 477 | match := misc.GetTagImportRegex(pd).FindStringSubmatch(matchString) 478 | 479 | md.PushBranch = pd.BranchPrefix + strings.TrimPrefix(match[2], pd.ImportBranchPrefix) 480 | 481 | newTag := "imports/" + pd.BranchPrefix + strings.TrimPrefix(match[1], "imports/"+pd.ImportBranchPrefix) 482 | newTag = strings.Replace(newTag, "%", "_", -1) 483 | 484 | createdFs, err := pd.FsCreator(md.PushBranch) 485 | if err != nil { 486 | return nil, err 487 | } 488 | 489 | // create new Repo for final dist 490 | repo, err := git.Init(memory.NewStorage(), createdFs) 491 | if err != nil { 492 | return nil, fmt.Errorf("could not create new dist Repo: %v", err) 493 | } 494 | w, err := repo.Worktree() 495 | if err != nil { 496 | return nil, fmt.Errorf("could not get dist Worktree: %v", err) 497 | } 498 | 499 | shouldContinue := true 500 | for _, ignoredTag := range tagIgnoreList { 501 | if ignoredTag == "refs/tags/"+newTag { 502 | pd.Log.Printf("skipping %s", ignoredTag) 503 | shouldContinue = false 504 | } 505 | } 506 | if !shouldContinue { 507 | continue 508 | } 509 | 510 | // create a new remote 511 | remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) 512 | pd.Log.Printf("using remote: %s", remoteUrl) 513 | refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch)) 514 | pd.Log.Printf("using refspec: %s", refspec) 515 | 516 | _, err = repo.CreateRemote(&config.RemoteConfig{ 517 | Name: "origin", 518 | URLs: []string{remoteUrl}, 519 | Fetch: []config.RefSpec{refspec}, 520 | }) 521 | if err != nil { 522 | return nil, fmt.Errorf("could not create remote: %v", err) 523 | } 524 | 525 | err = repo.Fetch(&git.FetchOptions{ 526 | RemoteName: "origin", 527 | RefSpecs: []config.RefSpec{refspec}, 528 | Auth: pd.Authenticator, 529 | }) 530 | 531 | refName := plumbing.NewBranchReferenceName(md.PushBranch) 532 | pd.Log.Printf("set reference to ref: %s", refName) 533 | 534 | var hash plumbing.Hash 535 | if commitPin[md.PushBranch] != "" { 536 | hash = plumbing.NewHash(commitPin[md.PushBranch]) 537 | } 538 | 539 | if err != nil { 540 | h := plumbing.NewSymbolicReference(plumbing.HEAD, refName) 541 | if err := repo.Storer.CheckAndSetReference(h, nil); err != nil { 542 | return nil, fmt.Errorf("could not set reference: %v", err) 543 | } 544 | } else { 545 | err = w.Checkout(&git.CheckoutOptions{ 546 | Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch), 547 | Hash: hash, 548 | Force: true, 549 | }) 550 | if err != nil { 551 | return nil, fmt.Errorf("could not checkout: %v", err) 552 | } 553 | } 554 | 555 | err = pd.Importer.WriteSource(pd, md) 556 | if err != nil { 557 | return nil, err 558 | } 559 | 560 | err = data.CopyFromFs(md.Worktree.Filesystem, w.Filesystem, ".") 561 | if err != nil { 562 | return nil, err 563 | } 564 | md.Repo = repo 565 | md.Worktree = w 566 | 567 | if pd.ModuleMode { 568 | err := patchModuleYaml(pd, md) 569 | if err != nil { 570 | return nil, err 571 | } 572 | } else { 573 | err := executePatchesRpm(pd, md) 574 | if err != nil { 575 | return nil, err 576 | } 577 | } 578 | 579 | // get ignored files hash and add to .{Name}.metadata 580 | metadataFile := "" 581 | ls, err := md.Worktree.Filesystem.ReadDir(".") 582 | if err != nil { 583 | return nil, fmt.Errorf("could not read directory: %v", err) 584 | } 585 | for _, f := range ls { 586 | if strings.HasSuffix(f.Name(), ".metadata") { 587 | if metadataFile != "" { 588 | return nil, fmt.Errorf("multiple metadata files found") 589 | } 590 | metadataFile = f.Name() 591 | } 592 | } 593 | if metadataFile == "" { 594 | metadataFile = fmt.Sprintf(".%s.metadata", md.Name) 595 | } 596 | metadata, err := w.Filesystem.Create(metadataFile) 597 | if err != nil { 598 | return nil, fmt.Errorf("could not create metadata file: %v", err) 599 | } 600 | for _, source := range md.SourcesToIgnore { 601 | sourcePath := source.Name 602 | 603 | _, err := w.Filesystem.Stat(sourcePath) 604 | if source.Expired || err != nil { 605 | continue 606 | } 607 | 608 | sourceFile, err := w.Filesystem.Open(sourcePath) 609 | if err != nil { 610 | return nil, fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err) 611 | } 612 | sourceFileBts, err := io.ReadAll(sourceFile) 613 | if err != nil { 614 | return nil, fmt.Errorf("could not read the whole of ignored source file: %v", err) 615 | } 616 | 617 | source.HashFunction.Reset() 618 | _, err = source.HashFunction.Write(sourceFileBts) 619 | if err != nil { 620 | return nil, fmt.Errorf("could not write bytes to hash function: %v", err) 621 | } 622 | checksum := hex.EncodeToString(source.HashFunction.Sum(nil)) 623 | checksumLine := fmt.Sprintf("%s %s\n", checksum, sourcePath) 624 | _, err = metadata.Write([]byte(checksumLine)) 625 | if err != nil { 626 | return nil, fmt.Errorf("could not write to metadata file: %v", err) 627 | } 628 | 629 | if data.StrContains(alreadyUploadedBlobs, checksum) { 630 | continue 631 | } 632 | exists, err := pd.BlobStorage.Exists(checksum) 633 | if err != nil { 634 | return nil, err 635 | } 636 | if !exists && !pd.NoStorageUpload { 637 | err := pd.BlobStorage.Write(checksum, sourceFileBts) 638 | if err != nil { 639 | return nil, err 640 | } 641 | pd.Log.Printf("wrote %s to blob storage", checksum) 642 | } 643 | alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum) 644 | } 645 | 646 | _, err = w.Add(metadataFile) 647 | if err != nil { 648 | return nil, fmt.Errorf("could not add metadata file: %v", err) 649 | } 650 | 651 | lastFilesToAdd := []string{".gitignore", "SPECS"} 652 | for _, f := range lastFilesToAdd { 653 | _, err := w.Filesystem.Stat(f) 654 | if err == nil { 655 | _, err := w.Add(f) 656 | if err != nil { 657 | return nil, fmt.Errorf("could not add %s: %v", f, err) 658 | } 659 | } 660 | } 661 | 662 | nvrMatch := rpmutils.Nvr.FindStringSubmatch(match[3]) 663 | if len(nvrMatch) >= 4 { 664 | versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{ 665 | Version: nvrMatch[2], 666 | Release: nvrMatch[3], 667 | } 668 | } 669 | 670 | if pd.TmpFsMode != "" { 671 | continue 672 | } 673 | 674 | err = pd.Importer.PostProcess(md) 675 | if err != nil { 676 | return nil, err 677 | } 678 | 679 | // show status 680 | status, _ := w.Status() 681 | if !pd.ModuleMode { 682 | if status.IsClean() { 683 | pd.Log.Printf("No changes detected. Our downstream is up to date.") 684 | head, err := repo.Head() 685 | if err != nil { 686 | return nil, fmt.Errorf("error getting HEAD: %v", err) 687 | } 688 | latestHashForBranch[md.PushBranch] = head.Hash().String() 689 | continue 690 | } 691 | } 692 | pd.Log.Printf("successfully processed:\n%s", status) 693 | 694 | statusLines := strings.Split(status.String(), "\n") 695 | for _, line := range statusLines { 696 | trimmed := strings.TrimSpace(line) 697 | if strings.HasPrefix(trimmed, "D") { 698 | path := strings.TrimPrefix(trimmed, "D ") 699 | _, err := w.Remove(path) 700 | if err != nil { 701 | return nil, fmt.Errorf("could not delete extra file %s: %v", path, err) 702 | } 703 | } 704 | } 705 | 706 | var hashes []plumbing.Hash 707 | var pushRefspecs []config.RefSpec 708 | 709 | head, err := repo.Head() 710 | if err != nil { 711 | hashes = nil 712 | pushRefspecs = append(pushRefspecs, "*:*") 713 | } else { 714 | pd.Log.Printf("tip %s", head.String()) 715 | hashes = append(hashes, head.Hash()) 716 | refOrigin := "refs/heads/" + md.PushBranch 717 | pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", refOrigin))) 718 | } 719 | 720 | // we are now finished with the tree and are going to push it to the src Repo 721 | // create import commit 722 | commit, err := w.Commit("import "+pd.Importer.ImportName(pd, md), &git.CommitOptions{ 723 | Author: &object.Signature{ 724 | Name: pd.GitCommitterName, 725 | Email: pd.GitCommitterEmail, 726 | When: time.Now(), 727 | }, 728 | Parents: hashes, 729 | }) 730 | if err != nil { 731 | return nil, fmt.Errorf("could not commit object: %v", err) 732 | } 733 | 734 | obj, err := repo.CommitObject(commit) 735 | if err != nil { 736 | return nil, fmt.Errorf("could not get commit object: %v", err) 737 | } 738 | 739 | pd.Log.Printf("committed:\n%s", obj.String()) 740 | 741 | _, err = repo.CreateTag(newTag, commit, &git.CreateTagOptions{ 742 | Tagger: &object.Signature{ 743 | Name: pd.GitCommitterName, 744 | Email: pd.GitCommitterEmail, 745 | When: time.Now(), 746 | }, 747 | Message: "import " + md.TagBranch + " from " + pd.RpmLocation, 748 | SignKey: nil, 749 | }) 750 | if err != nil { 751 | return nil, fmt.Errorf("could not create tag: %v", err) 752 | } 753 | 754 | pushRefspecs = append(pushRefspecs, config.RefSpec("HEAD:"+plumbing.NewTagReferenceName(newTag))) 755 | 756 | err = repo.Push(&git.PushOptions{ 757 | RemoteName: "origin", 758 | Auth: pd.Authenticator, 759 | RefSpecs: pushRefspecs, 760 | Force: true, 761 | }) 762 | if err != nil { 763 | return nil, fmt.Errorf("could not push to remote: %v", err) 764 | } 765 | 766 | hashString := obj.Hash.String() 767 | latestHashForBranch[md.PushBranch] = hashString 768 | } 769 | 770 | return &srpmprocpb.ProcessResponse{ 771 | BranchCommits: latestHashForBranch, 772 | BranchVersions: versionForBranch, 773 | }, nil 774 | } 775 | 776 | // Process for when we want to import a tagless repo (like from CentOS Stream) 777 | func processRPMTagless(pd *data.ProcessData) (*srpmprocpb.ProcessResponse, error) { 778 | pd.Log.Println("Tagless mode detected, attempting import of latest commit") 779 | 780 | // In tagless mode, we *automatically* set StrictBranchMode to true 781 | // Only the exact branch should be pulled from the source repo 782 | pd.StrictBranchMode = true 783 | 784 | // our return values: a mapping of branches -> commits (1:1) that we're bringing in, 785 | // and a mapping of branches to: version = X, release = Y 786 | latestHashForBranch := map[string]string{} 787 | versionForBranch := map[string]*srpmprocpb.VersionRelease{} 788 | 789 | md, err := pd.Importer.RetrieveSource(pd) 790 | if err != nil { 791 | pd.Log.Println("Error detected in RetrieveSource!") 792 | return nil, err 793 | } 794 | 795 | md.BlobCache = map[string][]byte{} 796 | 797 | // TODO: add tagless module support 798 | remotePrefix := "rpms" 799 | if pd.ModuleMode { 800 | remotePrefix = "modules" 801 | } 802 | 803 | // Set up our remote URL for pushing our repo to 804 | var tagIgnoreList []string 805 | if pd.NoDupMode { 806 | repo, err := git.Init(memory.NewStorage(), memfs.New()) 807 | if err != nil { 808 | return nil, fmt.Errorf("could not init git repo: %v", err) 809 | } 810 | remoteUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) 811 | refspec := config.RefSpec("+refs/heads/*:refs/remotes/origin/*") 812 | 813 | remote, err := repo.CreateRemote(&config.RemoteConfig{ 814 | Name: "origin", 815 | URLs: []string{remoteUrl}, 816 | Fetch: []config.RefSpec{refspec}, 817 | }) 818 | if err != nil { 819 | return nil, fmt.Errorf("could not create remote: %v", err) 820 | } 821 | 822 | list, err := remote.List(&git.ListOptions{ 823 | Auth: pd.Authenticator, 824 | }) 825 | if err != nil { 826 | log.Println("ignoring no-dup-mode") 827 | } else { 828 | for _, ref := range list { 829 | if !strings.HasPrefix(string(ref.Name()), "refs/tags/imports") { 830 | continue 831 | } 832 | tagIgnoreList = append(tagIgnoreList, string(ref.Name())) 833 | } 834 | } 835 | } 836 | 837 | sourceRepo := *md.Repo 838 | sourceWorktree := *md.Worktree 839 | localPath := "" 840 | 841 | // if a manual commit list is provided, we want to create our md.Branches[] array in a special format: 842 | if len(pd.ManualCommits) > 0 { 843 | md.Branches = []string{} 844 | for _, commit := range pd.ManualCommits { 845 | branchCommit := strings.Split(commit, ":") 846 | if len(branchCommit) != 2 { 847 | return nil, fmt.Errorf("invalid manual commit list") 848 | } 849 | 850 | head := fmt.Sprintf("COMMIT:%s:%s", branchCommit[0], branchCommit[1]) 851 | md.Branches = append(md.Branches, head) 852 | } 853 | } 854 | 855 | for _, branch := range md.Branches { 856 | md.Repo = &sourceRepo 857 | md.Worktree = &sourceWorktree 858 | md.TagBranch = branch 859 | 860 | for _, source := range md.SourcesToIgnore { 861 | source.Expired = true 862 | } 863 | 864 | // Create a temporary place to check out our tag/branch : /tmp/srpmproctmp_/ 865 | localPath, _ = os.MkdirTemp("/tmp", fmt.Sprintf("srpmproctmp_%s", md.Name)) 866 | 867 | if err := os.RemoveAll(localPath); err != nil { 868 | return nil, fmt.Errorf("Could not remove previous temporary directory: %s", localPath) 869 | } 870 | if err := os.Mkdir(localPath, 0o755); err != nil { 871 | return nil, fmt.Errorf("Could not create temporary directory: %s", localPath) 872 | } 873 | 874 | // we'll make our branch we're processing more presentable if it's in the COMMIT:: format: 875 | if strings.HasPrefix(branch, "COMMIT:") { 876 | branch = fmt.Sprintf("refs/heads/%s", strings.Split(branch, ":")[1]) 877 | } 878 | 879 | // Clone repo into the temporary path, but only the tag we're interested in: 880 | // (TODO: will probably need to assign this a variable or use the md struct gitrepo object to perform a successful tag+push later) 881 | rTmp, err := git.PlainClone(localPath, false, &git.CloneOptions{ 882 | URL: pd.RpmLocation, 883 | SingleBranch: true, 884 | ReferenceName: plumbing.ReferenceName(branch), 885 | }) 886 | if err != nil { 887 | return nil, err 888 | } 889 | 890 | // If we're dealing with a special manual commit to import ("COMMIT::"), then we need to check out that 891 | // specific hash from the repo we are importing from: 892 | if strings.HasPrefix(md.TagBranch, "COMMIT:") { 893 | commitList := strings.Split(md.TagBranch, ":") 894 | 895 | wTmp, _ := rTmp.Worktree() 896 | err = wTmp.Checkout(&git.CheckoutOptions{ 897 | Hash: plumbing.NewHash(commitList[2]), 898 | }) 899 | if err != nil { 900 | return nil, fmt.Errorf("Could not find manual commit %s in the repository. Must be a valid commit hash.", commitList[2]) 901 | } 902 | } 903 | 904 | // Now that we're cloned into localPath, we need to "covert" the import into the old format 905 | // We want sources to become .PKGNAME.metadata, we want SOURCES and SPECS folders, etc. 906 | repoFixed, _ := convertLocalRepo(md.Name, localPath) 907 | if !repoFixed { 908 | return nil, fmt.Errorf("Error converting repository into SOURCES + SPECS + .package.metadata format") 909 | } 910 | 911 | // call extra function to determine the proper way to convert the tagless branch name. 912 | // c9s becomes r9s (in the usual case), or in the modular case, stream-httpd-2.4-rhel-9.1.0 becomes r9s-stream-httpd-2.4_r9.1.0 913 | md.PushBranch = taglessBranchName(branch, pd) 914 | 915 | rpmVersion := "" 916 | 917 | // get name-version-release of tagless repo, only if we're not a module repo: 918 | if !pd.ModuleMode { 919 | nvrString, err := getVersionFromSpec(localPath, pd.Version) 920 | if err != nil { 921 | return nil, err 922 | } 923 | 924 | // Set version and release fields we extracted (name|version|release are separated by pipes) 925 | pd.PackageVersion = strings.Split(nvrString, "|")[1] 926 | pd.PackageRelease = strings.Split(nvrString, "|")[2] 927 | 928 | // Set full rpm version: name-version-release (for tagging properly) 929 | rpmVersion = fmt.Sprintf("%s-%s-%s", md.Name, pd.PackageVersion, pd.PackageRelease) 930 | 931 | pd.Log.Println("Successfully determined version of tagless checkout: ", rpmVersion) 932 | } else { 933 | // In case of module mode, we just set rpmVersion to the current date - that's what our tag will end up being 934 | rpmVersion = time.Now().Format("2006-01-02") 935 | } 936 | 937 | // Make an initial repo we will use to push to our target 938 | pushRepo, err := git.PlainInit(localPath+"_gitpush", false) 939 | if err != nil { 940 | return nil, fmt.Errorf("could not create new dist Repo: %v", err) 941 | } 942 | 943 | w, err := pushRepo.Worktree() 944 | if err != nil { 945 | return nil, fmt.Errorf("could not get dist Worktree: %v", err) 946 | } 947 | 948 | // Create a remote "origin" in our empty git, make the upstream equal to the branch we want to modify 949 | pushUrl := fmt.Sprintf("%s/%s/%s.git", pd.UpstreamPrefix, remotePrefix, gitlabify(md.Name)) 950 | refspec := config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", md.PushBranch, md.PushBranch)) 951 | 952 | // Make our remote repo the target one - the one we want to push our update to 953 | pushRepoRemote, err := pushRepo.CreateRemote(&config.RemoteConfig{ 954 | Name: "origin", 955 | URLs: []string{pushUrl}, 956 | Fetch: []config.RefSpec{refspec}, 957 | }) 958 | if err != nil { 959 | return nil, fmt.Errorf("could not create remote: %v", err) 960 | } 961 | 962 | // fetch our branch data (md.PushBranch) into this new repo 963 | err = pushRepo.Fetch(&git.FetchOptions{ 964 | RemoteName: "origin", 965 | RefSpecs: []config.RefSpec{refspec}, 966 | Auth: pd.Authenticator, 967 | }) 968 | 969 | refName := plumbing.NewBranchReferenceName(md.PushBranch) 970 | 971 | var hash plumbing.Hash 972 | h := plumbing.NewSymbolicReference(plumbing.HEAD, refName) 973 | if err := pushRepo.Storer.CheckAndSetReference(h, nil); err != nil { 974 | return nil, fmt.Errorf("Could not set symbolic reference: %v", err) 975 | } 976 | 977 | err = w.Checkout(&git.CheckoutOptions{ 978 | Branch: plumbing.NewRemoteReferenceName("origin", md.PushBranch), 979 | Hash: hash, 980 | Force: true, 981 | }) 982 | 983 | // These os commands actually move the data from our cloned import repo to our cloned "push" repo (upstream -> downstream) 984 | // First we clobber the target push repo's data, and then move the new data into it from the upstream repo we cloned earlier 985 | os.RemoveAll(fmt.Sprintf("%s_gitpush/SPECS", localPath)) 986 | os.RemoveAll(fmt.Sprintf("%s_gitpush/SOURCES", localPath)) 987 | os.RemoveAll(fmt.Sprintf("%s_gitpush/.gitignore", localPath)) 988 | os.RemoveAll(fmt.Sprintf("%s_gitpush/%s.metadata", localPath, md.Name)) 989 | os.Rename(fmt.Sprintf("%s/SPECS", localPath), fmt.Sprintf("%s_gitpush/SPECS", localPath)) 990 | os.Rename(fmt.Sprintf("%s/SOURCES", localPath), fmt.Sprintf("%s_gitpush/SOURCES", localPath)) 991 | os.Rename(fmt.Sprintf("%s/.gitignore", localPath), fmt.Sprintf("%s_gitpush/.gitignore", localPath)) 992 | os.Rename(fmt.Sprintf("%s/.%s.metadata", localPath, md.Name), fmt.Sprintf("%s_gitpush/.%s.metadata", localPath, md.Name)) 993 | 994 | md.Repo = pushRepo 995 | md.Worktree = w 996 | 997 | // Download lookaside sources (tarballs) into the push git repo: 998 | err = pd.Importer.WriteSource(pd, md) 999 | if err != nil { 1000 | return nil, err 1001 | } 1002 | 1003 | // Call function to upload source to target lookaside and 1004 | // ensure the sources are added to .gitignore 1005 | err = processLookasideSources(pd, md, localPath+"_gitpush") 1006 | if err != nil { 1007 | return nil, err 1008 | } 1009 | 1010 | // Apply patch(es) if needed: 1011 | if pd.ModuleMode { 1012 | err := patchModuleYaml(pd, md) 1013 | if err != nil { 1014 | return nil, err 1015 | } 1016 | } else { 1017 | err := executePatchesRpm(pd, md) 1018 | if err != nil { 1019 | return nil, err 1020 | } 1021 | } 1022 | 1023 | err = w.AddWithOptions(&git.AddOptions{All: true}) 1024 | if err != nil { 1025 | return nil, fmt.Errorf("error adding SOURCES/ , SPECS/ or .metadata file to commit list") 1026 | } 1027 | 1028 | status, _ := w.Status() 1029 | if !pd.ModuleMode { 1030 | if status.IsClean() { 1031 | pd.Log.Printf("No changes detected. Our downstream is up to date.") 1032 | head, err := pushRepo.Head() 1033 | if err != nil { 1034 | return nil, fmt.Errorf("error getting HEAD: %v", err) 1035 | } 1036 | latestHashForBranch[md.PushBranch] = head.Hash().String() 1037 | continue 1038 | } 1039 | } 1040 | pd.Log.Printf("successfully processed:\n%s", status) 1041 | 1042 | // assign tag for our new remote we're about to push (derived from the SRPM version) 1043 | newTag := "refs/tags/imports/" + md.PushBranch + "/" + rpmVersion 1044 | newTag = strings.Replace(newTag, "%", "_", -1) 1045 | 1046 | // pushRefspecs is a list of all the references we want to push (tags + heads) 1047 | // It's an array of colon-separated strings which map local references to their remote counterparts 1048 | var pushRefspecs []config.RefSpec 1049 | 1050 | // We need to find out if the remote repo already has this branch 1051 | // If it doesn't, we want to add *:* to our references for commit. This will allow us to push the new branch 1052 | // If it does, we can simply push HEAD:refs/heads/ 1053 | newRepo := true 1054 | refList, _ := pushRepoRemote.List(&git.ListOptions{Auth: pd.Authenticator}) 1055 | for _, ref := range refList { 1056 | if strings.HasSuffix(ref.Name().String(), fmt.Sprintf("heads/%s", md.PushBranch)) { 1057 | newRepo = false 1058 | break 1059 | } 1060 | } 1061 | 1062 | if newRepo { 1063 | pushRefspecs = append(pushRefspecs, config.RefSpec("*:*")) 1064 | pd.Log.Printf("New remote repo detected, creating new remote branch") 1065 | } 1066 | 1067 | // Identify specific references we want to push 1068 | // Should be refs/heads/, and a tag called imports// 1069 | pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:refs/heads/%s", md.PushBranch))) 1070 | pushRefspecs = append(pushRefspecs, config.RefSpec(fmt.Sprintf("HEAD:%s", newTag))) 1071 | 1072 | // Actually do the commit (locally) 1073 | commit, err := w.Commit("import from tagless source "+pd.Importer.ImportName(pd, md), &git.CommitOptions{ 1074 | Author: &object.Signature{ 1075 | Name: pd.GitCommitterName, 1076 | Email: pd.GitCommitterEmail, 1077 | When: time.Now(), 1078 | }, 1079 | }) 1080 | if err != nil { 1081 | return nil, fmt.Errorf("could not commit object: %v", err) 1082 | } 1083 | 1084 | obj, err := pushRepo.CommitObject(commit) 1085 | if err != nil { 1086 | return nil, fmt.Errorf("could not get commit object: %v", err) 1087 | } 1088 | 1089 | pd.Log.Printf("Committed local repo tagless mode transform:\n%s", obj.String()) 1090 | 1091 | // After commit, we will now tag our local repo on disk: 1092 | _, err = pushRepo.CreateTag(newTag, commit, &git.CreateTagOptions{ 1093 | Tagger: &object.Signature{ 1094 | Name: pd.GitCommitterName, 1095 | Email: pd.GitCommitterEmail, 1096 | When: time.Now(), 1097 | }, 1098 | Message: "import " + md.TagBranch + " from " + pd.RpmLocation + "(import from tagless source)", 1099 | SignKey: nil, 1100 | }) 1101 | if err != nil { 1102 | return nil, fmt.Errorf("could not create tag: %v", err) 1103 | } 1104 | 1105 | pd.Log.Printf("Pushing these references to the remote: %+v \n", pushRefspecs) 1106 | 1107 | // Do the actual push to the remote target repository 1108 | err = pushRepo.Push(&git.PushOptions{ 1109 | RemoteName: "origin", 1110 | Auth: pd.Authenticator, 1111 | RefSpecs: pushRefspecs, 1112 | Force: true, 1113 | }) 1114 | 1115 | if err != nil { 1116 | return nil, fmt.Errorf("could not push to remote: %v", err) 1117 | } 1118 | 1119 | if err := os.RemoveAll(localPath); err != nil { 1120 | log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", localPath) 1121 | } 1122 | if err := os.RemoveAll(fmt.Sprintf("%s_gitpush", localPath)); err != nil { 1123 | log.Printf("Error cleaning up temporary git checkout directory %s . Non-fatal, continuing anyway...\n", fmt.Sprintf("%s_gitpush", localPath)) 1124 | } 1125 | 1126 | // append our processed branch to the return structures: 1127 | latestHashForBranch[md.PushBranch] = obj.Hash.String() 1128 | 1129 | versionForBranch[md.PushBranch] = &srpmprocpb.VersionRelease{ 1130 | Version: pd.PackageVersion, 1131 | Release: pd.PackageRelease, 1132 | } 1133 | } 1134 | 1135 | // return struct with all our branch:commit and branch:version+release mappings 1136 | return &srpmprocpb.ProcessResponse{ 1137 | BranchCommits: latestHashForBranch, 1138 | BranchVersions: versionForBranch, 1139 | }, nil 1140 | } 1141 | 1142 | // Given a local repo on disk, ensure it's in the "traditional" format. This means: 1143 | // - metadata file is named .pkgname.metadata 1144 | // - metadata file has the old " SOURCES/" format 1145 | // - SPECS/ and SOURCES/ exist and are populated correctly 1146 | func convertLocalRepo(pkgName string, localRepo string) (bool, error) { 1147 | // Make sure we have a SPECS and SOURCES folder made: 1148 | if err := os.MkdirAll(fmt.Sprintf("%s/SOURCES", localRepo), 0o755); err != nil { 1149 | return false, fmt.Errorf("Could not create SOURCES directory in: %s", localRepo) 1150 | } 1151 | 1152 | if err := os.MkdirAll(fmt.Sprintf("%s/SPECS", localRepo), 0o755); err != nil { 1153 | return false, fmt.Errorf("Could not create SPECS directory in: %s", localRepo) 1154 | } 1155 | 1156 | // Loop through each file/folder and operate accordingly: 1157 | files, err := os.ReadDir(localRepo) 1158 | if err != nil { 1159 | return false, err 1160 | } 1161 | 1162 | for _, f := range files { 1163 | // We don't want to process SOURCES, SPECS, or any of our .git folders 1164 | if f.Name() == "SOURCES" || f.Name() == "SPECS" || strings.HasPrefix(f.Name(), ".git") || f.Name() == "."+pkgName+".metadata" { 1165 | continue 1166 | } 1167 | 1168 | // If we have a metadata "sources" file, we need to read it and convert to the old ..metadata format 1169 | if f.Name() == "sources" { 1170 | convertStatus := convertMetaData(pkgName, localRepo) 1171 | 1172 | if convertStatus != true { 1173 | return false, fmt.Errorf("Error converting sources metadata file to .metadata format") 1174 | } 1175 | 1176 | continue 1177 | } 1178 | 1179 | // Any file that ends in a ".spec" should be put into SPECS/ 1180 | if strings.HasSuffix(f.Name(), ".spec") { 1181 | err := os.Rename(fmt.Sprintf("%s/%s", localRepo, f.Name()), fmt.Sprintf("%s/SPECS/%s", localRepo, f.Name())) 1182 | if err != nil { 1183 | return false, fmt.Errorf("Error moving .spec file to SPECS/") 1184 | } 1185 | } 1186 | 1187 | // if a file isn't skipped in one of the above checks, then it must be a file that belongs in SOURCES/ 1188 | os.Rename(fmt.Sprintf("%s/%s", localRepo, f.Name()), fmt.Sprintf("%s/SOURCES/%s", localRepo, f.Name())) 1189 | } 1190 | 1191 | return true, nil 1192 | } 1193 | 1194 | // Given a local "sources" metadata file (new CentOS Stream format), convert it into the older 1195 | // classic CentOS style: " SOURCES/" 1196 | func convertMetaData(pkgName string, localRepo string) bool { 1197 | lookAside, err := os.Open(fmt.Sprintf("%s/sources", localRepo)) 1198 | if err != nil { 1199 | return false 1200 | } 1201 | 1202 | // Split file into lines and start processing: 1203 | scanner := bufio.NewScanner(lookAside) 1204 | scanner.Split(bufio.ScanLines) 1205 | 1206 | // convertedLA is our array of new "converted" lookaside lines 1207 | var convertedLA []string 1208 | 1209 | // loop through each line, and: 1210 | // - split by whitespace 1211 | // - check each line begins with "SHA" or "MD" - validate 1212 | // - take the 1213 | // Then check 1214 | for scanner.Scan() { 1215 | tmpLine := strings.Fields(scanner.Text()) 1216 | // make sure line starts with a "SHA" or "MD" before processing - otherwise it might not be a valid format lookaside line! 1217 | if !(strings.HasPrefix(tmpLine[0], "SHA") || strings.HasPrefix(tmpLine[0], "MD")) { 1218 | continue 1219 | } 1220 | 1221 | // Strip out "( )" characters from file name and prepend SOURCES/ to it 1222 | tmpLine[1] = strings.ReplaceAll(tmpLine[1], "(", "") 1223 | tmpLine[1] = strings.ReplaceAll(tmpLine[1], ")", "") 1224 | tmpLine[1] = fmt.Sprintf("SOURCES/%s", tmpLine[1]) 1225 | 1226 | convertedLA = append(convertedLA, fmt.Sprintf("%s %s", tmpLine[3], tmpLine[1])) 1227 | } 1228 | lookAside.Close() 1229 | 1230 | // open ..metadata file for writing our old-format lines 1231 | lookAside, err = os.OpenFile(fmt.Sprintf("%s/.%s.metadata", localRepo, pkgName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) 1232 | if err != nil { 1233 | fmt.Errorf("Error opening new .metadata file for writing.") 1234 | return false 1235 | } 1236 | 1237 | writer := bufio.NewWriter(lookAside) 1238 | 1239 | for _, convertedLine := range convertedLA { 1240 | _, _ = writer.WriteString(convertedLine + "\n") 1241 | } 1242 | 1243 | writer.Flush() 1244 | lookAside.Close() 1245 | 1246 | // Remove old "sources" metadata file - we don't need it now that conversion is complete 1247 | os.Remove(fmt.Sprintf("%s/sources", localRepo)) 1248 | 1249 | return true 1250 | } 1251 | 1252 | // Given a local checked out folder and package name, including SPECS/ , SOURCES/ , and .package.metadata, this will: 1253 | // - create a "dummy" SRPM (using dummy sources files we use to populate tarballs from lookaside) 1254 | // - extract RPM version info from that SRPM, and return it 1255 | // 1256 | // If we are in tagless mode, we need to get a package version somehow! 1257 | func getVersionFromSpec(localRepo string, majorVersion int) (string, error) { 1258 | // Make sure we have "rpm" and "rpmbuild" and "cp" available in our PATH. Otherwise, this won't work: 1259 | _, err := exec.LookPath("rpmspec") 1260 | if err != nil { 1261 | return "", fmt.Errorf("Could not find rpmspec program in PATH") 1262 | } 1263 | 1264 | // Read the first file from SPECS/ to get our spec file 1265 | // (there should only be one file - we check that it ends in ".spec" just to be sure!) 1266 | lsTmp, err := os.ReadDir(fmt.Sprintf("%s/SPECS/", localRepo)) 1267 | if err != nil { 1268 | return "", err 1269 | } 1270 | specFile := lsTmp[0].Name() 1271 | 1272 | if !strings.HasSuffix(specFile, ".spec") { 1273 | return "", fmt.Errorf("First file found in SPECS/ is not a .spec file! Check the SPECS/ directory in the repo?") 1274 | } 1275 | 1276 | // Call the rpmspec binary to extract the version-release info out of it, and tack on ".el" at the end: 1277 | cmdArgs := []string{ 1278 | "--srpm", 1279 | fmt.Sprintf(`--define=dist .el%d`, majorVersion), 1280 | fmt.Sprintf(`--define=_topdir %s`, localRepo), 1281 | "-q", 1282 | "--queryformat", 1283 | `%{NAME}|%{VERSION}|%{RELEASE}\n`, 1284 | fmt.Sprintf("%s/SPECS/%s", localRepo, specFile), 1285 | } 1286 | cmd := exec.Command("rpmspec", cmdArgs...) 1287 | nvrTmp, err := cmd.Output() 1288 | if err != nil { 1289 | return "", fmt.Errorf("Error running rpmspec command to determine RPM name-version-release identifier. \nCommand attempted: %s \nCommand output: %s", cmd.String(), string(nvrTmp)) 1290 | } 1291 | 1292 | // Pull first line of the version output to get the name-version-release number (there should only be 1 line) 1293 | nvr := string(nvrTmp) 1294 | nvr = strings.Fields(nvr)[0] 1295 | 1296 | // return name-version-release string we derived: 1297 | log.Printf("Derived NVR %s from tagless repo via rpmspec command\n", nvr) 1298 | return nvr, nil 1299 | } 1300 | 1301 | // We need to loop through the lookaside blob files ("SourcesToIgnore"), 1302 | // and upload them to our target storage (usually an S3 bucket, but could be a local folder) 1303 | // 1304 | // We also need to add the source paths to .gitignore in the git repo, so we don't accidentally commit + push them 1305 | func processLookasideSources(pd *data.ProcessData, md *data.ModeData, localDir string) error { 1306 | w := md.Worktree 1307 | metadata, err := w.Filesystem.Create(fmt.Sprintf(".%s.metadata", md.Name)) 1308 | if err != nil { 1309 | return fmt.Errorf("could not create metadata file: %v", err) 1310 | } 1311 | 1312 | // Keep track of files we've already uploaded - don't want duplicates! 1313 | var alreadyUploadedBlobs []string 1314 | 1315 | gitIgnore, err := os.OpenFile(fmt.Sprintf("%s/.gitignore", localDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) 1316 | if err != nil { 1317 | return err 1318 | } 1319 | 1320 | for _, source := range md.SourcesToIgnore { 1321 | 1322 | sourcePath := source.Name 1323 | _, err := w.Filesystem.Stat(sourcePath) 1324 | if source.Expired || err != nil { 1325 | continue 1326 | } 1327 | 1328 | sourceFile, err := w.Filesystem.Open(sourcePath) 1329 | if err != nil { 1330 | return fmt.Errorf("could not open ignored source file %s: %v", sourcePath, err) 1331 | } 1332 | sourceFileBts, err := io.ReadAll(sourceFile) 1333 | if err != nil { 1334 | return fmt.Errorf("could not read the whole of ignored source file: %v", err) 1335 | } 1336 | 1337 | source.HashFunction.Reset() 1338 | _, err = source.HashFunction.Write(sourceFileBts) 1339 | if err != nil { 1340 | return fmt.Errorf("could not write bytes to hash function: %v", err) 1341 | } 1342 | checksum := hex.EncodeToString(source.HashFunction.Sum(nil)) 1343 | checksumLine := fmt.Sprintf("%s %s\n", checksum, sourcePath) 1344 | _, err = metadata.Write([]byte(checksumLine)) 1345 | if err != nil { 1346 | return fmt.Errorf("could not write to metadata file: %v", err) 1347 | } 1348 | 1349 | if data.StrContains(alreadyUploadedBlobs, checksum) { 1350 | continue 1351 | } 1352 | exists, err := pd.BlobStorage.Exists(checksum) 1353 | if err != nil { 1354 | return err 1355 | } 1356 | if !exists && !pd.NoStorageUpload { 1357 | err := pd.BlobStorage.Write(checksum, sourceFileBts) 1358 | if err != nil { 1359 | return err 1360 | } 1361 | pd.Log.Printf("wrote %s to blob storage", checksum) 1362 | } 1363 | alreadyUploadedBlobs = append(alreadyUploadedBlobs, checksum) 1364 | 1365 | // Add this SOURCES/ lookaside file to be excluded 1366 | w.Excludes = append(w.Excludes, gitignore.ParsePattern(sourcePath, nil)) 1367 | 1368 | // Append the SOURCES/ path to .gitignore: 1369 | _, err = gitIgnore.Write([]byte(fmt.Sprintf("%s\n", sourcePath))) 1370 | if err != nil { 1371 | return err 1372 | } 1373 | 1374 | } 1375 | 1376 | err = gitIgnore.Close() 1377 | if err != nil { 1378 | return err 1379 | } 1380 | 1381 | return nil 1382 | } 1383 | 1384 | // Given an input branch name to import from, like "refs/heads/c9s", produce the tagless branch name we want to commit to, like "r9s" 1385 | // Modular translation of CentOS stream branches i is also done - branch stream-maven-3.8-rhel-9.1.0 ----> r9s-stream-maven-3.8_9.1.0 1386 | func taglessBranchName(fullBranch string, pd *data.ProcessData) string { 1387 | // Split the full branch name "refs/heads/blah" to only get the short name - last entry 1388 | tmpBranch := strings.Split(fullBranch, "/") 1389 | branch := tmpBranch[len(tmpBranch)-1] 1390 | 1391 | // Simple case: if our branch is not a modular stream branch, just return the normal pattern 1392 | if !strings.HasPrefix(branch, "stream-") { 1393 | return fmt.Sprintf("%s%d%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix) 1394 | } 1395 | 1396 | // index where the "-rhel-" starts near the end of the string 1397 | rhelSpot := strings.LastIndex(branch, "-rhel-") 1398 | 1399 | // module name will be everything from the start until that "-rhel-" string (like "stream-httpd-2.4") 1400 | moduleString := branch[0:rhelSpot] 1401 | 1402 | // major minor version is everything after the "-rhel-" string 1403 | majorMinor := branch[rhelSpot+6:] 1404 | 1405 | // return translated modular branch: 1406 | return fmt.Sprintf("%s%d%s-%s_%s", pd.BranchPrefix, pd.Version, pd.BranchSuffix, moduleString, majorMinor) 1407 | } 1408 | -------------------------------------------------------------------------------- /proto/cfg.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "github.com/rocky-linux/srpmproc/pb;srpmprocpb"; 4 | 5 | package srpmproc; 6 | 7 | // Replace directive replaces a file from the rpm repository 8 | // with a file from the patch repository. 9 | // Replacing content can either be inline or in the same patch-tree. 10 | message Replace { 11 | // Required - Replaced file 12 | string file = 1; 13 | 14 | oneof replacing { 15 | // Replace with in-tree file 16 | string with_file = 2; 17 | 18 | // Replace with inline content 19 | string with_inline = 3; 20 | 21 | // Replace with lookaside cache object 22 | string with_lookaside = 4; 23 | } 24 | } 25 | 26 | // Delete directive deletes literal files from the rpm repository. 27 | // Won't delete from spec automatically. 28 | // Use the `SpecChange` directive for that 29 | message Delete { 30 | // Required 31 | string file = 1; 32 | } 33 | 34 | // Add directive adds a file from the patch repository to the rpm repository. 35 | // The file is added in the `SOURCES` directory 36 | // Won't add to spec automatically. 37 | // Use the `SpecChange` directive for that 38 | message Add { 39 | // Required - file to add 40 | oneof source { 41 | string file = 1; 42 | string lookaside = 2; 43 | } 44 | 45 | // Overrides file name if specified 46 | string name = 3; 47 | } 48 | 49 | // Lookaside directive puts patched files in blob storage. 50 | // If tar is true, the files will be put into a tarball and gzipped 51 | message Lookaside { 52 | // Required - List of files that should be stored in blob storage 53 | repeated string file = 1; 54 | 55 | // Whether files should be put into a tarball and gzipped 56 | bool tar = 2; 57 | 58 | // Name of tar file, only used and required if tar is true 59 | string archive_name = 3; 60 | 61 | // Whether if files should be retrieved from patch tree 62 | bool from_patch_tree = 4; 63 | } 64 | 65 | // SpecChange directive makes it possible to execute certain 66 | // plans against the package spec 67 | message SpecChange { 68 | // The FileOperation plan allows patchers to add or delete 69 | // a file from the spec. 70 | message FileOperation { 71 | enum Type { 72 | Unknown = 0; 73 | Source = 1; 74 | Patch = 2; 75 | } 76 | // File name 77 | string name = 1; 78 | // File type 79 | Type type = 2; 80 | 81 | oneof mode { 82 | // Add won't add the file to the tree. 83 | // Use the `Add` directive for that 84 | bool add = 3; 85 | // Delete won't delete the file from the tree. 86 | // Use the `Delete` directive for that 87 | bool delete = 4; 88 | } 89 | 90 | // Only works for patch type 91 | bool add_to_prep = 5; 92 | int32 n_path = 6; 93 | } 94 | // ChangelogOperation adds a new changelog entry 95 | message ChangelogOperation { 96 | string author_name = 1; 97 | string author_email = 2; 98 | repeated string message = 3; 99 | } 100 | // SearchAndReplaceOperation replaces substring with value 101 | // in a specified field 102 | message SearchAndReplaceOperation { 103 | oneof identifier { 104 | // replace occurrences in field value 105 | string field = 1; 106 | // replace occurrences in any line 107 | bool any = 2; 108 | // replace occurrences that starts with find 109 | bool starts_with = 3; 110 | // replace occurrences that ends with find 111 | bool ends_with = 4; 112 | } 113 | string find = 5; 114 | string replace = 6; 115 | // How many occurences to replace. 116 | // Set to -1 for all 117 | sint32 n = 7; 118 | } 119 | // AppendOperation appends a value to specified field or section 120 | message AppendOperation { 121 | string field = 1; 122 | string value = 2; 123 | } 124 | // NewFieldOperation adds a new kv to the spec 125 | // The field will be grouped if other fields of same name exists 126 | message NewFieldOperation { 127 | // Key cannot be Source or Patch 128 | string key = 1; 129 | string value = 2; 130 | } 131 | 132 | repeated FileOperation file = 1; 133 | repeated ChangelogOperation changelog = 2; 134 | repeated SearchAndReplaceOperation search_and_replace = 3; 135 | repeated AppendOperation append = 4; 136 | repeated NewFieldOperation new_field = 5; 137 | bool disable_auto_align = 6; 138 | } 139 | 140 | message Patch { 141 | // Path to patch file from repo root 142 | string file = 1; 143 | 144 | // Srpmproc adds `SOURCES/` to files in a diff 145 | // without a prefix if strict is false. 146 | // If strict is true, then that is disabled. 147 | bool strict = 2; 148 | } 149 | 150 | message Cfg { 151 | repeated Replace replace = 1; 152 | repeated Delete delete = 2; 153 | repeated Add add = 3; 154 | repeated Lookaside lookaside = 4; 155 | SpecChange spec_change = 5; 156 | repeated Patch patch = 6; 157 | } 158 | -------------------------------------------------------------------------------- /proto/response.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "github.com/rocky-linux/srpmproc/pb;srpmprocpb"; 4 | 5 | package srpmproc; 6 | 7 | message VersionRelease { 8 | string version = 1; 9 | string release = 2; 10 | } 11 | 12 | message ProcessResponse { 13 | map branch_commits = 1; 14 | map branch_versions = 2; 15 | } 16 | --------------------------------------------------------------------------------