├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ ├── codeql.yml │ ├── lint.yml │ └── release.yml ├── .gitignore ├── .golangci.yaml ├── .goreleaser.yaml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── cmd └── xurlunpack3r │ └── main.go ├── go.mod ├── go.sum ├── internal ├── configuration │ └── configuration.go └── input │ └── input.go └── urls.txt /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - 5 | package-ecosystem: "github-actions" 6 | directory: "/" 7 | schedule: 8 | interval: "weekly" 9 | target-branch: "dev" 10 | commit-message: 11 | prefix: "chore" 12 | include: "scope" 13 | labels: 14 | - "Type: Maintenance" 15 | - 16 | package-ecosystem: "gomod" 17 | directory: "/" 18 | schedule: 19 | interval: "weekly" 20 | target-branch: "dev" 21 | commit-message: 22 | prefix: "chore" 23 | include: "scope" 24 | labels: 25 | - "Type: Maintenance" -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: 🔨 Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - "main" 7 | paths: 8 | - '**.go' 9 | - '**.mod' 10 | pull_request: 11 | branches: 12 | - "main" 13 | paths: 14 | - '**.go' 15 | - '**.mod' 16 | workflow_dispatch: 17 | 18 | jobs: 19 | build: 20 | name: Build 21 | strategy: 22 | matrix: 23 | os: [ubuntu-latest, windows-latest, macOS-12] 24 | runs-on: ${{ matrix.os }} 25 | steps: 26 | - 27 | name: Set up Go 28 | uses: actions/setup-go@v5 29 | with: 30 | go-version: '>=1.23' 31 | - 32 | name: Code Checkout 33 | uses: actions/checkout@v4 34 | with: 35 | fetch-depth: 0 36 | - 37 | name: Go Module Management 38 | run: | 39 | make go-mod-clean 40 | make go-mod-tidy 41 | working-directory: . 42 | - 43 | name: Go Build 44 | run: | 45 | make go-build 46 | working-directory: . 47 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: 🚨 Analyze Code (CodeQL) 2 | 3 | on: 4 | push: 5 | branches: 6 | - "main" 7 | paths: 8 | - '**.go' 9 | - '**.mod' 10 | pull_request: 11 | branches: 12 | - "main" 13 | paths: 14 | - '**.go' 15 | - '**.mod' 16 | workflow_dispatch: 17 | 18 | jobs: 19 | analyze: 20 | name: Analyze Code (CodeQL) 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | language: [ 'go' ] 25 | runs-on: ubuntu-latest 26 | permissions: 27 | actions: read 28 | contents: read 29 | security-events: write 30 | steps: 31 | - 32 | name: Code Checkout 33 | uses: actions/checkout@v4 34 | with: 35 | fetch-depth: 0 36 | - 37 | name: Initialize CodeQL 38 | uses: github/codeql-action/init@v3 39 | with: 40 | languages: ${{ matrix.language }} 41 | - 42 | name: Go Module Management 43 | run: | 44 | make go-mod-clean 45 | make go-mod-tidy 46 | working-directory: . 47 | - 48 | name: Go Build 49 | run: | 50 | make go-build 51 | working-directory: . 52 | - 53 | name: Perform CodeQL Analysis 54 | uses: github/codeql-action/analyze@v3 -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: 💅 Lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - "main" 7 | paths: 8 | - '**.go' 9 | - '**.mod' 10 | pull_request: 11 | branches: 12 | - "main" 13 | paths: 14 | - '**.go' 15 | - '**.mod' 16 | workflow_dispatch: 17 | 18 | permissions: 19 | contents: read 20 | 21 | jobs: 22 | lint: 23 | name: Lint 24 | runs-on: ubuntu-latest 25 | steps: 26 | - 27 | name: Set up Go 28 | uses: actions/setup-go@v5 29 | with: 30 | go-version: '>=1.23' 31 | cache: false 32 | - 33 | name: Code Checkout 34 | uses: actions/checkout@v4 35 | with: 36 | fetch-depth: 0 37 | - 38 | name: golangci-lint 39 | uses: golangci/golangci-lint-action@v6 40 | with: 41 | version: v1.62.2 42 | args: --timeout 5m 43 | working-directory: . -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: 🎉 Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*.*.*' 7 | - '*.*.*' 8 | workflow_dispatch: 9 | 10 | jobs: 11 | release: 12 | name: Release 13 | runs-on: ubuntu-latest 14 | steps: 15 | - 16 | name: Set up Go 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version: '>=1.23' 20 | - 21 | name: Code Checkout 22 | uses: actions/checkout@v4 23 | with: 24 | fetch-depth: 0 25 | - 26 | name: GoReleaser 27 | uses: goreleaser/goreleaser-action@v6 28 | with: 29 | distribution: goreleaser 30 | version: latest 31 | args: "release --clean" 32 | workdir: . 33 | env: 34 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 35 | DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" 36 | DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE stuff 2 | .vscode 3 | 4 | # Binaries 5 | bin -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | # Options for analysis running. 2 | run: 3 | # Number of operating system threads (`GOMAXPROCS`) that can execute golangci-lint simultaneously. 4 | # If it is explicitly set to 0 (i.e. not the default) then golangci-lint will automatically set the value to match Linux container CPU quota. 5 | # Default: the number of logical CPUs in the machine 6 | # concurrency: 4 7 | # Timeout for analysis, e.g. 30s, 5m. 8 | # Default: 1m 9 | timeout: 5m 10 | # Exit code when at least one issue was found. 11 | # Default: 1 12 | issues-exit-code: 1 13 | # Include test files or not. 14 | # Default: true 15 | tests: true 16 | # List of build tags, all linters use it. 17 | # Default: [] 18 | build-tags: [] 19 | # If set, we pass it to "go list -mod={option}". From "go help modules": 20 | # If invoked with -mod=readonly, the go command is disallowed from the implicit 21 | # automatic updating of go.mod described above. Instead, it fails when any changes 22 | # to go.mod are needed. This setting is most useful to check that go.mod does 23 | # not need updates, such as in a continuous integration and testing system. 24 | # If invoked with -mod=vendor, the go command assumes that the vendor 25 | # directory holds the correct copies of dependencies and ignores 26 | # the dependency descriptions in go.mod. 27 | # 28 | # Allowed values: readonly|vendor|mod 29 | # Default: "" 30 | modules-download-mode: readonly 31 | # Allow multiple parallel golangci-lint instances running. 32 | # If false, golangci-lint acquires file lock on start. 33 | # Default: false 34 | allow-parallel-runners: true 35 | # Allow multiple golangci-lint instances running, but serialize them around a lock. 36 | # If false, golangci-lint exits with an error if it fails to acquire file lock on start. 37 | # Default: false 38 | allow-serial-runners: true 39 | # Define the Go version limit. 40 | # Mainly related to generics support since go1.18. 41 | # Default: use Go version from the go.mod file, fallback on the env var `GOVERSION`, fallback on 1.17 42 | go: '1.23' 43 | 44 | # output configuration options 45 | output: 46 | # The formats used to render issues. 47 | # Formats: 48 | # - `colored-line-number` 49 | # - `line-number` 50 | # - `json` 51 | # - `colored-tab` 52 | # - `tab` 53 | # - `html` 54 | # - `checkstyle` 55 | # - `code-climate` 56 | # - `junit-xml` 57 | # - `junit-xml-extended` 58 | # - `github-actions` 59 | # - `teamcity` 60 | # - `sarif` 61 | # Output path can be either `stdout`, `stderr` or path to the file to write to. 62 | # 63 | # For the CLI flag (`--out-format`), multiple formats can be specified by separating them by comma. 64 | # The output can be specified for each of them by separating format name and path by colon symbol. 65 | # Example: "--out-format=checkstyle:report.xml,json:stdout,colored-line-number" 66 | # The CLI flag (`--out-format`) override the configuration file. 67 | # 68 | # Default: 69 | # formats: 70 | # - format: colored-line-number 71 | # path: stdout 72 | formats: 73 | # - 74 | # format: json 75 | # path: stderr 76 | # - 77 | # format: checkstyle 78 | # path: report.xml 79 | - 80 | format: colored-line-number 81 | path: stderr 82 | # Print lines of code with issue. 83 | # Default: true 84 | print-issued-lines: true 85 | # Print linter name in the end of issue text. 86 | # Default: true 87 | print-linter-name: true 88 | # Make issues output unique by line. 89 | # Default: true 90 | uniq-by-line: false 91 | # Add a prefix to the output file references. 92 | # Default: "" 93 | path-prefix: "" 94 | # Sort results by the order defined in `sort-order`. 95 | # Default: false 96 | sort-results: true 97 | # Order to use when sorting results. 98 | # Require `sort-results` to `true`. 99 | # Possible values: `file`, `linter`, and `severity`. 100 | # 101 | # If the severity values are inside the following list, they are ordered in this order: 102 | # 1. error 103 | # 2. warning 104 | # 3. high 105 | # 4. medium 106 | # 5. low 107 | # Either they are sorted alphabetically. 108 | # 109 | # Default: ["file"] 110 | sort-order: 111 | - linter 112 | - severity 113 | - file # filepath, line, and column. 114 | # Show statistics per linter. 115 | # Default: false 116 | show-stats: false 117 | 118 | linters: 119 | # Disable all linters. 120 | # Default: false 121 | disable-all: true 122 | # Enable specific linter 123 | # https://golangci-lint.run/usage/linters/#enabled-by-default 124 | enable: 125 | - asasalint 126 | - asciicheck 127 | - bidichk 128 | - bodyclose 129 | - canonicalheader 130 | - containedctx 131 | - contextcheck 132 | - copyloopvar 133 | # - cyclop 134 | - decorder 135 | # - depguard 136 | - dogsled 137 | - dupl 138 | - dupword 139 | - durationcheck 140 | - err113 141 | - errcheck 142 | - errchkjson 143 | - errname 144 | - errorlint 145 | - exhaustive 146 | # - exhaustruct 147 | - fatcontext 148 | - forbidigo 149 | - forcetypeassert 150 | # - funlen 151 | - gci 152 | - ginkgolinter 153 | - gocheckcompilerdirectives 154 | # - gochecknoglobals 155 | # - gochecknoinits 156 | - gochecksumtype 157 | # - gocognit 158 | - goconst 159 | - gocritic 160 | # - gocyclo 161 | - godot 162 | - godox 163 | - gofmt 164 | - gofumpt 165 | - goheader 166 | - goimports 167 | - gomoddirectives 168 | - gomodguard 169 | - goprintffuncname 170 | - gosec 171 | - gosimple 172 | - gosmopolitan 173 | - govet 174 | - grouper 175 | - importas 176 | - inamedparam 177 | - ineffassign 178 | - interfacebloat 179 | - intrange 180 | - ireturn 181 | # - lll 182 | - loggercheck 183 | - maintidx 184 | - makezero 185 | - mirror 186 | - misspell 187 | # - mnd 188 | - musttag 189 | # - nakedret 190 | - nestif 191 | - nilerr 192 | - nilnil 193 | - nlreturn 194 | - noctx 195 | - nolintlint 196 | # - nonamedreturns 197 | - nosprintfhostport 198 | - paralleltest 199 | # - perfsprint 200 | - prealloc 201 | - predeclared 202 | - promlinter 203 | - protogetter 204 | - reassign 205 | - revive 206 | - rowserrcheck 207 | - sloglint 208 | - spancheck 209 | - sqlclosecheck 210 | - staticcheck 211 | - stylecheck 212 | - tagalign 213 | # - tagliatelle 214 | - tenv 215 | - testableexamples 216 | - testifylint 217 | - testpackage 218 | - thelper 219 | - tparallel 220 | - unconvert 221 | - unparam 222 | - unused 223 | - usestdlibvars 224 | # - varnamelen 225 | - wastedassign 226 | - whitespace 227 | - wrapcheck 228 | - wsl 229 | - zerologlint 230 | 231 | linters-settings: 232 | goconst: 233 | min-len: 2 234 | min-occurrences: 3 235 | gocritic: 236 | enabled-tags: 237 | - performance 238 | - experimental 239 | - style 240 | - opinionated 241 | disabled-checks: 242 | - captLocal 243 | - whyNoLint 244 | gocyclo: 245 | # Minimal code complexity to report. 246 | # Default: 30 (but we recommend 10-20) 247 | min-complexity: 10 248 | # varnamelen: 249 | # # The minimum length of a variable's name that is considered "long". 250 | # # Variable names that are at least this long will be ignored. 251 | # # Default: 3 252 | # min-name-length: 2 253 | # # Check method receivers. 254 | # # Default: false 255 | # check-receiver: true 256 | # # Check named return values. 257 | # # Default: false 258 | # check-return: true 259 | # # Check type parameters. 260 | # # Default: false 261 | # check-type-param: true 262 | whitespace: 263 | # Enforces newlines (or comments) after every multi-line if statement. 264 | # Default: false 265 | multi-if: true 266 | # Enforces newlines (or comments) after every multi-line function signature. 267 | # Default: false 268 | multi-func: true 269 | 270 | issues: 271 | # Which dirs to exclude: issues from them won't be reported. 272 | # Can use regexp here: `generated.*`, regexp is applied on full path, 273 | # including the path prefix if one is set. 274 | # Default dirs are skipped independently of this option's value (see exclude-dirs-use-default). 275 | # "/" will be replaced by current OS file path separator to properly work on Windows. 276 | # Default: [] 277 | exclude-dirs: [] 278 | # Show issues in any part of update files (requires new-from-rev or new-from-patch). 279 | # Default: false 280 | whole-files: false 281 | # Fix found issues (if it's supported by the linter). 282 | # Default: false 283 | fix: true -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod tidy 4 | 5 | builds: 6 | - 7 | id: xurlunpack3r-cli 8 | main: cmd/xurlunpack3r/main.go 9 | binary: xurlunpack3r 10 | 11 | env: 12 | - CGO_ENABLED=0 13 | 14 | goos: 15 | - linux 16 | - windows 17 | - darwin 18 | goarch: 19 | - amd64 20 | - 386 21 | - arm 22 | - arm64 23 | ignore: 24 | - 25 | goos: darwin 26 | goarch: 386 27 | - 28 | goos: windows 29 | goarch: arm 30 | - 31 | goos: windows 32 | goarch: arm64 33 | 34 | flags: 35 | - -trimpath 36 | 37 | archives: 38 | - 39 | id: tgz 40 | builds: 41 | - xurlunpack3r-cli 42 | format: tar.gz 43 | format_overrides: 44 | - 45 | goos: windows 46 | format: zip 47 | 48 | checksum: 49 | algorithm: sha256 50 | 51 | announce: 52 | discord: 53 | enabled: true 54 | message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are welcomed! 4 | 5 | Please review the following guidelines before contributing. Also, feel free to propose changes to these guidelines by updating this file and submitting a pull request. 6 | 7 | * [I have a question...](#have-a-question) 8 | * [I found a bug...](#found-a-bug) 9 | * [I have a feature request...](#have-a-feature-request) 10 | * [I have a contribution to share...](#ready-to-contribute) 11 | 12 | ## Have a Question? 13 | 14 | Please don't open a GitHub issue for questions about how to use `xurlunpack3r`, as the goal is to use issues for managing bugs and feature requests. Issues that are related to general support will be closed. 15 | 16 | ## Found a Bug? 17 | 18 | If you've identified a bug in `xurlunpack3r`, please [submit an issue](#create-an-issue) to our GitHub repo: [hueristiq/xurlunpack3r](https://github.com/hueristiq/xurlunpack3r/issues/new). Please also feel free to submit a [Pull Request](#pull-requests) with a fix for the bug! 19 | 20 | ## Have a Feature Request? 21 | 22 | All feature requests should start with [submitting an issue](#create-an-issue) documenting the user story and acceptance criteria. Again, feel free to submit a [Pull Request](#pull-requests) with a proposed implementation of the feature. 23 | 24 | ## Ready to Contribute 25 | 26 | ### Create an issue 27 | 28 | Before submitting a new issue, please search the issues to make sure there isn't a similar issue doesn't already exist. 29 | 30 | Assuming no existing issues exist, please ensure you include required information when submitting the issue to ensure we can quickly reproduce your issue. 31 | 32 | We may have additional questions and will communicate through the GitHub issue, so please respond back to our questions to help reproduce and resolve the issue as quickly as possible. 33 | 34 | New issues can be created with in our [GitHub repo](https://github.com/hueristiq/xurlunpack3r/issues/new). 35 | 36 | ### Pull Requests 37 | 38 | Pull requests should target the `dev` branch. Please also reference the issue from the description of the pull request using [special keyword syntax](https://help.github.com/articles/closing-issues-via-commit-messages/) to auto close the issue when the PR is merged. For example, include the phrase `fixes #14` in the PR description to have issue #14 auto close. 39 | 40 | ### Styleguide 41 | 42 | When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible. Here are a few points to keep in mind: 43 | 44 | * All dependencies must be defined in the `go.mod` file. 45 | * Advanced IDEs and code editors (like VSCode) will take care of that, but to be sure, run `make go-mod-tidy` to validate dependencies. 46 | * Please run `make go-fmt` before committing to ensure code aligns with go standards. 47 | * We use [`golangci-lint`](https://golangci-lint.run/) for linting Go code, run `make go-lint` before submitting PR. Editors such as Visual Studio Code or JetBrains IntelliJ; with Go support plugin will offer `golangci-lint` automatically. 48 | * For details on the approved style, check out [Effective Go](https://golang.org/doc/effective_go.html). 49 | 50 | ### License 51 | 52 | By contributing your code, you agree to license your contribution under the terms of the [MIT License](https://github.com/hueristiq/xurlunpack3r/blob/master/LICENSE). 53 | 54 | All files are released with the MIT license. 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Hueristiq 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Specifies the shell to be used for executing commands. In this case, it's set to `/bin/bash`. 2 | # Bash is chosen for its advanced scripting capabilities, including string manipulation and conditional checks. 3 | SHELL = /bin/bash 4 | 5 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 6 | # --- Prepare | Setup ------------------------------------------------------------------------------------------------------------------------------------------------------ 7 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 8 | 9 | .PHONY: git-hooks-install 10 | # Target: git-hooks-install 11 | # Purpose: 12 | # Installs and configures Git hooks using Lefthook, a Git hooks manager. 13 | # Details: 14 | # - First, the target checks if the `lefthook` command is available in the system PATH. 15 | # - If `lefthook` is not installed, it installs the latest version using Go. 16 | # - Finally, it executes `lefthook install` to set up Git hooks based on the repository's configuration. 17 | git-hooks-install: 18 | @command -v lefthook || go install github.com/evilmartians/lefthook@latest; lefthook install 19 | 20 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 21 | # --- Go (Golang) ---------------------------------------------------------------------------------------------------------------------------------------------------------- 22 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 23 | 24 | .PHONY: go-mod-clean 25 | # Target: go-mod-clean 26 | # Purpose: 27 | # Cleans the Go module cache to remove any cached module files. 28 | # Details: 29 | # - This target runs `go clean -modcache` which is useful if you encounter issues with outdated or corrupt module cache. 30 | go-mod-clean: 31 | go clean -modcache 32 | 33 | .PHONY: go-mod-tidy 34 | # Target: go-mod-tidy 35 | # Purpose: 36 | # Tidies up the go.mod file by adding missing and removing unused modules. 37 | # Details: 38 | # - Running `go mod tidy` ensures that the go.mod file accurately reflects the dependencies used in the project. 39 | go-mod-tidy: 40 | go mod tidy 41 | 42 | .PHONY: go-mod-update 43 | # Target: go-mod-update 44 | # Purpose: 45 | # Updates all Go modules to their latest versions. 46 | # Details: 47 | # - First, updates test dependencies with the flags: -f (force), -t (include test packages), and -u (update). 48 | # - Then, updates all other dependencies. 49 | go-mod-update: 50 | go get -f -t -u ./... 51 | go get -f -u ./... 52 | 53 | .PHONY: go-fmt 54 | # Target: go-fmt 55 | # Purpose: 56 | # Formats the Go source code. 57 | # Details: 58 | # - Uses `go fmt ./...` to format all Go source files across the module, ensuring consistent code style. 59 | go-fmt: 60 | go fmt ./... 61 | 62 | .PHONY: go-lint 63 | # Target: go-lint 64 | # Purpose: 65 | # Lints the Go source code to catch potential issues and enforce code quality. 66 | # Details: 67 | # - The target first ensures that the code is properly formatted by invoking the `go-fmt` target. 68 | # - Then, it checks if `golangci-lint` is available; if not, it installs a specific version. 69 | # - Finally, it runs the linter across all packages. 70 | go-lint: go-fmt 71 | @(command -v golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.64.5) && golangci-lint run ./... 72 | 73 | .PHONY: go-test 74 | # Target: go-test 75 | # Purpose: 76 | # Executes the test suite for the Go project. 77 | # Details: 78 | # - Runs tests in verbose mode (`-v`) and with race condition detection (`-race`) to ensure thread safety. 79 | # - The tests are executed for all packages in the module. 80 | go-test: 81 | go test -v -race ./... 82 | 83 | .PHONY: go-build 84 | go-build: 85 | go build -v -ldflags '-s -w' -o bin/xurlunpack3r cmd/xurlunpack3r/main.go 86 | 87 | .PHONY: go-install 88 | go-install: 89 | go install -v ./... 90 | 91 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 92 | # --- Docker --------------------------------------------------------------------------------------------------------------------------------------------------------------- 93 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 94 | 95 | DOCKERCMD = docker 96 | DOCKERBUILD = $(DOCKERCMD) build 97 | 98 | DOCKERFILE := ./Dockerfile 99 | 100 | IMAGE_NAME = hueristiq/xurlunpack3r 101 | IMAGE_TAG = $(shell cat internal/configuration/configuration.go | grep "VERSION =" | sed 's/.*VERSION = "\([0-9.]*\)".*/\1/') 102 | IMAGE = $(IMAGE_NAME):$(IMAGE_TAG) 103 | 104 | .PHONY: docker-build 105 | docker-build: 106 | @$(DOCKERBUILD) -f $(DOCKERFILE) -t $(IMAGE) -t $(IMAGE_NAME):latest . 107 | 108 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 109 | # --- Help ----------------------------------------------------------------------------------------------------------------------------------------------------------------- 110 | # -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 111 | 112 | .PHONY: help 113 | # Target: help 114 | # Purpose: 115 | # Displays an overview of available targets along with their descriptions. 116 | # Details: 117 | # - When no target is provided, the default action (set by .DEFAULT_GOAL) is to show this help text. 118 | # - This target prints categorized sections for environment management, Go commands, Docker commands, and help. 119 | help: 120 | @echo "" 121 | @echo "*****************************************************************************" 122 | @echo "" 123 | @echo "PROJECT : $(PROJECT)" 124 | @echo "" 125 | @echo "*****************************************************************************" 126 | @echo "" 127 | @echo "Available commands:" 128 | @echo "" 129 | @echo " Git Hooks:" 130 | @echo " git-hooks-install ........ Install Git hooks." 131 | @echo "" 132 | @echo " Go Commands:" 133 | @echo " go-mod-clean ............. Clean Go module cache." 134 | @echo " go-mod-tidy .............. Tidy Go modules." 135 | @echo " go-mod-update ............ Update Go modules." 136 | @echo " go-fmt ................... Format Go code." 137 | @echo " go-lint .................. Lint Go code." 138 | @echo " go-test .................. Run Go tests." 139 | @echo " go-build ................. Build Go program." 140 | @echo " go-install ............... Install Go program." 141 | @echo "" 142 | @echo " Docker Commands:" 143 | @echo " docker-build ............. Build Docker image." 144 | @echo "" 145 | @echo " Help Commands:" 146 | @echo " help ..................... Display this help information." 147 | @echo "" 148 | 149 | # Set the default target to the help command. 150 | # This ensures that running `make` without arguments provides a summary of available targets. 151 | .DEFAULT_GOAL = help -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xurlunpack3r 2 | 3 | ![made with go](https://img.shields.io/badge/made%20with-Go-1E90FF.svg) [![go report card](https://goreportcard.com/badge/github.com/hueristiq/xurlunpack3r)](https://goreportcard.com/report/github.com/hueristiq/xurlunpack3r) [![release](https://img.shields.io/github/release/hueristiq/xurlunpack3r?style=flat&color=1E90FF)](https://github.com/hueristiq/xurlunpack3r/releases) [![open issues](https://img.shields.io/github/issues-raw/hueristiq/xurlunpack3r.svg?style=flat&color=1E90FF)](https://github.com/hueristiq/xurlunpack3r/issues?q=is:issue+is:open) [![closed issues](https://img.shields.io/github/issues-closed-raw/hueristiq/xurlunpack3r.svg?style=flat&color=1E90FF)](https://github.com/hueristiq/xurlunpack3r/issues?q=is:issue+is:closed) [![license](https://img.shields.io/badge/license-MIT-gray.svg?color=1E90FF)](https://github.com/hueristiq/xurlunpack3r/blob/master/LICENSE) ![maintenance](https://img.shields.io/badge/maintained%3F-yes-1E90FF.svg) [![contribution](https://img.shields.io/badge/contributions-welcome-1E90FF.svg)](https://github.com/hueristiq/xurlunpack3r/blob/master/CONTRIBUTING.md) 4 | 5 | `xurlunpack3r` is a command-line utility designed to extract specific parts from URLs. 6 | 7 | ## Resources 8 | 9 | * [Features](#features) 10 | * [Installation](#installation) 11 | * [Install release binaries (Without Go Installed)](#install-release-binaries-without-go-installed) 12 | * [Install source (With Go Installed)](#install-source-with-go-installed) 13 | * [`go install ...`](#go-install) 14 | * [`go build ...` the development Version](#go-build--the-development-version) 15 | * [Usage](#usage) 16 | * [Examples](#examples) 17 | * [Domains](#domains) 18 | * [Apex Domains](#apex-domains) 19 | * [Paths](#paths) 20 | * [Query String Key/Value Pairs](#query-string-keyvalue-pairs) 21 | * [Query String Keys (Parameters)](#query-string-keys-parameters) 22 | * [Query String Values](#query-string-values) 23 | * [Custom Formats](#custom-formats) 24 | * [Contributing](#contributing) 25 | * [Licensing](#licensing) 26 | 27 | ## Features 28 | 29 | - Multiple Extraction Modes 30 | - Custom Formats 31 | - Cross-Platform (Windows, Linux, and macOS) 32 | 33 | ## Installation 34 | 35 | ### Install release binaries (without Go installed) 36 | 37 | Visit the [releases page](https://github.com/hueristiq/xurlunpack3r/releases) and find the appropriate archive for your operating system and architecture. Download the archive from your browser or copy its URL and retrieve it with `wget` or `curl`: 38 | 39 | * ...with `wget`: 40 | 41 | ```bash 42 | wget https://github.com/hueristiq/xurlunpack3r/releases/download/v/xurlunpack3r--linux-amd64.tar.gz 43 | ``` 44 | 45 | * ...or, with `curl`: 46 | 47 | ```bash 48 | curl -OL https://github.com/hueristiq/xurlunpack3r/releases/download/v/xurlunpack3r--linux-amd64.tar.gz 49 | ``` 50 | 51 | ...then, extract the binary: 52 | 53 | ```bash 54 | tar xf xurlunpack3r--linux-amd64.tar.gz 55 | ``` 56 | 57 | > [!TIP] 58 | > The above steps, download and extract, can be combined into a single step with this onliner 59 | > 60 | > ```bash 61 | > curl -sL https://github.com/hueristiq/xurlunpack3r/releases/download/v/xurlunpack3r--linux-amd64.tar.gz | tar -xzv 62 | > ``` 63 | 64 | > [!NOTE] 65 | > On Windows systems, you should be able to double-click the zip archive to extract the `xurlunpack3r` executable. 66 | 67 | ...move the `xurlunpack3r` binary to somewhere in your `PATH`. For example, on GNU/Linux and OS X systems: 68 | 69 | ```bash 70 | sudo mv xurlunpack3r /usr/local/bin/ 71 | ``` 72 | 73 | > [!NOTE] 74 | > Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xurlunpack3r` to their `PATH`. 75 | 76 | ### Install source (with Go installed) 77 | 78 | Before you install from source, you need to make sure that Go is installed on your system. You can install Go by following the official instructions for your operating system. For this, we will assume that Go is already installed. 79 | 80 | #### `go install ...` 81 | 82 | ```bash 83 | go install -v github.com/hueristiq/xurlunpack3r/cmd/xurlunpack3r@latest 84 | ``` 85 | 86 | #### `go build ...` the development version 87 | 88 | * Clone the repository 89 | 90 | ```bash 91 | git clone https://github.com/hueristiq/xurlunpack3r.git 92 | ``` 93 | 94 | * Build the utility 95 | 96 | ```bash 97 | cd xurlunpack3r/cmd/xurlunpack3r && \ 98 | go build . 99 | ``` 100 | 101 | * Move the `xurlunpack3r` binary to somewhere in your `PATH`. For example, on GNU/Linux and OS X systems: 102 | 103 | ```bash 104 | sudo mv xurlunpack3r /usr/local/bin/ 105 | ``` 106 | 107 | Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xurlunpack3r` to their `PATH`. 108 | 109 | 110 | > [!CAUTION] 111 | > While the development version is a good way to take a peek at `xurlunpack3r`'s latest features before they get released, be aware that it may have bugs. Officially released versions will generally be more stable. 112 | 113 | ## Usage 114 | 115 | To display help message for xurlunpack3r use the `-h` flag: 116 | 117 | ```bash 118 | xurlunpack3r -h 119 | ``` 120 | 121 | help message: 122 | 123 | ```text 124 | 125 | _ _ _____ 126 | __ ___ _ _ __| |_ _ _ __ _ __ __ _ ___| | _|___ / _ __ 127 | \ \/ / | | | '__| | | | | '_ \| '_ \ / _` |/ __| |/ / |_ \| '__| 128 | > <| |_| | | | | |_| | | | | |_) | (_| | (__| < ___) | | 129 | /_/\_\\__,_|_| |_|\__,_|_| |_| .__/ \__,_|\___|_|\_\____/|_| 130 | |_| v0.1.0 131 | 132 | USAGE: 133 | xurlunpack3r [MODE] [FORMATSTRING] [OPTIONS] 134 | 135 | MODES: 136 | domains the hostname (e.g. sub.example.com) 137 | apexes the apex domain (e.g. example.com from sub.example.com) 138 | paths the request path (e.g. /users) 139 | query `key=value` pairs from the query string (one per line) 140 | params keys from the query string (one per line) 141 | values query string values (one per line) 142 | format custom format (see below) 143 | 144 | FORMAT DIRECTIVES: 145 | %% a literal percent character 146 | %s the request scheme (e.g. https) 147 | %u the user info (e.g. user:pass) 148 | %d the domain (e.g. sub.example.com) 149 | %S the subdomain (e.g. sub) 150 | %r the root of domain (e.g. example) 151 | %t the TLD (e.g. com) 152 | %P the port (e.g. 8080) 153 | %p the path (e.g. /users) 154 | %e the path's file extension (e.g. jpg, html) 155 | %q the raw query string (e.g. a=1&b=2) 156 | %f the page fragment (e.g. page-section) 157 | %@ inserts an @ if user info is specified 158 | %: inserts a colon if a port is specified 159 | %? inserts a question mark if a query string exists 160 | %# inserts a hash if a fragment exists 161 | %a authority (alias for %u%@%d%:%P) 162 | 163 | INPUT: 164 | -u, --url string[] target URL 165 | -l, --list string target URLs list file path 166 | 167 | TIP: For multiple input URLs use comma(,) separated value with `-u`, 168 | specify multiple `-u`, load from file with `-l` or load from stdin. 169 | 170 | OUTPUT: 171 | --unique bool output unique values 172 | --monochrome bool display no color output 173 | -s, --silent bool stdout values only output 174 | -v, --verbose bool stdout verbose output 175 | 176 | ``` 177 | 178 | ### Examples 179 | 180 | ``` 181 | $ cat urls.txt 182 | 183 | https://sub.example.com/users?id=123&name=Sam 184 | https://sub.example.com/orgs?org=ExCo#about 185 | http://example.net/about#contact 186 | ``` 187 | 188 | #### Domains 189 | 190 | You can extract the domains from the URLs with the `domains` mode: 191 | 192 | ``` 193 | $ cat urls.txt | xurlunpack3r domains -i - 194 | 195 | sub.example.com 196 | sub.example.com 197 | example.net 198 | ``` 199 | 200 | If you don't want to output duplicate values you can use the `-u` or `--unique` flag: 201 | 202 | ``` 203 | $ cat urls.txt | xurlunpack3r domains -i - --unique 204 | sub.example.com 205 | example.net 206 | ``` 207 | 208 | The `-u`/`--unique` flag works for all modes. 209 | 210 | #### Apex Domains 211 | 212 | You can extract the apex part of the domain (e.g. the `example.com` in `http://sub.example.com`) using the `apexes` mode: 213 | 214 | ``` 215 | $ cat urls.txt | unfurl apexes -i - -u 216 | example.com 217 | example.net 218 | ``` 219 | 220 | #### Paths 221 | 222 | ``` 223 | $ cat urls.txt | xurlunpack3r paths -i - 224 | 225 | /users 226 | /orgs 227 | /about 228 | ``` 229 | 230 | #### Query String Key/Value Pairs 231 | 232 | ``` 233 | $ cat urls.txt | xurlunpack3r query -i - 234 | 235 | id=123 236 | name=Sam 237 | org=ExCo 238 | ``` 239 | 240 | #### Query String Keys (Parameters) 241 | 242 | ``` 243 | $ cat urls.txt | xurlunpack3r params -i - 244 | 245 | id 246 | name 247 | org 248 | ``` 249 | 250 | #### Query String Values 251 | 252 | ``` 253 | $ cat urls.txt | xurlunpack3r values -i - 254 | 255 | 123 256 | Sam 257 | ExCo 258 | ``` 259 | 260 | #### Custom Formats 261 | 262 | You can use the `format` mode to specify a custom output format: 263 | 264 | ``` 265 | $ cat urls.txt | xurlunpack3r format %d%p -i - 266 | 267 | sub.example.com/users 268 | sub.example.com/orgs 269 | example.net/about 270 | ``` 271 | 272 | The available format directives are: 273 | 274 | ``` 275 | %% A literal percent character 276 | %s The request scheme (e.g. https) 277 | %u The user info (e.g. user:pass) 278 | %d The domain (e.g. sub.example.com) 279 | %S The subdomain (e.g. sub) 280 | %r The root of domain (e.g. example) 281 | %t The TLD (e.g. com) 282 | %P The port (e.g. 8080) 283 | %p The path (e.g. /users) 284 | %e The path's file extension (e.g. jpg, html) 285 | %q The raw query string (e.g. a=1&b=2) 286 | %f The page fragment (e.g. page-section) 287 | %@ Inserts an @ if user info is specified 288 | %: Inserts a colon if a port is specified 289 | %? Inserts a question mark if a query string exists 290 | %# Inserts a hash if a fragment exists 291 | %a Authority (alias for %u%@%d%:%P) 292 | ``` 293 | 294 | > For more format directives, checkout the help message `xurlunpack3r -h` under `Format Directives`. 295 | 296 | Any characters that don't match a format directive remain untouched: 297 | 298 | ``` 299 | $ cat urls.txt | xurlunpack3r format "%d (%s)" -i - -u 300 | 301 | sub.example.com (https) 302 | example.net (http) 303 | ``` 304 | 305 | **Note** that if a URL does not include the data requested, there will be no output for that URL: 306 | 307 | ``` 308 | $ echo http://example.com | xurlunpack3r format "%P" -i - 309 | 310 | $ echo http://example.com:8080 | xurlunpack3r format "%P" -i - 311 | 8080 312 | ``` 313 | 314 | ## Contributing 315 | 316 | Feel free to submit [Pull Requests](https://github.com/hueristiq/xurlunpack3r/pulls) or report [Issues](https://github.com/hueristiq/xurlunpack3r/issues). For more details, check out the [contribution guidelines](https://github.com/hueristiq/xurlunpack3r/blob/master/CONTRIBUTING.md). 317 | 318 | Huge thanks to the [contributors](https://github.com/hueristiq/xurlunpack3r/graphs/contributors) thus far! 319 | 320 | ![contributors](https://contrib.rocks/image?repo=hueristiq/xurlunpack3r&max=500) 321 | 322 | ## Licensing 323 | 324 | This package is licensed under the [MIT license](https://opensource.org/license/mit). You are free to use, modify, and distribute it, as long as you follow the terms of the license. You can find the full license text in the repository - [Full MIT license text](https://github.com/hueristiq/xurlunpack3r/blob/master/LICENSE). -------------------------------------------------------------------------------- /cmd/xurlunpack3r/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "os" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/hueristiq/xurlunpack3r/internal/configuration" 12 | "github.com/hueristiq/xurlunpack3r/internal/input" 13 | "github.com/logrusorgru/aurora/v4" 14 | "github.com/spf13/pflag" 15 | "go.source.hueristiq.com/logger" 16 | "go.source.hueristiq.com/logger/formatter" 17 | "go.source.hueristiq.com/logger/levels" 18 | "go.source.hueristiq.com/url/parser" 19 | ) 20 | 21 | type Extractor func(URL *parser.URL, format string) []string 22 | 23 | var ( 24 | au = aurora.New(aurora.WithColors(true)) 25 | 26 | inputURLs []string 27 | inputURLsListFilePath string 28 | 29 | unique bool 30 | monochrome bool 31 | silent bool 32 | verbose bool 33 | ) 34 | 35 | func init() { 36 | pflag.StringSliceVarP(&inputURLs, "url", "u", []string{}, "") 37 | pflag.StringVarP(&inputURLsListFilePath, "list", "l", "", "") 38 | 39 | pflag.BoolVar(&unique, "unique", false, "") 40 | pflag.BoolVarP(&monochrome, "monochrome", "m", false, "") 41 | pflag.BoolVarP(&silent, "silent", "s", false, "") 42 | pflag.BoolVarP(&verbose, "verbose", "v", false, "") 43 | 44 | pflag.Usage = func() { 45 | logger.Info().Label("").Msg(configuration.BANNER(au)) 46 | 47 | h := "USAGE:\n" 48 | h += fmt.Sprintf(" %s [MODE] [FORMATSTRING] [OPTIONS]\n", configuration.NAME) 49 | 50 | h += "\nMODES:\n" 51 | h += " domains the hostname (e.g. sub.example.com)\n" 52 | h += " apexes the apex domain (e.g. example.com from sub.example.com)\n" 53 | h += " paths the request path (e.g. /users)\n" 54 | h += " query `key=value` pairs from the query string (one per line)\n" 55 | h += " params keys from the query string (one per line)\n" 56 | h += " values query string values (one per line)\n" 57 | h += " format custom format (see below)\n" 58 | 59 | h += "\nFORMAT DIRECTIVES:\n" 60 | h += " %% a literal percent character\n" 61 | h += " %s the request scheme (e.g. https)\n" 62 | h += " %u the user info (e.g. user:pass)\n" 63 | h += " %d the domain (e.g. sub.example.com)\n" 64 | h += " %S the subdomain (e.g. sub)\n" 65 | h += " %r the root of domain (e.g. example)\n" 66 | h += " %t the TLD (e.g. com)\n" 67 | h += " %P the port (e.g. 8080)\n" 68 | h += " %p the path (e.g. /users)\n" 69 | h += " %e the path's file extension (e.g. jpg, html)\n" 70 | h += " %q the raw query string (e.g. a=1&b=2)\n" 71 | h += " %f the page fragment (e.g. page-section)\n" 72 | h += " %@ inserts an @ if user info is specified\n" 73 | h += " %: inserts a colon if a port is specified\n" 74 | h += " %? inserts a question mark if a query string exists\n" 75 | h += " %# inserts a hash if a fragment exists\n" 76 | h += " %a authority (alias for %u%@%d%:%P)\n" 77 | 78 | h += "\nINPUT:\n" 79 | h += " -u, --url string[] target URL\n" 80 | h += " -l, --list string target URLs list file path\n" 81 | 82 | h += "\nTIP: For multiple input URLs use comma(,) separated value with `-u`,\n" 83 | h += " specify multiple `-u`, load from file with `-l` or load from stdin.\n" 84 | 85 | h += "\nOUTPUT:\n" 86 | h += " --unique bool output unique values\n" 87 | h += " --monochrome bool display no color output\n" 88 | h += " -s, --silent bool stdout values only output\n" 89 | h += " -v, --verbose bool stdout verbose output\n" 90 | 91 | logger.Info().Label("").Msg(h) 92 | logger.Print().Msg("") 93 | } 94 | 95 | pflag.Parse() 96 | 97 | logger.DefaultLogger.SetFormatter(formatter.NewConsoleFormatter(&formatter.ConsoleFormatterConfiguration{ 98 | Colorize: !monochrome, 99 | })) 100 | 101 | if verbose { 102 | logger.DefaultLogger.SetMaxLogLevel(levels.LevelDebug) 103 | } 104 | 105 | if silent { 106 | logger.DefaultLogger.SetMaxLogLevel(levels.LevelSilent) 107 | } 108 | 109 | au = aurora.New(aurora.WithColors(!monochrome)) 110 | } 111 | 112 | func main() { 113 | logger.Info().Label("").Msg(configuration.BANNER(au)) 114 | 115 | var err error 116 | 117 | // Set the mode (e.g., "domains", "paths") and format string from CLI arguments. 118 | mode := pflag.Arg(0) 119 | fmtStr := pflag.Arg(1) 120 | 121 | // Map each mode to its corresponding extractor function. 122 | procFn, ok := map[string]Extractor{ 123 | "domains": Domains, 124 | "apexes": Apexes, 125 | "paths": Paths, 126 | "query": Query, 127 | "params": Parameters, 128 | "values": Values, 129 | "format": Format, 130 | }[mode] 131 | 132 | // If an unknown mode is provided, log a fatal error and exit 133 | if !ok { 134 | logger.Fatal().Msgf("unknown mode: %s", mode) 135 | } 136 | 137 | URLs := make(chan string) 138 | 139 | go func() { 140 | defer close(URLs) 141 | 142 | if len(inputURLs) > 0 { 143 | for _, URL := range inputURLs { 144 | URLs <- URL 145 | } 146 | } 147 | 148 | if inputURLsListFilePath != "" { 149 | var file *os.File 150 | 151 | file, err = os.Open(inputURLsListFilePath) 152 | if err != nil { 153 | logger.Error().Msg(err.Error()) 154 | } 155 | 156 | scanner := bufio.NewScanner(file) 157 | 158 | for scanner.Scan() { 159 | URL := scanner.Text() 160 | 161 | if URL != "" { 162 | URLs <- URL 163 | } 164 | } 165 | 166 | if err = scanner.Err(); err != nil { 167 | logger.Error().Msg(err.Error()) 168 | } 169 | 170 | file.Close() 171 | } 172 | 173 | if input.HasStdin() { 174 | scanner := bufio.NewScanner(os.Stdin) 175 | 176 | for scanner.Scan() { 177 | URL := scanner.Text() 178 | 179 | if URL != "" { 180 | URLs <- URL 181 | } 182 | } 183 | 184 | if err = scanner.Err(); err != nil { 185 | logger.Error().Msg(err.Error()) 186 | } 187 | } 188 | }() 189 | 190 | wg := &sync.WaitGroup{} 191 | 192 | p := parser.NewURLParser(parser.URLParserWithDefaultScheme("http")) 193 | 194 | seen := &sync.Map{} 195 | 196 | for URL := range URLs { 197 | wg.Add(1) 198 | 199 | go func(URL string) { 200 | defer wg.Done() 201 | 202 | parsed, err := p.Parse(URL) 203 | if err != nil { 204 | logger.Error().Msgf("parse failure: %s", err.Error()) 205 | 206 | return 207 | } 208 | 209 | for _, value := range procFn(parsed, fmtStr) { 210 | if value == "" { 211 | continue 212 | } 213 | 214 | if unique { 215 | _, loaded := seen.LoadOrStore(value, struct{}{}) 216 | if loaded { 217 | continue 218 | } 219 | } 220 | 221 | logger.Print().Msg(value) 222 | } 223 | }(URL) 224 | } 225 | 226 | wg.Wait() 227 | } 228 | 229 | func Format(u *parser.URL, f string) []string { 230 | out := &bytes.Buffer{} 231 | 232 | inFormat := false 233 | 234 | for _, r := range f { 235 | if r == '%' && !inFormat { 236 | inFormat = true 237 | 238 | continue 239 | } 240 | 241 | if !inFormat { 242 | out.WriteRune(r) 243 | 244 | continue 245 | } 246 | 247 | switch r { 248 | // a literal percent rune 249 | case '%': 250 | out.WriteByte('%') 251 | 252 | // the scheme; e.g. http 253 | case 's': 254 | out.WriteString(u.Scheme) 255 | 256 | // the userinfo; e.g. user:pass 257 | case 'u': 258 | if u.User != nil { 259 | out.WriteString(u.User.String()) 260 | } 261 | 262 | // the domain; e.g. sub.example.com 263 | case 'd': 264 | out.WriteString(u.Hostname()) 265 | 266 | // the port; e.g. 8080 267 | case 'P': 268 | out.WriteString(u.Port()) 269 | 270 | // the subdomain; e.g. www 271 | case 'S': 272 | out.WriteString(u.Domain.Subdomain) 273 | 274 | // the root; e.g. example 275 | case 'r': 276 | out.WriteString(u.Domain.SLD) 277 | 278 | // the tld; e.g. com 279 | case 't': 280 | out.WriteString(u.Domain.TLD) 281 | 282 | // the path; e.g. /users 283 | case 'p': 284 | out.WriteString(u.EscapedPath()) 285 | 286 | // the paths's file extension 287 | case 'e': 288 | paths := strings.Split(u.EscapedPath(), "/") 289 | if len(paths) > 1 { 290 | parts := strings.Split(paths[len(paths)-1], ".") 291 | if len(parts) > 1 { 292 | out.WriteString(parts[len(parts)-1]) 293 | } 294 | } else { 295 | parts := strings.Split(u.EscapedPath(), ".") 296 | if len(parts) > 1 { 297 | out.WriteString(parts[len(parts)-1]) 298 | } 299 | } 300 | 301 | // the query string; e.g. one=1&two=2 302 | case 'q': 303 | out.WriteString(u.RawQuery) 304 | 305 | // the fragment / hash value; e.g. section-1 306 | case 'f': 307 | out.WriteString(u.Fragment) 308 | 309 | // an @ if user info is specified 310 | case '@': 311 | if u.User != nil { 312 | out.WriteByte('@') 313 | } 314 | 315 | // a colon if a port is specified 316 | case ':': 317 | if u.Port() != "" { 318 | out.WriteByte(':') 319 | } 320 | 321 | // a question mark if there's a query string 322 | case '?': 323 | if u.RawQuery != "" { 324 | out.WriteByte('?') 325 | } 326 | 327 | // a hash if there is a fragment 328 | case '#': 329 | if u.Fragment != "" { 330 | out.WriteByte('#') 331 | } 332 | 333 | // the authority; e.g. user:pass@example.com:8080 334 | case 'a': 335 | out.WriteString(Format(u, "%u%@%d%:%P")[0]) 336 | 337 | // default to literal 338 | default: 339 | // output untouched 340 | out.WriteByte('%') 341 | out.WriteRune(r) 342 | } 343 | 344 | inFormat = false 345 | } 346 | 347 | return []string{out.String()} 348 | } 349 | 350 | func Domains(u *parser.URL, _ string) []string { 351 | return Format(u, "%d") 352 | } 353 | 354 | func Apexes(u *parser.URL, _ string) []string { 355 | return Format(u, "%r.%t") 356 | } 357 | 358 | func Paths(u *parser.URL, _ string) []string { 359 | return Format(u, "%p") 360 | } 361 | 362 | func Query(u *parser.URL, _ string) []string { 363 | out := make([]string, 0) 364 | 365 | for key, vals := range u.Query() { 366 | for _, val := range vals { 367 | out = append(out, fmt.Sprintf("%s=%s", key, val)) 368 | } 369 | } 370 | 371 | return out 372 | } 373 | 374 | func Parameters(u *parser.URL, _ string) []string { 375 | out := make([]string, 0) 376 | 377 | for key := range u.Query() { 378 | out = append(out, key) 379 | } 380 | 381 | return out 382 | } 383 | 384 | func Values(u *parser.URL, _ string) []string { 385 | out := make([]string, 0) 386 | 387 | for _, value := range u.Query() { 388 | out = append(out, value...) 389 | } 390 | 391 | return out 392 | } 393 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hueristiq/xurlunpack3r 2 | 3 | go 1.23.3 4 | 5 | require ( 6 | github.com/logrusorgru/aurora/v4 v4.0.0 7 | github.com/spf13/pflag v1.0.6 8 | go.source.hueristiq.com/logger v0.0.0-20250224134018-212c4546deb7 9 | go.source.hueristiq.com/url v0.0.0-20250212180013-cf9654cb75ee 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= 4 | github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= 5 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 6 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 8 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 9 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 10 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 11 | go.source.hueristiq.com/logger v0.0.0-20250224134018-212c4546deb7 h1:9lCwrERLiRqqT2agYR95k5HJQCcF/rZzI2GY+hL3xDw= 12 | go.source.hueristiq.com/logger v0.0.0-20250224134018-212c4546deb7/go.mod h1:AOQwgh2c5MRAy87IVpoJ9ZVE/XaSrMLcAVTRKQGOazc= 13 | go.source.hueristiq.com/url v0.0.0-20250212180013-cf9654cb75ee h1:MrTxNek9Ho0Jkbk5fmUQ4N5IXjGFgAoHqSY1asZdMf8= 14 | go.source.hueristiq.com/url v0.0.0-20250212180013-cf9654cb75ee/go.mod h1:AHuHYgNJAvUCk70o7Fm3GFlXZGGTrAL4L7APZZiKyHk= 15 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 16 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 17 | -------------------------------------------------------------------------------- /internal/configuration/configuration.go: -------------------------------------------------------------------------------- 1 | package configuration 2 | 3 | import "github.com/logrusorgru/aurora/v4" 4 | 5 | const ( 6 | NAME = "xurlunpack3r" 7 | VERSION = "0.1.0" 8 | ) 9 | 10 | var BANNER = func(au *aurora.Aurora) (banner string) { 11 | banner = au.Sprintf( 12 | au.BrightBlue(` 13 | _ _ _____ 14 | __ ___ _ _ __| |_ _ _ __ _ __ __ _ ___| | _|___ / _ __ 15 | \ \/ / | | | '__| | | | | '_ \| '_ \ / _`+"`"+` |/ __| |/ / |_ \| '__| 16 | > <| |_| | | | | |_| | | | | |_) | (_| | (__| < ___) | | 17 | /_/\_\\__,_|_| |_|\__,_|_| |_| .__/ \__,_|\___|_|\_\____/|_| 18 | |_| %s`).Bold(), 19 | au.BrightRed("v"+VERSION).Bold().Italic(), 20 | ) + "\n\n" 21 | 22 | return 23 | } 24 | -------------------------------------------------------------------------------- /internal/input/input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import "os" 4 | 5 | func HasStdin() bool { 6 | stat, err := os.Stdin.Stat() 7 | if err != nil { 8 | return false 9 | } 10 | 11 | isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 12 | isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 13 | 14 | return isPipedFromChrDev || isPipedFromFIFO 15 | } 16 | -------------------------------------------------------------------------------- /urls.txt: -------------------------------------------------------------------------------- 1 | https://sub.example.com/users?id=123&name=Sam 2 | https://sub.example.com/orgs?org=ExCo#about 3 | http://example.net/about#contact 4 | example.net/about#contact --------------------------------------------------------------------------------