├── .github ├── FUNDING.yml └── workflows │ └── release.yaml ├── .goreleaser.yaml ├── LICENSE ├── README.md └── main.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: ["dwisiswant0"] 2 | custom: ["https://paypal.me/dw1s", "https://saweria.co/dwisiswant0", "https://unstoppabledomains.com/d/dwisiswant0.crypto"] 3 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | create: 4 | tags: 5 | - v* 6 | 7 | jobs: 8 | release: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: "Check out code" 12 | uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | 16 | - name: "Set up Go" 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.11 20 | 21 | - name: "Create release on GitHub" 22 | uses: goreleaser/goreleaser-action@v2 23 | env: 24 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 25 | with: 26 | args: "release --rm-dist" 27 | version: latest 28 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | builds: 2 | - binary: unew 3 | main: main.go 4 | goos: 5 | - linux 6 | - windows 7 | - darwin 8 | goarch: 9 | - amd64 10 | - 386 11 | - arm 12 | - arm64 13 | 14 | archives: 15 | - id: tgz 16 | format: tar.gz 17 | replacements: 18 | darwin: macOS 19 | format_overrides: 20 | - goos: windows 21 | format: zip 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 dwisiswant0 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # unew 2 | 3 | **u**_**(rl)**_**new** — A tool for append URLs, skipping duplicates & combine parameters. Inspired by [anew](https://github.com/tomnomnom/anew) & [qsreplace](https://github.com/tomnomnom/qsreplace). 4 | 5 | ## Usage 6 | 7 | ```bash 8 | ▶ cat urls.txt | unew 9 | # or 10 | ▶ unew urls.txt 11 | # or, save the results 12 | ▶ unew urls.txt output.txt 13 | ``` 14 | 15 | ### Flags 16 | 17 | Usage of `unew`: 18 | ``` 19 | -combine 20 | Combine parameters 21 | -r string 22 | Replace parameters value 23 | -skip-path value 24 | Skip specific paths (regExp pattern) 25 | ``` 26 | 27 | ## Install 28 | 29 | with [Go](https://golang.org/doc/install): 30 | 31 | ```bash 32 | ▶ go get -u github.com/dwisiswant0/unew 33 | ``` 34 | 35 | ## Workaround 36 | 37 | If you have a `urls.txt` list as 38 | 39 | ```txt 40 | https://twitter.com/dwisiswant0?href=evilzone.org 41 | https://twitter.com/dwisiswant0 42 | https://twitter.com/dwisiswant0?ref=github&utm_source=github 43 | https://twitter.com/dwisiswant0/status/1305022512590278656 44 | https://www.linkedin.com/in/dwisiswanto/ 45 | https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id 46 | https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id&utm_medium=github 47 | ``` 48 | 49 | ### Regular 50 | 51 | Sample workarounds: 52 | 53 | ```bash 54 | ▶ cat urls.txt | unew 55 | https://twitter.com/dwisiswant0?href=evilzone.org 56 | https://www.linkedin.com/in/dwisiswanto/ 57 | ``` 58 | 59 | If the list contains multiple URLs with same path, it will save the first one and its parameters. 60 | 61 | ### Combining parameters 62 | 63 | But you can combine parameters if the same path exists by using `-combine` flag. 64 | 65 | ```bash 66 | ▶ cat urls.txt | unew -combine 67 | https://twitter.com/dwisiswant0?href=evilzone.org&ref=github&utm_source=github 68 | https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id&utm_medium=github 69 | ``` 70 | 71 | ### Query replacers 72 | 73 | Use the `-r` flag if you want to change the value of all parameters. 74 | 75 | ```bash 76 | ▶ cat urls.txt | unew -combine -r "/etc/passwd" 77 | https://twitter.com/dwisiswant0?href=%2Fetc%2Fpasswd&ref=%2Fetc%2Fpasswd&utm_source=%2Fetc%2Fpasswd 78 | https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=%2Fetc%2Fpasswd&utm_medium=%2Fetc%2Fpasswd 79 | ``` 80 | 81 | ### Skipping paths 82 | 83 | In case if you want to pass specific/multiple URL paths, you can use `-skip-path` flag for it _(can be set multiple times)_. But, you have to write it with regExp pattern. 84 | 85 | ```bash 86 | ▶ cat urls.txt | unew -skip-path "^/[\w]+/status/[0-9]+" -skip-path "/in/[\w]+" 87 | https://twitter.com/dwisiswant0?href=evilzone.org 88 | ``` -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "net/url" 8 | "os" 9 | "regexp" 10 | "strings" 11 | ) 12 | 13 | type skipPaths []string 14 | 15 | var ( 16 | combine bool 17 | outfile *os.File 18 | sc *bufio.Scanner 19 | skipPath skipPaths 20 | replace, outText, fr, fs string 21 | ) 22 | 23 | func (s *skipPaths) String() string { 24 | return "" 25 | } 26 | 27 | func (s *skipPaths) Set(value string) error { 28 | *s = append(*s, value) 29 | return nil 30 | } 31 | 32 | func init() { 33 | flag.StringVar(&replace, "r", "", "Replace parameters value") 34 | flag.BoolVar(&combine, "combine", false, "Combine parameters") 35 | flag.Var(&skipPath, "skip-path", "Skip specific paths (regExp pattern)") 36 | flag.Parse() 37 | 38 | fr := flag.Arg(0) 39 | 40 | if isStdin() { 41 | sc = bufio.NewScanner(os.Stdin) 42 | } else if fr != "" { 43 | r, err := os.Open(fr) 44 | if err == nil { 45 | sc = bufio.NewScanner(r) 46 | } 47 | } else { 48 | os.Exit(1) 49 | } 50 | } 51 | 52 | func main() { 53 | urls := make(map[string]string) 54 | 55 | fs := flag.Arg(1) 56 | if fs != "" { 57 | outfile, _ = os.OpenFile(fs, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) 58 | defer outfile.Close() 59 | } 60 | 61 | for sc.Scan() { 62 | u, err := url.ParseRequestURI(sc.Text()) 63 | if err != nil { 64 | continue 65 | } 66 | 67 | if matchPath(skipPath, u) { 68 | continue 69 | } 70 | 71 | b := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Path) 72 | 73 | if _, d := urls[b]; d { 74 | if combine { 75 | q := []string{urls[b], u.RawQuery} 76 | u.RawQuery = remDup(strings.Join(q, "&")) 77 | } else { 78 | continue 79 | } 80 | } else { 81 | urls[b] = u.RawQuery 82 | } 83 | 84 | if replace != "" { 85 | u.RawQuery = qsReplace(u.Query(), replace) 86 | } 87 | 88 | if !combine { 89 | outText = fmt.Sprintf("%s%s\n", b, qMark(u.RawQuery)) 90 | fmt.Printf("%s", outText) 91 | if fs != "" { 92 | fmt.Fprintf(outfile, "%s", outText) 93 | } 94 | 95 | continue 96 | } 97 | 98 | urls[b] = u.RawQuery 99 | } 100 | 101 | if !combine { 102 | return 103 | } 104 | 105 | for k, v := range urls { 106 | outText = fmt.Sprintf("%s%s\n", k, qMark(v)) 107 | fmt.Printf("%s", outText) 108 | if fs != "" { 109 | fmt.Fprintf(outfile, "%s", outText) 110 | } 111 | } 112 | } 113 | 114 | func isStdin() bool { 115 | f, e := os.Stdin.Stat() 116 | if e != nil { 117 | return false 118 | } 119 | 120 | if f.Mode()&os.ModeNamedPipe == 0 { 121 | return false 122 | } 123 | 124 | return true 125 | } 126 | 127 | func qMark(q string) string { 128 | if q == "" { 129 | return "" 130 | } 131 | 132 | return "?" + q 133 | } 134 | 135 | func remDup(q string) string { 136 | qs := url.Values{} 137 | ps, _ := url.ParseQuery(q) 138 | 139 | for p, v := range ps { 140 | for range v { 141 | v = v[:1:1] 142 | qs.Set(p, v[0]) 143 | } 144 | } 145 | 146 | return qs.Encode() 147 | } 148 | 149 | func qsReplace(q url.Values, r string) string { 150 | qs := url.Values{} 151 | for p := range q { 152 | qs.Set(p, r) 153 | } 154 | 155 | return qs.Encode() 156 | } 157 | 158 | func matchPath(s []string, u *url.URL) bool { 159 | for _, p := range s { 160 | m, e := regexp.MatchString(p, u.Path) 161 | if e != nil { 162 | continue 163 | } 164 | 165 | if m { 166 | return true 167 | } 168 | } 169 | 170 | return false 171 | } 172 | --------------------------------------------------------------------------------