├── gitpull
├── go.mod
├── go.sum
├── README.md
└── gitpull.go
/gitpull:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KingOfBugbounty/gitPullScrapper/HEAD/gitpull
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module gitPullScrapper
2 |
3 | go 1.22.2
4 |
5 | require (
6 | github.com/google/go-github v17.0.0+incompatible // indirect
7 | github.com/google/go-querystring v1.1.0 // indirect
8 | golang.org/x/oauth2 v0.22.0 // indirect
9 | gopkg.in/yaml.v3 v3.0.1 // indirect
10 | )
11 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
2 | github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
3 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
4 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
5 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
6 | golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
7 | golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
8 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
10 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
11 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
12 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 🚀 gitPullScrapper
2 |
3 |
4 | 
5 |
6 |
7 |
8 |
9 | ## 🛠️ About the Script
10 |
11 | This Go script allows you to download all the YAML templates from the [Nuclei Templates](https://github.com/projectdiscovery/nuclei-templates) repository that haven't yet made it to the master branch. 🚀
12 |
13 | ### 🔥 Why Use This Script?
14 |
15 | By using this tool, you'll gain access to the latest templates **before** they're officially released. Stay ahead of the game and use the most up-to-date templates before anyone else! 💪
16 |
17 | ## 📦 Installation
18 |
19 | 1. **Export Your GitHub Token:**
20 |
21 | To enable the script to download templates, you'll need to export your GitHub token. Add the following line to your `.bashrc` or `.zshrc` file:
22 |
23 | ```bash
24 | export GITHUB_TOKEN=your_github_token_here
25 | source ~/.bashrc
26 |
27 | 2. **Build go:**
28 | ```bash
29 | go build gitpull.go
30 | ```
31 |
32 | ## 📦 Run
33 | ```bash
34 | ./gitpull
35 | ```
36 |
37 | 
38 |
39 |
40 | 
41 |
42 |
--------------------------------------------------------------------------------
/gitpull.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "io/ioutil"
7 | "log"
8 | "net/http"
9 | "os"
10 | "path/filepath"
11 | "strings"
12 |
13 | "github.com/google/go-github/github"
14 | "golang.org/x/oauth2"
15 | )
16 |
17 | const (
18 | repoOwner = "projectdiscovery"
19 | repoName = "nuclei-templates"
20 | saveDir = "nuclei_templates"
21 | )
22 |
23 | func main() {
24 | token := os.Getenv("GITHUB_TOKEN")
25 | if token == "" {
26 | log.Fatal("GITHUB_TOKEN environment variable is not set")
27 | }
28 |
29 | ctx := context.Background()
30 | ts := oauth2.StaticTokenSource(
31 | &oauth2.Token{AccessToken: token},
32 | )
33 | tc := oauth2.NewClient(ctx, ts)
34 | client := github.NewClient(tc)
35 |
36 | opts := &github.PullRequestListOptions{
37 | State: "open",
38 | }
39 |
40 | prs, _, err := client.PullRequests.List(ctx, repoOwner, repoName, opts)
41 | if err != nil {
42 | log.Fatalf("Error listing pull requests: %v", err)
43 | }
44 |
45 | fmt.Printf("Found %d open pull requests\n", len(prs))
46 | if len(prs) == 0 {
47 | return
48 | }
49 |
50 | os.MkdirAll(saveDir, os.ModePerm)
51 |
52 | for _, pr := range prs {
53 | files, _, err := client.PullRequests.ListFiles(ctx, repoOwner, repoName, *pr.Number, nil)
54 | if err != nil {
55 | log.Printf("Error listing files for PR #%d: %v", *pr.Number, err)
56 | continue
57 | }
58 |
59 | for _, file := range files {
60 | if strings.HasSuffix(*file.Filename, ".yaml") {
61 | err := downloadFile(*file.RawURL, filepath.Join(saveDir, filepath.Base(*file.Filename)))
62 | if err != nil {
63 | log.Printf("Error downloading file %s: %v", *file.Filename, err)
64 | } else {
65 | fmt.Printf("Downloaded file: %s\n", *file.Filename)
66 | }
67 | }
68 | }
69 | }
70 | }
71 |
72 | func downloadFile(url, filepath string) error {
73 | resp, err := http.Get(url)
74 | if err != nil {
75 | return err
76 | }
77 | defer resp.Body.Close()
78 |
79 | if resp.StatusCode != http.StatusOK {
80 | return fmt.Errorf("bad status: %s", resp.Status)
81 | }
82 |
83 | data, err := ioutil.ReadAll(resp.Body)
84 | if err != nil {
85 | return err
86 | }
87 |
88 | return ioutil.WriteFile(filepath, data, 0644)
89 | }
90 |
--------------------------------------------------------------------------------