├── README.md
└── cmd
    └── parameterMiner
        └── main.go


/README.md:
--------------------------------------------------------------------------------
 1 | # ParameterMiner
 2 | Built on a lazy Sunday after seeing this tweet (https://twitter.com/intigriti/status/1272145863868104705?s=20) I present to you, ParameterMiner! Pipe in a list of javascript urls and ParameterMiner pulls all the variable names.  
 3 | 
 4 | 
 5 | #### USAGE:
 6 | 
 7 | ````
 8 |   -length int
 9 |         Minimum length variable to collect
10 |   -s    saves output to file named from base url of source
11 |   -save
12 |         saves output to file named from base url of source
13 | ````
14 | 
15 | #### Example with single URL:
16 | 
17 | ````echo https://news.ycombinator.com/hn.js?qjUdg9dZheJfdx2Zo5qF | paramMiner.exe````
18 | 
19 | 
20 | #### Example output:
21 | 
22 | ````
23 | j
24 | on
25 | n1
26 | rks
27 | a
28 | id
29 | unv
30 | ks
31 | el
32 | sp
33 | n
34 | up
35 | trs
36 | req
37 | url
38 | i
39 | s
40 | pair
41 | next
42 | ````
43 | 
44 | #### Example with single URL and Length Filter:
45 | 
46 | ````echo https://news.ycombinator.com/hn.js?qjUdg9dZheJfdx2Zo5qF | paramMiner.exe -l 1````
47 | 
48 | 
49 | #### Example Output with single URL and Length Filter:
50 | 
51 | ````
52 | unv
53 | ks
54 | req
55 | next
56 | id
57 | el
58 | pair
59 | url
60 | up
61 | trs
62 | n1
63 | sp
64 | rks
65 | on
66 | ````
67 | 


--------------------------------------------------------------------------------
/cmd/parameterMiner/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"bufio"
 5 | 	"flag"
 6 | 	"fmt"
 7 | 	"log"
 8 | 	"net/http"
 9 | 	"net/url"
10 | 	"os"
11 | 	"regexp"
12 | 	"strings"
13 | 	"time"
14 | )
15 | 
16 | type Results struct {
17 | 	Results map[string]bool
18 | 	Source string
19 | }
20 | 
21 | const regexStr  = " ([$A-Z\\_a-z0-9]+\\s*=[^=])"
22 | 
23 | func main()  {
24 | 	scanTime := strings.Replace(time.Now().Format("15:04:05"),":","-",-1)
25 | 	saveResultsFlag := flag.Bool("s", false, "saves output to file named from base url of source")
26 | 	saveResultsLongFlag := flag.Bool("save", false, "saves output to file named from base url of source")
27 | 	paramLengthFilterFlag := flag.Int("length",0, "Minimum length variable to collect")
28 | 	flag.Parse()
29 | 	r, _ := regexp.Compile(regexStr)
30 | 	scan := bufio.NewScanner(os.Stdin)
31 | 	for scan.Scan() {
32 | 		extractParams(r, scan.Text(),*saveResultsFlag||*saveResultsLongFlag, scanTime, *paramLengthFilterFlag)
33 | 	}
34 | 
35 | 
36 | 
37 | }
38 | 
39 | func extractParams(r *regexp.Regexp, source string, saveOutput bool, scanTime string, lengthFilter int) {
40 | 	resultsMap := &Results{ make(map[string]bool),source}
41 | 	var client http.Client
42 | 	resp, err := client.Get(source)
43 | 	if err != nil {
44 | 		log.Fatal(err)
45 | 	}
46 | 	defer resp.Body.Close()
47 | 	if resp.StatusCode == http.StatusOK {
48 | 		scanner := bufio.NewScanner(resp.Body)
49 | 		for scanner.Scan() {
50 | 			line := scanner.Text()
51 | 			if strings.Contains(line, "var ") {
52 | 				results := r.FindAllString(line, -1)
53 | 				if len(results) != 0 {
54 | 					for _, variable := range r.FindAllString(line, -1) {
55 | 						cleaned := strings.TrimSpace(strings.Split(variable, "=")[0])
56 | 						if len(cleaned) > lengthFilter {
57 | 							if !resultsMap.Results[cleaned] {
58 | 								resultsMap.Results[cleaned] = true
59 | 							}
60 | 						}
61 | 					}
62 | 				}
63 | 			}
64 | 		}
65 | 		parsedSource, _ := url.Parse(source)
66 | 		count := 0
67 | 		if !saveOutput {
68 | 			for k := range resultsMap.Results {
69 | 				if count == len(resultsMap.Results)-1 {
70 | 					fmt.Print(k)
71 | 				} else {
72 | 					fmt.Println(k)
73 | 				}
74 | 				count+=1
75 | 			}
76 | 		} else {
77 | 			f, err := os.OpenFile(parsedSource.Host+"_"+scanTime,
78 | 				os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
79 | 			if err != nil {
80 | 				log.Println(err)
81 | 			}
82 | 			defer f.Close()
83 | 			for k := range resultsMap.Results {
84 | 				if count == len(resultsMap.Results)-1 {
85 | 					if _, err := f.WriteString(k); err != nil {
86 | 						log.Fatalln(err)
87 | 					}
88 | 				} else {
89 | 					if _, err := f.WriteString(k+"\n"); err != nil {
90 | 						log.Fatalln(err)
91 | 					}
92 | 				}
93 | 				count+=1
94 | 			}
95 | 		}
96 | 	}
97 | }


--------------------------------------------------------------------------------