├── logo.gif ├── favicon.ico ├── README.md ├── index.tmpl └── main.go /logo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/montanaflynn/codehn/HEAD/logo.gif -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/montanaflynn/codehn/HEAD/favicon.ico -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Code HN 2 | 3 | Hacker news with only links from GitHub or GitLab. 4 | 5 | __Demo__: [code.hn](https://code.hn) 6 | 7 | ### Usage 8 | 9 | ``` 10 | $ go run main.go & 11 | $ curl localhost:8080 12 | ``` 13 | 14 | ### TODOS 15 | 16 | - Add a filter for programming language using github's own API 17 | - Use channels for results / errors from individual story API requests 18 | - Use some internal scheduler that keeps all the stories in a cache 19 | - Use brute force to ensure we have at least 30 stories for all pages 20 | -------------------------------------------------------------------------------- /index.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 108 | 109 | 110 |
111 | 134 | 147 |
148 | 149 | 150 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // codehn is a hn clone that only displays posts from github 2 | package main 3 | 4 | // lots of imports means lots of time saved 5 | import ( 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "html/template" 10 | "io/ioutil" 11 | "log" 12 | "net/http" 13 | "net/url" 14 | "strings" 15 | "sync" 16 | "time" 17 | 18 | // for "33 minutes ago" in the template 19 | humanize "github.com/dustin/go-humanize" 20 | 21 | // because the HN API is awkward and slow 22 | cache "github.com/pmylund/go-cache" 23 | ) 24 | 25 | // baseURL is the URL for the hacker news API 26 | var baseURL = "https://hacker-news.firebaseio.com/v0/" 27 | 28 | // cash rules everything around me, get the money y'all 29 | var cash *cache.Cache 30 | 31 | // we will ensure the template is valid and only load 32 | var tmpl *template.Template 33 | 34 | func init() { 35 | 36 | // cash will have default expiration time of 37 | // 30 minutes and be swept every 10 minutes 38 | cash = cache.New(30*time.Minute, 10*time.Minute) 39 | 40 | // this will panic if the index.tmpl isn't valid 41 | tmpl = template.Must(template.ParseFiles("index.tmpl")) 42 | } 43 | 44 | // story holds the response from the HN API and 45 | // two other fields I use to render the template 46 | type story struct { 47 | By string `json:"by"` 48 | Descendants int `json:"descendants"` 49 | ID int `json:"id"` 50 | Kids []int `json:"kids"` 51 | Score int `json:"score"` 52 | Time int `json:"time"` 53 | Title string `json:"title"` 54 | Type string `json:"type"` 55 | URL string `json:"url"` 56 | DomainName string 57 | HumanTime string 58 | } 59 | 60 | // stories is just a bunch of story pointers 61 | type stories []*story 62 | 63 | var storiesMutex sync.Mutex 64 | 65 | // getStories if you couldn't guess it, gets the stories 66 | func getStories(res *http.Response) (stories, error) { 67 | 68 | // this is bad! we should limit the request body size 69 | body, err := ioutil.ReadAll(res.Body) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | // get all the story keys into a slice of ints 75 | var keys []int 76 | json.Unmarshal(body, &keys) 77 | 78 | // concurrency is cool, but needs to be limited 79 | semaphore := make(chan struct{}, 10) 80 | 81 | // how we know when all our goroutines are done 82 | wg := sync.WaitGroup{} 83 | 84 | // somewhere to store all the stories when we're done 85 | var stories []*story 86 | 87 | // go over all the stories 88 | for _, key := range keys { 89 | 90 | // stop when we have 30 stories 91 | storiesMutex.Lock() 92 | storiesCnt := len(stories) 93 | storiesMutex.Unlock() 94 | if storiesCnt >= 30 { 95 | break 96 | } 97 | 98 | // sleep to avoid rate limiting from API 99 | time.Sleep(10 * time.Millisecond) 100 | 101 | // in a goroutine with the story key 102 | go func(storyKey int) { 103 | 104 | // add one to the wait group 105 | wg.Add(1) 106 | 107 | // add one to the semaphore 108 | semaphore <- struct{}{} 109 | 110 | // make sure this gets fired 111 | defer func() { 112 | 113 | // remove one from the wait group 114 | wg.Done() 115 | 116 | // remove one from the semaphore 117 | <-semaphore 118 | }() 119 | 120 | // get the story with reckless abandon for errors 121 | keyURL := fmt.Sprintf(baseURL+"item/%d.json", storyKey) 122 | resp, err := http.Get(keyURL) 123 | if err != nil { 124 | return 125 | } 126 | defer resp.Body.Close() 127 | 128 | body, err := ioutil.ReadAll(resp.Body) 129 | if err != nil { 130 | return 131 | } 132 | 133 | s := &story{} 134 | err = json.Unmarshal(body, s) 135 | if err != nil { 136 | return 137 | } 138 | 139 | // parse the url 140 | u, err := url.Parse(s.URL) 141 | if err != nil { 142 | return 143 | } 144 | 145 | // get the hostname from the url 146 | h := u.Hostname() 147 | 148 | // check if it's from github or gitlab before adding to stories 149 | if strings.Contains(h, "github") || strings.Contains(h, "gitlab") { 150 | s.HumanTime = humanize.Time(time.Unix(int64(s.Time), 0)) 151 | s.DomainName = h 152 | storiesMutex.Lock() 153 | stories = append(stories, s) 154 | storiesMutex.Unlock() 155 | } 156 | 157 | }(key) 158 | } 159 | 160 | // wait for all the goroutines 161 | wg.Wait() 162 | 163 | return stories, nil 164 | } 165 | 166 | // getStoriesFromType gets the different types of stories the API exposes 167 | func getStoriesFromType(pageType string) (stories, error) { 168 | var url string 169 | switch pageType { 170 | case "best": 171 | url = baseURL + "beststories.json" 172 | case "new": 173 | url = baseURL + "newstories.json" 174 | case "show": 175 | url = baseURL + "showstories.json" 176 | default: 177 | url = baseURL + "topstories.json" 178 | } 179 | 180 | res, err := http.Get(url) 181 | if err != nil { 182 | return nil, errors.New("could not get " + pageType + " hacker news posts list") 183 | } 184 | 185 | defer res.Body.Close() 186 | s, err := getStories(res) 187 | if err != nil { 188 | return nil, errors.New("could not get " + pageType + " hacker news posts data") 189 | } 190 | 191 | return s, nil 192 | } 193 | 194 | // container holds data used by the template 195 | type container struct { 196 | Page string 197 | Stories stories 198 | } 199 | 200 | // pageHandler returns a handler for the correct page type 201 | func pageHandler(pageType string) func(w http.ResponseWriter, r *http.Request) { 202 | return func(w http.ResponseWriter, r *http.Request) { 203 | 204 | // we'll get all the stories 205 | var s stories 206 | 207 | // only because of shadowing 208 | var err error 209 | 210 | // know if we should use the cache 211 | var ok bool 212 | 213 | // check if we hit the cached stories for this page type 214 | x, found := cash.Get(pageType) 215 | if found { 216 | 217 | // check if valid stories 218 | s, ok = x.(stories) 219 | } 220 | 221 | // if it's not or we didn't hit the cached stories 222 | if !ok { 223 | 224 | // get the stories from the API 225 | s, err = getStoriesFromType(pageType) 226 | if err != nil { 227 | w.WriteHeader(500) 228 | w.Write([]byte(err.Error())) 229 | return 230 | } 231 | 232 | // set the cached stories for this page type 233 | cash.Set(pageType, s, cache.DefaultExpiration) 234 | } 235 | 236 | // parse the template file and return an error if it's broken 237 | tmpl, err := template.ParseFiles("index.tmpl") 238 | if err != nil { 239 | w.WriteHeader(500) 240 | w.Write([]byte("could not parse template file")) 241 | return 242 | } 243 | 244 | // finally let's just return 200 and write the template out 245 | w.WriteHeader(200) 246 | 247 | // set the content type header with html and utf encoding 248 | w.Header().Set("Content-type", "text/html;charset=utf-8") 249 | 250 | // execute the template which writes to w and uses container input 251 | tmpl.Execute(w, container{ 252 | Stories: s, 253 | Page: pageType, 254 | }) 255 | } 256 | } 257 | 258 | // fileHandler serves a file like the favicon or logo 259 | func fileHandler(file string) func(w http.ResponseWriter, r *http.Request) { 260 | return func(w http.ResponseWriter, r *http.Request) { 261 | switch file { 262 | case "favicon": 263 | http.ServeFile(w, r, "./favicon.ico") 264 | case "logo": 265 | http.ServeFile(w, r, "./logo.gif") 266 | default: 267 | w.WriteHeader(404) 268 | w.Write([]byte("file not found")) 269 | } 270 | } 271 | } 272 | 273 | // the main attraction, what you've all been waiting for 274 | func main() { 275 | 276 | // port 8080 is a good choice 277 | port := ":8080" 278 | 279 | // set up our routes and handlers 280 | http.HandleFunc("/", pageHandler("top")) 281 | http.HandleFunc("/new", pageHandler("new")) 282 | http.HandleFunc("/show", pageHandler("show")) 283 | http.HandleFunc("/best", pageHandler("best")) 284 | 285 | // serve the favicon and logo files 286 | http.HandleFunc("/favicon.ico", fileHandler("favicon")) 287 | http.HandleFunc("/logo.gif", fileHandler("logo")) 288 | 289 | // start the server up on our port 290 | log.Printf("Running on %s\n", port) 291 | log.Fatalln(http.ListenAndServe(port, nil)) 292 | } 293 | --------------------------------------------------------------------------------