├── README.md ├── storage └── storage.go ├── LICENSE ├── main.go └── cl ├── search.go └── city.go /README.md: -------------------------------------------------------------------------------- 1 | # multicraig 2 | -------------------------------------------------------------------------------- /storage/storage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "sync" 4 | 5 | type Store interface { 6 | Set(key string, val interface{}) error 7 | Get(key string) (interface{}, error) 8 | Del(key string) error 9 | } 10 | 11 | func NewStore() Store { 12 | return newMemory() 13 | } 14 | 15 | func newMemory() *memory { 16 | return &memory{ 17 | m: make(map[string]interface{}), 18 | } 19 | } 20 | 21 | type memory struct { 22 | m map[string]interface{} 23 | sync.RWMutex 24 | } 25 | 26 | func (m *memory) Set(key string, val interface{}) error { 27 | m.Lock() 28 | defer m.Unlock() 29 | m.m[key] = val 30 | return nil 31 | } 32 | 33 | func (m *memory) Get(key string) (interface{}, error) { 34 | m.RLock() 35 | defer m.RUnlock() 36 | return m.m[key], nil 37 | } 38 | 39 | func (m *memory) Del(key string) error { 40 | m.Lock() 41 | defer m.Unlock() 42 | delete(m.m, key) 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Ian Lozinski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/i/multicraig/cl" 10 | ) 11 | 12 | func main() { 13 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 14 | fmt.Fprintf(w, "ok") 15 | }) 16 | 17 | http.HandleFunc("/cities", func(w http.ResponseWriter, r *http.Request) { 18 | cities, err := cl.GetCities() 19 | if err != nil { 20 | fmt.Fprint(w, err) 21 | return 22 | } 23 | buf, err := json.Marshal(cities) 24 | if err != nil { 25 | fmt.Fprint(w, err) 26 | return 27 | } 28 | 29 | w.Write(buf) 30 | }) 31 | 32 | http.HandleFunc("/closeCities", func(w http.ResponseWriter, r *http.Request) { 33 | distanceMi, err := strconv.ParseFloat(r.FormValue("distanceMi"), 64) 34 | if err != nil { 35 | fmt.Fprint(w, err.Error()) 36 | return 37 | } 38 | 39 | city, err := cl.GetCity(r.FormValue("city")) 40 | if err != nil { 41 | fmt.Fprint(w, err.Error()) 42 | return 43 | } 44 | cities := city.CitiesWithin(distanceMi) 45 | buf, err := json.Marshal(cities) 46 | if err != nil { 47 | fmt.Fprint(w, err.Error()) 48 | return 49 | } 50 | w.Write(buf) 51 | }) 52 | 53 | http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) { 54 | city := r.FormValue("city") 55 | query := r.FormValue("query") 56 | category := r.FormValue("category") 57 | if category == "" { 58 | category = "sss" 59 | } 60 | distanceMi, err := strconv.ParseFloat(r.FormValue("distanceMi"), 64) 61 | if err != nil { 62 | distanceMi = 0 63 | } 64 | 65 | results, err := cl.Search(city, category, query, distanceMi) 66 | if err != nil { 67 | fmt.Fprintf(w, err.Error()) 68 | return 69 | } 70 | 71 | buf, err := json.Marshal(results) 72 | if err != nil { 73 | w.WriteHeader(http.StatusInternalServerError) 74 | fmt.Fprint(w, err.Error()) 75 | } 76 | 77 | w.Write(buf) 78 | }) 79 | 80 | http.ListenAndServe("localhost:3000", nil) 81 | } 82 | -------------------------------------------------------------------------------- /cl/search.go: -------------------------------------------------------------------------------- 1 | package cl 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | "time" 8 | 9 | "github.com/SlyMarbo/rss" 10 | ) 11 | 12 | const ( 13 | jpgType = "image/jpeg" 14 | searchCityStr = "%s/search/%s?format=rss&query=%s" 15 | ) 16 | 17 | func init() { 18 | rss.CacheParsedItemIDs(false) 19 | } 20 | 21 | func Search(cityName, category, query string, distanceMI float64) ([]SearchResult, error) { 22 | city, err := GetCity(cityName) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | cities := append([]City{city}, city.CitiesWithin(distanceMI)...) 28 | results := make([]SearchResult, 0) 29 | 30 | var wg sync.WaitGroup 31 | var m sync.Mutex 32 | for _, city := range cities { 33 | wg.Add(1) 34 | go func() { 35 | defer wg.Done() 36 | posts, err := city.Search(category, query) 37 | if err != nil { 38 | log.Print(err) 39 | return 40 | } 41 | m.Lock() 42 | results = append(results, SearchResult{City: city, Posts: posts}) 43 | m.Unlock() 44 | }() 45 | } 46 | wg.Wait() 47 | 48 | return results, nil 49 | } 50 | 51 | type Post struct { 52 | Title string 53 | URL string 54 | Date time.Time 55 | Image string 56 | } 57 | 58 | type SearchResult struct { 59 | City City 60 | Posts []Post 61 | } 62 | 63 | func (c City) Search(category, query string) ([]Post, error) { 64 | feed, err := rss.Fetch(fmt.Sprintf(searchCityStr, c.URL, category, query)) 65 | if err != nil { 66 | fmt.Println(fmt.Sprintf(searchCityStr, c.URL, category, query)) 67 | return nil, err 68 | } 69 | 70 | results := make([]Post, 0) 71 | for _, item := range feed.Items { 72 | results = append(results, newPost(item)) 73 | } 74 | return results, nil 75 | } 76 | 77 | func newPost(item *rss.Item) Post { 78 | post := Post{ 79 | Title: item.Title, 80 | Date: item.Date, 81 | URL: item.Link, 82 | } 83 | for _, e := range item.Enclosures { 84 | if e.Type == jpgType { 85 | post.Image = e.Url 86 | break 87 | } 88 | } 89 | return post 90 | } 91 | -------------------------------------------------------------------------------- /cl/city.go: -------------------------------------------------------------------------------- 1 | package cl 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "math" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "regexp" 13 | "strings" 14 | "sync" 15 | 16 | "github.com/i/jdog" 17 | "github.com/i/multicraig/storage" 18 | ) 19 | 20 | const ( 21 | kmtomiles = float64(0.621371192) 22 | earthRadius = float64(6371) 23 | clCityURL = "http://www.craigslist.org/about/sites" 24 | cityLocationURL = "https://maps.googleapis.com/maps/api/geocode/json?address=%s,+%s&key=%s" 25 | ) 26 | 27 | var ( 28 | //cityRE = regexp.MustCompile(`
  • .*http://.*"([a-z]*)\.craigslist.*">(.*)`) 29 | //cityRE = regexp.MustCompile(`^
  • (.*)
    • $`) 30 | cityRE = regexp.MustCompile(`^
  • ([a-zA-Z|\s]+)
    • $`) 31 | regionRE = regexp.MustCompile(`

    (.*)

    `) 32 | wwwRE = regexp.MustCompile(`www.craigslist.org`) 33 | gapikey = os.Getenv("MAPSAPIKEY") 34 | 35 | cache = storage.NewStore() 36 | ) 37 | 38 | type City struct { 39 | Name string 40 | Region string 41 | URL string 42 | Lat float64 43 | Lng float64 44 | } 45 | 46 | func GetCity(name string) (City, error) { 47 | cities, err := GetCities() 48 | if err != nil { 49 | return City{}, err 50 | } 51 | for _, c := range cities { 52 | if c.Name == name { 53 | return c, nil 54 | } 55 | } 56 | return City{}, fmt.Errorf("city not found") 57 | } 58 | 59 | func GetCities() ([]City, error) { 60 | c, err := cache.Get("cities") 61 | if err != nil { 62 | log.Print(err) 63 | } 64 | if cities, ok := c.([]City); ok { 65 | if len(cities) > 0 { 66 | return cities, nil 67 | } 68 | } 69 | 70 | var cities []City 71 | defer func() { cache.Set("cities", cities) }() 72 | 73 | res, err := http.Get(clCityURL) 74 | defer res.Body.Close() 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | if res.StatusCode != http.StatusOK { 80 | return nil, fmt.Errorf("bad status from CL rss: %d", res.Status) 81 | } 82 | 83 | buf, err := ioutil.ReadAll(res.Body) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | var region string 89 | for _, line := range strings.Split(string(buf), "\n") { 90 | m := regionRE.FindStringSubmatch(line) 91 | if len(m) > 1 { 92 | region = m[1] 93 | continue 94 | } 95 | 96 | m = cityRE.FindStringSubmatch(line) 97 | if len(m) > 1 { 98 | if wwwRE.MatchString(m[1]) { 99 | continue 100 | } 101 | 102 | c := City{ 103 | Region: region, 104 | Name: m[2], 105 | URL: m[1], 106 | } 107 | cities = append(cities, c) 108 | } 109 | } 110 | 111 | var wg sync.WaitGroup 112 | for i, c := range cities { 113 | wg.Add(1) 114 | go func(i int, c City) { 115 | defer wg.Done() 116 | var err error 117 | c.Lat, c.Lng, err = c.GetLocation() 118 | if err != nil { 119 | log.Print(err) 120 | return 121 | } 122 | cities[i] = c 123 | }(i, c) 124 | } 125 | wg.Wait() 126 | 127 | return cities, nil 128 | } 129 | 130 | func (c City) GetLocation() (lat, lng float64, err error) { 131 | if c.Lat != 0 && c.Lng != 0 { 132 | return c.Lat, c.Lng, nil 133 | } 134 | 135 | url := fmt.Sprintf(cityLocationURL, url.QueryEscape(c.Name), url.QueryEscape(c.Region), gapikey) 136 | res, err := http.Get(url) 137 | defer res.Body.Close() 138 | if err != nil { 139 | return 0, 0, err 140 | } 141 | if res.StatusCode != http.StatusOK { 142 | return 0, 0, fmt.Errorf("%s\t%d", url, res.StatusCode) 143 | } 144 | 145 | buf, err := ioutil.ReadAll(res.Body) 146 | if err != nil { 147 | return 0, 0, err 148 | } 149 | 150 | var m map[string]interface{} 151 | if err := json.Unmarshal(buf, &m); err != nil { 152 | return 0, 0, err 153 | } 154 | 155 | v, err := jdog.Get(m, "results[0].geometry.location.lat") 156 | if err != nil { 157 | return 0, 0, fmt.Errorf("lat/lng not found for %s\nurl: %s\n: m: %v", c.Name, url, m) 158 | } 159 | lat = v.(float64) 160 | 161 | v, err = jdog.Get(m, "results[0].geometry.location.lng") 162 | if err != nil { 163 | return 0, 0, fmt.Errorf("lat/lng not found for %s", c.Name) 164 | } 165 | lng = v.(float64) 166 | 167 | return lat, lng, nil 168 | } 169 | 170 | func (c City) CitiesWithin(mi float64) []City { 171 | cities, err := GetCities() 172 | if err != nil { 173 | log.Printf("error fetching cities: %v", err) 174 | return nil 175 | } 176 | var closeCities []City 177 | for _, c2 := range cities { 178 | if c2.Name == c.Name { 179 | continue 180 | } 181 | dst := c.distanceToCity(c2) 182 | if dst == -1 { 183 | continue 184 | } 185 | if dst < mi { 186 | closeCities = append(closeCities, c2) 187 | } 188 | } 189 | return closeCities 190 | } 191 | 192 | func (c City) distanceToCity(dst City) float64 { 193 | if c.Lat == 0 || c.Lng == 0 || dst.Lat == 0 || dst.Lng == 0 { 194 | return -1 195 | } 196 | 197 | km := haversine(c.Lng, c.Lat, dst.Lng, dst.Lat) 198 | return km * kmtomiles 199 | } 200 | 201 | func haversine(lonFrom float64, latFrom float64, lonTo float64, latTo float64) (distance float64) { 202 | var deltaLat = (latTo - latFrom) * (math.Pi / 180) 203 | var deltaLon = (lonTo - lonFrom) * (math.Pi / 180) 204 | 205 | var a = math.Sin(deltaLat/2)*math.Sin(deltaLat/2) + 206 | math.Cos(latFrom*(math.Pi/180))*math.Cos(latTo*(math.Pi/180))* 207 | math.Sin(deltaLon/2)*math.Sin(deltaLon/2) 208 | var c = 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a)) 209 | 210 | distance = earthRadius * c 211 | 212 | return 213 | } 214 | --------------------------------------------------------------------------------