├── README.md
├── .gitignore
├── .idea
    ├── .gitignore
    ├── misc.xml
    ├── vcs.xml
    ├── modules.xml
    ├── pre-api.iml
    └── inspectionProfiles
    │   └── Project_Default.xml
├── mkdocs.yml
├── .env.example
├── mysql.go
├── go.mod
├── sphinx.go
├── logger.go
├── routes.go
├── team.go
├── main.go
├── nuke.go
├── LICENSE
├── router.go
├── go.sum
├── api.go
├── messages.go
├── websocket.go
├── pre.go
├── handlers.go
└── docs
    └── index.md


/README.md:
--------------------------------------------------------------------------------
1 | # pre-api


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .env
2 | 


--------------------------------------------------------------------------------
/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /workspace.xml


--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: Predb.ovh API
2 | theme: readthedocs
3 | nav:
4 |   - Home: index.md
5 | 


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="JavaScriptSettings">
4 |     <option name="languageLevel" value="ES6" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.env.example:
--------------------------------------------------------------------------------
1 | HOSTNAME=predb.ovh
2 | AMQP_HOST=amqp://guest:guest@localhost:5672/
3 | SEARCH_DATABASE=tcp(127.0.0.1:9306)/?interpolateParams=true
4 | NUKES_DATABASE=predb@tcp(127.0.0.1:3306)/predb?interpolateParams=true
5 | LISTEN_ADDRESS=127.0.0.1:8088
6 | 


--------------------------------------------------------------------------------
/mysql.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"database/sql"
 5 | 	"log"
 6 | )
 7 | 
 8 | func newMysql(dbString string) *sql.DB {
 9 | 	mysql, err := sql.Open("mysql", dbString)
10 | 	if err != nil {
11 | 		log.Fatal(err)
12 | 	}
13 | 
14 | 	return mysql
15 | }
16 | 


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/pre-api.iml" filepath="$PROJECT_DIR$/.idea/pre-api.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/pre-api.iml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <module type="WEB_MODULE" version="4">
3 |   <component name="Go" enabled="true" />
4 |   <component name="NewModuleRootManager">
5 |     <content url="file://$MODULE_DIR$" />
6 |     <orderEntry type="inheritedJdk" />
7 |     <orderEntry type="sourceFolder" forTests="false" />
8 |   </component>
9 | </module>


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/predbdotovh/pre-api
 2 | 
 3 | go 1.11
 4 | 
 5 | require (
 6 | 	github.com/go-sql-driver/mysql v1.6.0
 7 | 	github.com/gorilla/feeds v1.1.1
 8 | 	github.com/gorilla/mux v1.8.0
 9 | 	github.com/gorilla/websocket v1.4.2
10 | 	github.com/joho/godotenv v1.4.0
11 | 	github.com/kr/pretty v0.2.1 // indirect
12 | 	github.com/streadway/amqp v1.0.0 // indirect
13 | )
14 | 


--------------------------------------------------------------------------------
/sphinx.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"database/sql"
 5 | )
 6 | 
 7 | const realTimeIndex = "pre_rt"
 8 | const plainIndex = "pre_plain"
 9 | const sphinxTable = plainIndex + ", " + realTimeIndex
10 | 
11 | func sphinxMeta(tx *sql.Tx) (map[string]string, error) {
12 | 	sqlRows, err := tx.Query("SHOW META")
13 | 	if err != nil {
14 | 		return nil, err
15 | 	}
16 | 	defer sqlRows.Close()
17 | 
18 | 	rows := make(map[string]string, 0)
19 | 	for sqlRows.Next() {
20 | 		var name, value string
21 | 		err := sqlRows.Scan(&name, &value)
22 | 		if err != nil {
23 | 			continue
24 | 		}
25 | 		rows[name] = value
26 | 	}
27 | 
28 | 	return rows, nil
29 | }
30 | 


--------------------------------------------------------------------------------
/logger.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"net/http"
 6 | 	"os"
 7 | 	"time"
 8 | )
 9 | 
10 | var stdOutLog = log.New(os.Stdout, "", log.LstdFlags)
11 | 
12 | func httpLogger(inner http.Handler, name string) http.Handler {
13 | 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
14 | 		start := time.Now()
15 | 
16 | 		inner.ServeHTTP(w, r)
17 | 
18 | 		from := r.Header.Get("x-forwarded-for")
19 | 		if from == "" {
20 | 			from = r.RemoteAddr
21 | 		}
22 | 
23 | 		stdOutLog.Printf(
24 | 			"%s %s %s %s %s",
25 | 			from,
26 | 			r.Method,
27 | 			r.RequestURI,
28 | 			name,
29 | 			time.Since(start),
30 | 		)
31 | 	})
32 | }
33 | 


--------------------------------------------------------------------------------
/routes.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"net/http"
 5 | )
 6 | 
 7 | type muxRoute struct {
 8 | 	Name    string
 9 | 	Method  string
10 | 	Pattern string
11 | 	Handler http.HandlerFunc
12 | }
13 | 
14 | type muxRoutes []muxRoute
15 | type jsonMuxRoutes map[string]muxRoutes
16 | 
17 | var jsonRoutes = jsonMuxRoutes{
18 | 	"v1": muxRoutes{
19 | 		muxRoute{"Root", "GET", "/", rootHandlerV1},
20 | 		muxRoute{"Teams", "GET", "/teams", teamsHandlerV1},
21 | 		muxRoute{"Stats", "GET", "/stats", statsHandlerV1},
22 | 		muxRoute{"Live", "GET", "/live", liveHandlerV1},
23 | 		muxRoute{"Rss", "GET", "/rss", rssHandlerV1},
24 | 		muxRoute{"WS", "GET", "/ws", websocketHandlerV1},
25 | 	},
26 | }
27 | 
28 | var triggerRoutes = muxRoutes{
29 | 	muxRoute{"NukeTrigger", "POST", "/nuke", nukeTriggerHandlerV1},
30 | 	muxRoute{"PreTrigger", "POST", "/{action}", preTriggerHandlerV1},
31 | }
32 | 


--------------------------------------------------------------------------------
/team.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"database/sql"
 5 | 	"fmt"
 6 | 	"log"
 7 | )
 8 | 
 9 | type teamRow struct {
10 | 	Team      string `json:"team"`
11 | 	FirstPre  int    `json:"firstPre"`
12 | 	LatestPre int    `json:"latestPre"`
13 | 	Count     int    `json:"count"`
14 | }
15 | 
16 | const maxListedTeams = 1000
17 | 
18 | func listTeams(tx *sql.Tx) ([]teamRow, error) {
19 | 	sqlQuery := fmt.Sprintf("SELECT team, MIN(pre_at), MAX(pre_at), COUNT(*) AS count FROM %s GROUP BY team ORDER BY count DESC LIMIT %d", sphinxTable, maxListedTeams)
20 | 
21 | 	rows, err := tx.Query(sqlQuery)
22 | 	if err != nil {
23 | 		return nil, err
24 | 	}
25 | 	defer rows.Close()
26 | 
27 | 	res := make([]teamRow, 0)
28 | 	for rows.Next() {
29 | 		var r teamRow
30 | 		err := rows.Scan(&r.Team, &r.FirstPre, &r.LatestPre, &r.Count)
31 | 		if err != nil {
32 | 			log.Print(err)
33 | 			continue
34 | 		}
35 | 
36 | 		res = append(res, r)
37 | 	}
38 | 
39 | 	return res, nil
40 | }
41 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"database/sql"
 5 | 	"log"
 6 | 	"net/http"
 7 | 	"os"
 8 | 
 9 | 	_ "github.com/joho/godotenv/autoload"
10 | )
11 | 
12 | var sphinx *sql.DB
13 | var mysql *sql.DB
14 | var hostname string
15 | 
16 | func main() {
17 | 	hostname := getEnv("HOSTNAME", "example.com")
18 | 	amqpHost := getEnv("AMQP_HOST", "")
19 | 	sphinxDatabase := getEnv("SEARCH_DATABASE", "tcp(127.0.0.1:9306)/?interpolateParams=true")
20 | 	nukesDatabase := getEnv("NUKES_DATABASE", "")
21 | 	listenAddr := getEnv("LISTEN_ADDRESS", "127.0.0.1:8088")
22 | 
23 | 	sphinx = newMysql(sphinxDatabase)
24 | 	mysql = newMysql(nukesDatabase)
25 | 
26 | 	router := newRouter(hostname)
27 | 	newMQ(amqpHost)
28 | 
29 | 	log.Fatal(http.ListenAndServe(listenAddr, router))
30 | }
31 | 
32 | func getEnv(key, defaultValue string) string {
33 | 	value, exists := os.LookupEnv(key)
34 | 	if !exists {
35 | 		if defaultValue == "" {
36 | 			log.Fatal("Missing mandatory env variable : " + key)
37 | 		}
38 | 
39 | 		return defaultValue
40 | 	}
41 | 
42 | 	return value
43 | }
44 | 


--------------------------------------------------------------------------------
/nuke.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"database/sql"
 5 | 	"fmt"
 6 | )
 7 | 
 8 | const nukeTable = "nuke"
 9 | 
10 | type nuke struct {
11 | 	ID     int    `json:"id"`
12 | 	TypeID int    `json:"typeId"`
13 | 	Type   string `json:"type"`
14 | 	PreID  int    `json:"preId"`
15 | 	Reason string `json:"reason"`
16 | 	Net    string `json:"net"`
17 | 	source string
18 | 	NukeAt int64 `json:"nukeAt"`
19 | }
20 | 
21 | var nukeTypes = map[int]string{
22 | 	1: "nuke",
23 | 	2: "unnuke",
24 | 	3: "modnuke",
25 | 	4: "delpre",
26 | 	5: "undelpre",
27 | }
28 | 
29 | func (n *nuke) setType() {
30 | 	if n.Type == "" {
31 | 		n.Type = nukeTypes[n.TypeID]
32 | 	}
33 | }
34 | 
35 | func getNuke(db *sql.DB, preID int) (*nuke, error) {
36 | 	sqlQuery := fmt.Sprintf("SELECT id, pre_id, nuke_id, reason, net, source, UNIX_TIMESTAMP(set_at) FROM %s WHERE pre_id = ? ORDER BY id DESC LIMIT 1", nukeTable)
37 | 
38 | 	var r nuke
39 | 
40 | 	err := db.QueryRow(sqlQuery, preID).Scan(&r.ID, &r.PreID, &r.TypeID, &r.Reason, &r.Net, &r.source, &r.NukeAt)
41 | 	if err != nil {
42 | 		return nil, err
43 | 	}
44 | 
45 | 	return &r, nil
46 | }
47 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 predbdotovh
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/router.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"errors"
 5 | 	"log"
 6 | 	"net/http"
 7 | 
 8 | 	_ "github.com/go-sql-driver/mysql"
 9 | 	"github.com/gorilla/mux"
10 | )
11 | 
12 | func newRouter(hostname string) *mux.Router {
13 | 	backendUpdates = make(chan triggerAction)
14 | 	go backendPump()
15 | 
16 | 	router := mux.NewRouter()
17 | 
18 | 	publicRouter := router.
19 | 		StrictSlash(true).
20 | 		Host(hostname).
21 | 		PathPrefix("/api").
22 | 		Subrouter()
23 | 
24 | 	for ver, jRoutes := range jsonRoutes {
25 | 		versionRouter := publicRouter.PathPrefix("/" + ver).Subrouter()
26 | 		for _, r := range jRoutes {
27 | 			versionRouter.
28 | 				Methods(r.Method).
29 | 				Path(r.Pattern).
30 | 				Name(r.Name).
31 | 				Handler(httpLogger(r.Handler, r.Name))
32 | 		}
33 | 	}
34 | 
35 | 	privateRouter := router.
36 | 		StrictSlash(true).
37 | 		Host("localhost").
38 | 		PathPrefix("/trigger").
39 | 		Subrouter()
40 | 
41 | 	for _, r := range triggerRoutes {
42 | 		privateRouter.
43 | 			Methods(r.Method).
44 | 			Path(r.Pattern).
45 | 			Name(r.Name).
46 | 			Handler(httpLogger(r.Handler, r.Name))
47 | 	}
48 | 
49 | 	router.NotFoundHandler = http.HandlerFunc(notFound)
50 | 
51 | 	return router
52 | }
53 | 
54 | func notFound(w http.ResponseWriter, _ *http.Request) {
55 | 	w.Header().Set("Cache-Control", "no-store")
56 | 	err := apiErr(w, errors.New("not Found"))
57 | 	if err != nil {
58 | 		log.Println(err)
59 | 	}
60 | }
61 | 


--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
 1 | <component name="InspectionProjectProfileManager">
 2 |   <profile version="1.0">
 3 |     <option name="myName" value="Project Default" />
 4 |     <inspection_tool class="GoUnhandledErrorResult" enabled="true" level="WARNING" enabled_by_default="true">
 5 |       <methods>
 6 |         <method importPath="hash" receiver="Hash" name="Write" />
 7 |         <method importPath="strings" receiver="*Builder" name="Write" />
 8 |         <method importPath="strings" receiver="*Builder" name="WriteByte" />
 9 |         <method importPath="bytes" receiver="*Buffer" name="WriteRune" />
10 |         <method importPath="bytes" receiver="*Buffer" name="Write" />
11 |         <method importPath="bytes" receiver="*Buffer" name="WriteString" />
12 |         <method importPath="strings" receiver="*Builder" name="WriteString" />
13 |         <method importPath="bytes" receiver="*Buffer" name="WriteByte" />
14 |         <method importPath="strings" receiver="*Builder" name="WriteRune" />
15 |         <method importPath="math/rand" receiver="*Rand" name="Read" />
16 |         <method importPath="github.com/gorilla/websocket" receiver="*Conn" name="Close" />
17 |         <method importPath="database/sql" receiver="*Rows" name="Close" />
18 |         <method importPath="database/sql" receiver="*Tx" name="Commit" />
19 |         <method importPath="io" receiver="ReadCloser" name="Close" />
20 |       </methods>
21 |     </inspection_tool>
22 |   </profile>
23 | </component>


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
 1 | github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 2 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 3 | github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
 4 | github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
 5 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 6 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 7 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 8 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 9 | github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
10 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
11 | github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
12 | github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
13 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
14 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
15 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
16 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
17 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
18 | github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=
19 | github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
20 | 


--------------------------------------------------------------------------------
/api.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"net/http"
 6 | 	"time"
 7 | )
 8 | 
 9 | type apiResponse struct {
10 | 	Status  string      `json:"status"`
11 | 	Message string      `json:"message"`
12 | 	Data    interface{} `json:"data"`
13 | }
14 | 
15 | type apiRowData struct {
16 | 	RowCount int      `json:"rowCount"`
17 | 	Rows     []preRow `json:"rows"`
18 | 	Offset   int      `json:"offset"`
19 | 	ReqCount int      `json:"reqCount"`
20 | 	Total    int      `json:"total"`
21 | 	Time     float64  `json:"time"`
22 | }
23 | 
24 | type apiStatsData struct {
25 | 	Total int       `json:"total"`
26 | 	Date  time.Time `json:"date"`
27 | 	Time  float64   `json:"time"`
28 | }
29 | 
30 | type apiTeamsData struct {
31 | 	RowCount int       `json:"rowCount"`
32 | 	Rows     []teamRow `json:"rows"`
33 | 	Offset   int       `json:"offset"`
34 | 	ReqCount int       `json:"reqCount"`
35 | 	Total    int       `json:"total"`
36 | 	Time     float64   `json:"time"`
37 | }
38 | 
39 | const indent = "    "
40 | 
41 | func apiSuccess(w http.ResponseWriter, o interface{}) error {
42 | 	return apiSend(w, apiResponse{"success", "", o})
43 | }
44 | 
45 | func apiFail(w http.ResponseWriter, o interface{}) error {
46 | 	return apiSend(w, apiResponse{"error", "", o})
47 | }
48 | 
49 | func apiErr(w http.ResponseWriter, err error) error {
50 | 	return apiSend(w, apiResponse{"error", err.Error(), nil})
51 | }
52 | 
53 | func apiSend(w http.ResponseWriter, o interface{}) error {
54 | 	headers := w.Header()
55 | 	headers.Set("Content-Type", "application/json; charset=utf-8")
56 | 	headers.Set("Access-Control-Allow-Origin", "*")
57 | 	w.WriteHeader(http.StatusOK)
58 | 
59 | 	enc := json.NewEncoder(w)
60 | 	enc.SetIndent("", indent)
61 | 
62 | 	return enc.Encode(o)
63 | }
64 | 


--------------------------------------------------------------------------------
/messages.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"github.com/streadway/amqp"
  6 | 	"log"
  7 | 	"strings"
  8 | )
  9 | 
 10 | func newMQ(amqpHost string) {
 11 | 	if amqpHost == "" {
 12 | 		return
 13 | 	}
 14 | 
 15 | 	conn, err := amqp.Dial(amqpHost)
 16 | 	if err != nil {
 17 | 		log.Fatal(err)
 18 | 	}
 19 | 	// defer conn.Close()
 20 | 
 21 | 	ch, err := conn.Channel()
 22 | 	if err != nil {
 23 | 		log.Fatal(err)
 24 | 	}
 25 | 	// defer ch.Close()
 26 | 
 27 | 	err = ch.ExchangeDeclare(
 28 | 		"predb",
 29 | 		"fanout",
 30 | 		true,
 31 | 		false,
 32 | 		false,
 33 | 		false,
 34 | 		nil,
 35 | 	)
 36 | 	if err != nil {
 37 | 		log.Fatal(err)
 38 | 	}
 39 | 
 40 | 	q, err := ch.QueueDeclare(
 41 | 		"pre-releases",
 42 | 		false,
 43 | 		false,
 44 | 		false,
 45 | 		false,
 46 | 		nil,
 47 | 	)
 48 | 	if err != nil {
 49 | 		log.Fatal(err)
 50 | 	}
 51 | 
 52 | 	err = ch.QueueBind(
 53 | 		q.Name,
 54 | 		"",
 55 | 		"predb",
 56 | 		false,
 57 | 		nil,
 58 | 	)
 59 | 	if err != nil {
 60 | 		log.Fatal(err)
 61 | 	}
 62 | 
 63 | 	msgs, err := ch.Consume(
 64 | 		q.Name,
 65 | 		"pre-api",
 66 | 		true,
 67 | 		false,
 68 | 		false,
 69 | 		false,
 70 | 		nil,
 71 | 	)
 72 | 	if err != nil {
 73 | 		log.Fatal(err)
 74 | 	}
 75 | 
 76 | 	go mqRun(msgs)
 77 | }
 78 | 
 79 | func mqRun(msgs <-chan amqp.Delivery) {
 80 | 	for d := range msgs {
 81 | 		log.Println(d.RoutingKey)
 82 | 		switch d.RoutingKey {
 83 | 		case "pre.insert":
 84 | 			fallthrough
 85 | 		case "pre.update":
 86 | 			fallthrough
 87 | 		case "pre.delete":
 88 | 			var p preRow
 89 | 			err := json.Unmarshal(d.Body, &p)
 90 | 			if err != nil {
 91 | 				log.Println(err)
 92 | 				return
 93 | 			}
 94 | 
 95 | 			p.proc()
 96 | 			if d.RoutingKey == "pre.delete" {
 97 | 				_, err = p.deleteFromIndex()
 98 | 				if err != nil {
 99 | 					log.Println(err)
100 | 					return
101 | 				}
102 | 			} else {
103 | 				_, err = p.insertOrUpdateIndex()
104 | 				if err != nil {
105 | 					log.Println(err)
106 | 					return
107 | 				}
108 | 			}
109 | 			backendUpdates <- triggerAction{Action: strings.Split(d.RoutingKey, ".")[1], Row: &p}
110 | 			break
111 | 		case "nuke.insert":
112 | 			var n nuke
113 | 			err := json.Unmarshal(d.Body, &n)
114 | 			if err != nil {
115 | 				log.Println(err)
116 | 				return
117 | 			}
118 | 
119 | 			p, err := getPre(sphinx, n.PreID, false)
120 | 			if err != nil {
121 | 				log.Println(err)
122 | 				return
123 | 			}
124 | 
125 | 			p.setNuke(&n)
126 | 			backendUpdates <- triggerAction{Action: n.Type, Row: p}
127 | 			break
128 | 		}
129 | 	}
130 | }
131 | 


--------------------------------------------------------------------------------
/websocket.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"log"
  5 | 	"net/http"
  6 | 	"time"
  7 | 
  8 | 	"github.com/gorilla/websocket"
  9 | )
 10 | 
 11 | type triggerAction struct {
 12 | 	Action string  `json:"action"`
 13 | 	Row    *preRow `json:"row"`
 14 | }
 15 | 
 16 | type wsClient struct {
 17 | 	conn *websocket.Conn
 18 | 	send chan triggerAction
 19 | }
 20 | 
 21 | const (
 22 | 	writeTimeout = 10 * time.Second
 23 | 	pongWait     = 60 * time.Second
 24 | 	pingPeriod   = 50 * time.Second
 25 | )
 26 | 
 27 | var upgrader = websocket.Upgrader{
 28 | 	CheckOrigin: func(r *http.Request) bool { return true },
 29 | }
 30 | 
 31 | var backendUpdates chan triggerAction
 32 | 
 33 | func websocketUpgrader(w http.ResponseWriter, r *http.Request) {
 34 | 	ws, err := upgrader.Upgrade(w, r, nil)
 35 | 	if err != nil {
 36 | 		log.Println(err)
 37 | 		return
 38 | 	}
 39 | 	defer ws.Close()
 40 | 
 41 | 	c := &wsClient{conn: ws, send: make(chan triggerAction)}
 42 | 	clients[c] = true
 43 | 	go c.writePump()
 44 | 	c.readPump()
 45 | }
 46 | 
 47 | var clients map[*wsClient]bool
 48 | 
 49 | func backendPump() {
 50 | 	backendUpdates = make(chan triggerAction)
 51 | 	clients = make(map[*wsClient]bool)
 52 | 
 53 | 	for {
 54 | 		select {
 55 | 		case rls := <-backendUpdates:
 56 | 			for c := range clients {
 57 | 				select {
 58 | 				case c.send <- rls:
 59 | 				default:
 60 | 					log.Println("CLOSE")
 61 | 					close(c.send)
 62 | 					delete(clients, c)
 63 | 				}
 64 | 			}
 65 | 		}
 66 | 	}
 67 | }
 68 | 
 69 | func (c *wsClient) writePump() {
 70 | 	ticker := time.NewTicker(pingPeriod)
 71 | 
 72 | 	defer ticker.Stop()
 73 | 	for {
 74 | 		select {
 75 | 		case rls := <-c.send:
 76 | 			err := c.conn.SetWriteDeadline(time.Now().Add(writeTimeout))
 77 | 			if err != nil {
 78 | 				log.Println("WRITE DEADLINE ERR", err)
 79 | 			}
 80 | 			err = c.conn.WriteJSON(rls)
 81 | 			if err != nil {
 82 | 				log.Println("WRITE ERR", err)
 83 | 				return
 84 | 			}
 85 | 		case <-ticker.C:
 86 | 			err := c.conn.SetWriteDeadline(time.Now().Add(writeTimeout))
 87 | 			if err != nil {
 88 | 				log.Println("PING DEADLINE ERR", err)
 89 | 			}
 90 | 			err = c.conn.WriteMessage(websocket.PingMessage, []byte{})
 91 | 			if err != nil {
 92 | 				log.Println("PING ERR", err)
 93 | 				return
 94 | 			}
 95 | 		}
 96 | 	}
 97 | }
 98 | 
 99 | func (c *wsClient) readPump() {
100 | 	err := c.conn.SetReadDeadline(time.Now().Add(pongWait))
101 | 	if err != nil {
102 | 		log.Println("READ DEADLINE ERR", err)
103 | 		return
104 | 	}
105 | 	c.conn.SetPongHandler(func(string) error {
106 | 		err = c.conn.SetReadDeadline(time.Now().Add(pongWait))
107 | 		if err != nil {
108 | 			log.Println("READ DEADLINE ERR", err)
109 | 		}
110 | 		return nil
111 | 	})
112 | 	for {
113 | 		_, _, err := c.conn.NextReader()
114 | 		if err != nil {
115 | 			log.Println("READ ERR", err)
116 | 			break
117 | 		}
118 | 	}
119 | }
120 | 


--------------------------------------------------------------------------------
/pre.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"database/sql"
  5 | 	"fmt"
  6 | 	"log"
  7 | 	"strconv"
  8 | 	"strings"
  9 | )
 10 | 
 11 | type preRow struct {
 12 | 	ID    int     `json:"id"`
 13 | 	Name  string  `json:"name"`
 14 | 	Team  string  `json:"team"`
 15 | 	Cat   string  `json:"cat"`
 16 | 	Genre string  `json:"genre"`
 17 | 	URL   string  `json:"url"`
 18 | 	Size  float64 `json:"size"`
 19 | 	Files int     `json:"files"`
 20 | 	PreAt int     `json:"preAt"`
 21 | 	Nuke  *nuke   `json:"nuke"`
 22 | }
 23 | 
 24 | var replacer = strings.NewReplacer("(", "\\(", ")", "\\)")
 25 | 
 26 | const preColumns = "id, name, team, cat, genre, url, size, files, pre_at"
 27 | const defaultMaxMatches = 1000
 28 | 
 29 | func (p *preRow) proc() {
 30 | 	p.Size /= 1000
 31 | }
 32 | 
 33 | func scanPresRows(rows *sql.Rows, appendNukes bool) []preRow {
 34 | 	res := make([]preRow, 0)
 35 | 	for rows.Next() {
 36 | 		var r preRow
 37 | 		err := rows.Scan(&r.ID, &r.Name, &r.Team, &r.Cat, &r.Genre, &r.URL, &r.Size, &r.Files, &r.PreAt)
 38 | 		if err != nil {
 39 | 			log.Print(err)
 40 | 			continue
 41 | 		}
 42 | 
 43 | 		r.proc()
 44 | 		if appendNukes {
 45 | 			r.fetchNuke(mysql)
 46 | 		}
 47 | 
 48 | 		res = append(res, r)
 49 | 	}
 50 | 
 51 | 	return res
 52 | }
 53 | 
 54 | func getPre(db *sql.DB, preID int, withNuke bool) (*preRow, error) {
 55 | 	sqlQuery := fmt.Sprintf("SELECT %s FROM %s WHERE id = ?", preColumns, sphinxTable)
 56 | 
 57 | 	var r preRow
 58 | 
 59 | 	err := db.QueryRow(sqlQuery, preID).Scan(&r.ID, &r.Name, &r.Team, &r.Cat, &r.Genre, &r.URL, &r.Size, &r.Files, &r.PreAt)
 60 | 	if err != nil {
 61 | 		return nil, err
 62 | 	}
 63 | 
 64 | 	r.proc()
 65 | 	if withNuke {
 66 | 		r.fetchNuke(mysql)
 67 | 	}
 68 | 
 69 | 	return &r, nil
 70 | }
 71 | 
 72 | func getPresById(tx *sql.Tx, preID int, withNuke bool) ([]preRow, error) {
 73 | 	sqlQuery := fmt.Sprintf("SELECT %s FROM %s WHERE id = ?", preColumns, sphinxTable)
 74 | 
 75 | 	var r preRow
 76 | 
 77 | 	err := tx.QueryRow(sqlQuery, preID).Scan(&r.ID, &r.Name, &r.Team, &r.Cat, &r.Genre, &r.URL, &r.Size, &r.Files, &r.PreAt)
 78 | 	if err != nil {
 79 | 		return nil, err
 80 | 	}
 81 | 
 82 | 	r.proc()
 83 | 	if withNuke {
 84 | 		r.fetchNuke(mysql)
 85 | 	}
 86 | 
 87 | 	return []preRow{r}, nil
 88 | }
 89 | 
 90 | func searchPres(tx *sql.Tx, q string, offsetInt, countInt int, withNukes bool) ([]preRow, error) {
 91 | 	sqlQuery := fmt.Sprintf("SELECT %s FROM %s WHERE MATCH(?) ORDER BY id DESC LIMIT %d,%d", preColumns, sphinxTable, offsetInt, countInt)
 92 | 
 93 | 	if offsetInt+countInt > defaultMaxMatches {
 94 | 		sqlQuery += ", max_matches = " + strconv.Itoa(offsetInt+countInt)
 95 | 	}
 96 | 
 97 | 	sqlRows, err := tx.Query(sqlQuery, replacer.Replace(q))
 98 | 	if err != nil {
 99 | 		return nil, err
100 | 	}
101 | 	defer sqlRows.Close()
102 | 
103 | 	return scanPresRows(sqlRows, withNukes), nil
104 | }
105 | 
106 | func latestPres(tx *sql.Tx, offsetInt, countInt int, withNukes bool) ([]preRow, error) {
107 | 	sqlQuery := fmt.Sprintf("SELECT %s FROM %s ORDER BY id DESC LIMIT %d,%d", preColumns, sphinxTable, offsetInt, countInt)
108 | 
109 | 	if offsetInt+countInt > defaultMaxMatches {
110 | 		sqlQuery += ", max_matches = " + strconv.Itoa(offsetInt+countInt)
111 | 	}
112 | 
113 | 	sqlRows, err := tx.Query(sqlQuery)
114 | 	if err != nil {
115 | 		return nil, err
116 | 	}
117 | 	defer sqlRows.Close()
118 | 
119 | 	return scanPresRows(sqlRows, withNukes), nil
120 | }
121 | 
122 | func (p *preRow) fetchNuke(db *sql.DB) {
123 | 	n, err := getNuke(db, p.ID)
124 | 	if err != nil {
125 | 		return
126 | 	}
127 | 	p.setNuke(n)
128 | }
129 | 
130 | func (p *preRow) setNuke(n *nuke) {
131 | 	if n != nil {
132 | 		n.setType()
133 | 		p.Nuke = n
134 | 	}
135 | }
136 | 
137 | func (p *preRow) insertOrUpdateIndex() (int64, error) {
138 | 	sqlQuery := fmt.Sprintf("REPLACE INTO %s (%s) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", realTimeIndex, preColumns)
139 | 
140 | 	res, err := sphinx.Exec(sqlQuery, p.ID, p.Name, p.Team, p.Cat, p.Genre, p.URL, p.Size, p.Files, p.PreAt)
141 | 	if err != nil {
142 | 		return 0, err
143 | 	}
144 | 
145 | 	return res.RowsAffected()
146 | }
147 | 
148 | func (p *preRow) deleteFromIndex() (int64, error) {
149 | 	sqlQuery := fmt.Sprintf("DELETE FROM %s WHERE id = ?", realTimeIndex)
150 | 
151 | 	res, err := sphinx.Exec(sqlQuery, p.ID)
152 | 	if err != nil {
153 | 		return 0, err
154 | 	}
155 | 
156 | 	return res.RowsAffected()
157 | }
158 | 


--------------------------------------------------------------------------------
/handlers.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"errors"
  6 | 	"fmt"
  7 | 	"log"
  8 | 	"net/http"
  9 | 	"net/url"
 10 | 	"strconv"
 11 | 	"time"
 12 | 
 13 | 	"github.com/gorilla/feeds"
 14 | 	"github.com/gorilla/mux"
 15 | )
 16 | 
 17 | const defaultOffset = 0
 18 | const defaultCount = 20
 19 | const maxCount = 100
 20 | 
 21 | func pagination(query url.Values) (int, int, error) {
 22 | 	var err error
 23 | 	count := defaultCount
 24 | 	offset := defaultOffset
 25 | 
 26 | 	countStr := query.Get("count")
 27 | 	if countStr != "" {
 28 | 		count, err = strconv.Atoi(countStr)
 29 | 		if err != nil {
 30 | 			return 0, 0, fmt.Errorf("incorrect parameters (count)")
 31 | 		}
 32 | 
 33 | 	}
 34 | 	if count > maxCount {
 35 | 		count = maxCount
 36 | 	}
 37 | 	if count < 0 {
 38 | 		count = defaultCount
 39 | 	}
 40 | 
 41 | 	pageStr := query.Get("page")
 42 | 	if pageStr != "" {
 43 | 		page, err := strconv.Atoi(pageStr)
 44 | 		if err != nil {
 45 | 			return 0, 0, fmt.Errorf("incorrect parameters (page)")
 46 | 		}
 47 | 
 48 | 		offset = (page - 1) * count
 49 | 	}
 50 | 
 51 | 	offsetStr := query.Get("offset")
 52 | 	if offsetStr != "" {
 53 | 		offset, err = strconv.Atoi(offsetStr)
 54 | 		if err != nil {
 55 | 			return 0, 0, fmt.Errorf("incorrect parameters (offset)")
 56 | 		}
 57 | 	}
 58 | 	if offset < 0 {
 59 | 		offset = defaultOffset
 60 | 	}
 61 | 
 62 | 	return offset, count, nil
 63 | }
 64 | 
 65 | func handleQuery(r *http.Request) (*apiRowData, error) {
 66 | 	t := time.Now()
 67 | 
 68 | 	query := r.URL.Query()
 69 | 	idStr := query.Get("id")
 70 | 	q := query.Get("q")
 71 | 
 72 | 	offset, count, err := pagination(query)
 73 | 	if err != nil {
 74 | 		return nil, err
 75 | 	}
 76 | 
 77 | 	tx, err := sphinx.Begin()
 78 | 	if err != nil {
 79 | 		return nil, err
 80 | 	}
 81 | 	defer tx.Commit()
 82 | 
 83 | 	rows := make([]preRow, 0)
 84 | 	if idStr != "" {
 85 | 		id, err := strconv.Atoi(idStr)
 86 | 		if err != nil {
 87 | 			return nil, err
 88 | 		}
 89 | 		rows, err = getPresById(tx, id, true)
 90 | 	} else if q == "" {
 91 | 		rows, err = latestPres(tx, offset, count, true)
 92 | 	} else {
 93 | 		rows, err = searchPres(tx, q, offset, count, true)
 94 | 	}
 95 | 	if err != nil {
 96 | 		return nil, err
 97 | 	}
 98 | 
 99 | 	meta, err := sphinxMeta(tx)
100 | 	if err != nil {
101 | 		return nil, err
102 | 	}
103 | 
104 | 	total, err := strconv.Atoi(meta["total_found"])
105 | 	if err != nil {
106 | 		return nil, err
107 | 	}
108 | 
109 | 	data := &apiRowData{
110 | 		RowCount: len(rows),
111 | 		Rows:     rows,
112 | 		Offset:   offset,
113 | 		ReqCount: count,
114 | 		Total:    total,
115 | 		Time:     time.Since(t).Seconds(),
116 | 	}
117 | 
118 | 	return data, nil
119 | }
120 | 
121 | func rootHandlerV1(w http.ResponseWriter, r *http.Request) {
122 | 	data, err := handleQuery(r)
123 | 	if err != nil {
124 | 		err = apiErr(w, err)
125 | 		if err != nil {
126 | 			log.Println(err)
127 | 		}
128 | 		return
129 | 	}
130 | 
131 | 	w.Header().Set("Cache-Control", "public, max-age=60")
132 | 	err = apiSuccess(w, data)
133 | 	if err != nil {
134 | 		log.Println(err)
135 | 	}
136 | }
137 | 
138 | func teamsHandlerV1(w http.ResponseWriter, _ *http.Request) {
139 | 	t := time.Now()
140 | 
141 | 	tx, err := sphinx.Begin()
142 | 	if err != nil {
143 | 		err = apiErr(w, err)
144 | 		if err != nil {
145 | 			log.Println(err)
146 | 		}
147 | 		return
148 | 	}
149 | 	defer tx.Commit()
150 | 
151 | 	rows, err := listTeams(tx)
152 | 	if err != nil {
153 | 		err = apiErr(w, err)
154 | 		if err != nil {
155 | 			log.Println(err)
156 | 		}
157 | 
158 | 		return
159 | 	}
160 | 
161 | 	data := &apiTeamsData{
162 | 		RowCount: len(rows),
163 | 		Rows:     rows,
164 | 		Offset:   0,
165 | 		ReqCount: maxListedTeams,
166 | 		Total:    len(rows),
167 | 		Time:     time.Since(t).Seconds(),
168 | 	}
169 | 
170 | 	w.Header().Set("Cache-Control", "public, max-age=3600")
171 | 	err = apiSuccess(w, data)
172 | 	if err != nil {
173 | 		log.Println(err)
174 | 	}
175 | }
176 | 
177 | func liveHandlerV1(w http.ResponseWriter, r *http.Request) {
178 | 	data, err := handleQuery(r)
179 | 	if err != nil {
180 | 		err = apiErr(w, err)
181 | 		if err != nil {
182 | 			log.Println(err)
183 | 		}
184 | 		return
185 | 	}
186 | 
187 | 	w.Header().Set("Cache-Control", "no-cache")
188 | 	err = apiSuccess(w, data)
189 | 	if err != nil {
190 | 		log.Println(err)
191 | 	}
192 | }
193 | 
194 | func websocketHandlerV1(w http.ResponseWriter, r *http.Request) {
195 | 	websocketUpgrader(w, r)
196 | }
197 | 
198 | func preTriggerHandlerV1(w http.ResponseWriter, r *http.Request) {
199 | 	if r.Header.Get("x-forwarded-for") != "" {
200 | 		err := apiErr(w, errors.New("not authorized"))
201 | 		if err != nil {
202 | 			log.Println(err)
203 | 		}
204 | 		return
205 | 	}
206 | 
207 | 	var p preRow
208 | 	err := json.NewDecoder(r.Body).Decode(&p)
209 | 	if err != nil {
210 | 		err = apiFail(w, err)
211 | 		if err != nil {
212 | 			log.Println(err)
213 | 		}
214 | 		return
215 | 	}
216 | 	defer r.Body.Close()
217 | 
218 | 	p.proc()
219 | 
220 | 	backendUpdates <- triggerAction{Action: mux.Vars(r)["action"], Row: &p}
221 | 	err = apiSuccess(w, nil)
222 | 	if err != nil {
223 | 		log.Println(err)
224 | 	}
225 | }
226 | 
227 | func nukeTriggerHandlerV1(w http.ResponseWriter, r *http.Request) {
228 | 	if r.Header.Get("x-forwarded-for") != "" {
229 | 		err := apiErr(w, errors.New("not authorized"))
230 | 		if err != nil {
231 | 			log.Println(err)
232 | 		}
233 | 		return
234 | 	}
235 | 
236 | 	var n nuke
237 | 	err := json.NewDecoder(r.Body).Decode(&n)
238 | 	if err != nil {
239 | 		err = apiFail(w, err)
240 | 		if err != nil {
241 | 			log.Println(err)
242 | 		}
243 | 		return
244 | 	}
245 | 	defer r.Body.Close()
246 | 
247 | 	p, err := getPre(sphinx, n.PreID, false)
248 | 	if err != nil {
249 | 		err = apiFail(w, err)
250 | 		if err != nil {
251 | 			log.Println(err)
252 | 		}
253 | 		return
254 | 	}
255 | 
256 | 	p.setNuke(&n)
257 | 
258 | 	backendUpdates <- triggerAction{Action: n.Type, Row: p}
259 | 	err = apiSuccess(w, nil)
260 | 	if err != nil {
261 | 		log.Println(err)
262 | 	}
263 | }
264 | 
265 | func statsHandlerV1(w http.ResponseWriter, _ *http.Request) {
266 | 	t := time.Now()
267 | 
268 | 	tx, err := sphinx.Begin()
269 | 	if err != nil {
270 | 		err = apiErr(w, err)
271 | 		if err != nil {
272 | 			log.Println(err)
273 | 		}
274 | 		return
275 | 	}
276 | 	defer tx.Commit()
277 | 
278 | 	_, err = latestPres(tx, 0, 0, false)
279 | 	if err != nil {
280 | 		err = apiErr(w, err)
281 | 		if err != nil {
282 | 			log.Println(err)
283 | 		}
284 | 		return
285 | 	}
286 | 
287 | 	meta, err := sphinxMeta(tx)
288 | 	if err != nil {
289 | 		err = apiErr(w, err)
290 | 		if err != nil {
291 | 			log.Println(err)
292 | 		}
293 | 		return
294 | 	}
295 | 
296 | 	total, err := strconv.Atoi(meta["total_found"])
297 | 	if err != nil {
298 | 		log.Println(err)
299 | 	}
300 | 
301 | 	w.Header().Set("Cache-Control", "public, max-age=60")
302 | 	data := apiStatsData{
303 | 		Total: total,
304 | 		Date:  time.Now(),
305 | 		Time:  time.Since(t).Seconds(),
306 | 	}
307 | 	err = apiSuccess(w, data)
308 | 	if err != nil {
309 | 		log.Println(err)
310 | 	}
311 | }
312 | 
313 | func rssHandlerV1(w http.ResponseWriter, r *http.Request) {
314 | 	data, err := handleQuery(r)
315 | 	if err != nil {
316 | 		w.WriteHeader(http.StatusInternalServerError)
317 | 		return
318 | 	}
319 | 
320 | 	feed := &feeds.Feed{
321 | 		Title:   "PreDB",
322 | 		Link:    &feeds.Link{Href: fmt.Sprintf("https://%s/", hostname)},
323 | 		Created: time.Now(),
324 | 	}
325 | 
326 | 	for _, row := range data.Rows {
327 | 		feed.Items = append(feed.Items, &feeds.Item{
328 | 			Title:       row.Name,
329 | 			Link:        &feeds.Link{Href: fmt.Sprintf("https://%s/?id=%d", hostname, row.ID)},
330 | 			Description: fmt.Sprintf("Cat:%s | Genre:%s | Size:%0.fMB | Files:%d | ID:%d", row.Cat, row.Genre, row.Size, row.Files, row.ID),
331 | 			Created:     time.Unix(int64(row.PreAt), 0),
332 | 		})
333 | 	}
334 | 
335 | 	rss, err := feed.ToRss()
336 | 	if err != nil {
337 | 		w.WriteHeader(http.StatusInternalServerError)
338 | 		return
339 | 	}
340 | 
341 | 	_, err = w.Write([]byte(rss))
342 | 	if err != nil {
343 | 		log.Println(err)
344 | 	}
345 | }
346 | 


--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
  1 | # Predb.ovh API documentation
  2 | 
  3 | Most pre databases websites do not allow data scapping, nor do they offer an API.
  4 | 
  5 | Having a easily accessible API was a requirement for me, so I built this one using public pre sources.
  6 | 
  7 | ## Disclaimers
  8 | 
  9 | Disclaimer : This API has been built for my personal usage, and may not fit your needs.
 10 | [Github issues](https://github.com/predbdotovh/pre-api/issues) can be filled, but don't expect too much.
 11 | 
 12 | Disclaimer 2 : Once again, this API does **NOT** host any content, only metadata associated to scene releases.
 13 | 
 14 | Disclaimer 3 : Don't know what scene releases are ? You're probably at the wrong place.
 15 | 
 16 | ## Status
 17 | 
 18 | This API is currently usable, and used by [demo website](https://predb.ovh/) which is
 19 | also [open-sourced](https://github.com/predbdotovh/website-vuejs).
 20 | 
 21 | As the API is fed by Sphinx, results are hard limited at 1000, and I don't expect to modify this behaviour in the
 22 | future.
 23 | 
 24 | A monthly-ish sql dump is available here : [https://predb.ovh/download/](https://predb.ovh/download/)
 25 | 
 26 | ## API versions
 27 | 
 28 | Current version is **v1**. This version is expected to be updated on each breaking change.
 29 | 
 30 | Current base URL is : [https://predb.ovh/api/v1/](https://predb.ovh/api/v1/)
 31 | 
 32 | ## Responses
 33 | 
 34 | The API will always return HTTP 200 with application/json content (except for [RSS](#get-rss)).
 35 | 
 36 | On success :
 37 | 
 38 | ```
 39 | {
 40 |     "status": "success",
 41 |     "message": "",
 42 |     "data": "... See endpoints responses ..."
 43 | }
 44 | ```
 45 | 
 46 | On failure :
 47 | 
 48 | ```
 49 | {
 50 |     "status": "error",
 51 |     "message": "Human readable error message",
 52 |     "data": null
 53 | }
 54 | ```
 55 | 
 56 | ## Endpoints
 57 | 
 58 | ### GET /
 59 | 
 60 | List releases matching a set of filters given in parameters
 61 | 
 62 | - Cache : 60 seconds
 63 | - Usage : Get releases info
 64 | - Rate limit : 30/60s
 65 | 
 66 | #### Parameters
 67 | 
 68 | All parameters are optional
 69 | 
 70 | | param key | type   | content                              |
 71 | | --------- | ------ | ------------------------------------ |
 72 | | count     | int    | Maximum releases count expected      |
 73 | | page      | int    | Page offset                          |
 74 | | offset    | int    | Row offset (overwrites page param)   |
 75 | | q         | string | Query                                |
 76 | | id        | int    | Specific pre ID (overwrites q param) |
 77 | 
 78 | Query is directly fed to a SphinxSearch engine,
 79 | allowing [specific syntax](http://sphinxsearch.com/docs/current/extended-syntax.html). Note: cat and team are indexed,
 80 | allowing fast queries like
 81 | [https://predb.ovh/api/v1/?q=@cat%20EBOOK](https://predb.ovh/api/v1/?q=@cat%20EBOOK)
 82 | 
 83 | #### Response
 84 | 
 85 | ##### Data
 86 | 
 87 | | json key | type      | content                       |
 88 | | -------- | --------- | ----------------------------- |
 89 | | rowCount | int       | Count of rows returned        |
 90 | | offset   | int       | Row count offset requested    |
 91 | | reqCount | int       | Row count requested           |
 92 | | total    | int       | Total matching rows           |
 93 | | time     | float     | Request internal duration     |
 94 | | rows     | []release | Array of [releases](#release) |
 95 | 
 96 | ##### Release
 97 | 
 98 | | json key | type      | content                           |
 99 | | -------- | --------- | --------------------------------- |
100 | | id       | int       | Internal unique pre ID            |
101 | | name     | string    | Release name                      |
102 | | team     | string    | Release group extracted from name |
103 | | cat      | string    | Category                          |
104 | | genre    | string    | Genre                             |
105 | | url      | string    | Info link                         |
106 | | size     | float     | Release size in kb                |
107 | | files    | int       | Original file count               |
108 | | preAt    | int       | Release pre timestamp             |
109 | | nuke     | nuke/null | [Nuke](#nuke) info if available   |
110 | 
111 | ##### Nuke
112 | 
113 | | json key | type   | content                     |
114 | | -------- | ------ | --------------------------- |
115 | | id       | int    | Internal unique nuke ID     |
116 | | typeId   | int    | [Nuke type](#nuke-types) ID |
117 | | type     | string | [Nuke type](#nuke-types)    |
118 | | preId    | int    | Nuked pre ID                |
119 | | reason   | string | Nuke reason                 |
120 | | net      | string | Nuke source net             |
121 | | nukeAt   | int    | Nuke timestamp              |
122 | 
123 | ##### Nuke types
124 | 
125 | Known nuke types and type ids
126 | 
127 | | nuke type ID | nuke type |
128 | | ------------ | --------- |
129 | | 1            | nuke      |
130 | | 2            | unnuke    |
131 | | 3            | modnuke   |
132 | | 4            | delpre    |
133 | | 5            | undelpre  |
134 | 
135 | #### Example
136 | 
137 | - [https://predb.ovh/api/v1/?q=bdrip](https://predb.ovh/api/v1/?q=bdrip)
138 | 
139 | ```
140 | {
141 |     "status": "success",
142 |     "message": "",
143 |     "data": {
144 |         "rowCount": 20,
145 |         "rows": [
146 |             {
147 |                 "id": 7813747,
148 |                 "name": "Fortitude.S02E10.German.DUBBED.BDRip.x264-AIDA",
149 |                 "team": "AIDA",
150 |                 "cat": "TV-DVDRIP",
151 |                 "genre": "",
152 |                 "url": "http://imdb.com/title/tt3498622/",
153 |                 "size": 0,
154 |                 "files": 0,
155 |                 "preAt": 1493815560,
156 |                 "nuke": null
157 |             },
158 |             {
159 |                 "snip": "Content snipped for lisibility"
160 |             },
161 |             {
162 |                 "id": 7812994,
163 |                 "name": "Fortitude.S02E07.BDRip.x264-HAGGiS",
164 |                 "team": "HAGGiS",
165 |                 "cat": "TV-DVDRIP",
166 |                 "genre": "",
167 |                 "url": "http://imdb.com/title/tt3498622",
168 |                 "size": 258,
169 |                 "files": 19,
170 |                 "preAt": 1493762423,
171 |                 "nuke": null
172 |             }
173 |         ],
174 |         "offset": 0,
175 |         "reqCount": 20,
176 |         "total": 70729,
177 |         "time": 0.108095174
178 |     }
179 | }
180 | ```
181 | 
182 | ### GET /live
183 | 
184 | This method is the exact clone of [GET /](#get), without any HTTP cache.
185 | 
186 | - Cache : None
187 | - Usage : Get fresh data before listening to websocket updates
188 | - Rate limit : 2/20s
189 | 
190 | **To avoid abuse, this method is severly rate limited**
191 | 
192 | #### Example
193 | 
194 | - [https://predb.ovh/api/v1/live](https://predb.ovh/api/v1/live)
195 | 
196 | ```
197 | {
198 |     "status": "success",
199 |     "message": "",
200 |     "data": {
201 |         "rowCount": 20,
202 |         "rows": [
203 |             {
204 |                 "id": 7814699,
205 |                 "name": "AllOver30.com_17.05.03.Drew.Jones.XXX.iMAGESET-YAPG",
206 |                 "team": "YAPG",
207 |                 "cat": "XXX-iMGSET",
208 |                 "genre": "Mature.Women",
209 |                 "url": "http://HTTP://AllOver30.com",
210 |                 "size": 211.9,
211 |                 "files": 45,
212 |                 "preAt": 1493881734,
213 |                 "nuke": null
214 |             },
215 |             {
216 |                 "snip": "Content snipped for lisibility"
217 |             },
218 |             {
219 |                 "id": 7814680,
220 |                 "name": "Robert_Abigail_Feat_13_Amps_-_Living_On_The_Right_Side-(AG_010)-WEB-2017-ZzZz",
221 |                 "team": "ZzZz",
222 |                 "cat": "MP3",
223 |                 "genre": "Dance",
224 |                 "url": "http://junodownload.com",
225 |                 "size": 7,
226 |                 "files": 1,
227 |                 "preAt": 1493880783,
228 |                 "nuke": null
229 |             }
230 |         ],
231 |         "offset": 0,
232 |         "reqCount": 20,
233 |         "total": 7751247,
234 |         "time": 0.898239015
235 |     }
236 | }
237 | ```
238 | 
239 | ### GET /rss
240 | 
241 | This method is the exact clone of [GET /](#get), formatted using RSS2.0 spec.
242 | 
243 | - Cache : 60 seconds
244 | - Usage : Get releases info using a RSS reader
245 | - Rate limit : 30/60s
246 | 
247 | #### Example
248 | 
249 | - [https://predb.ovh/api/v1/rss](https://predb.ovh/api/v1/rss)
250 | 
251 | ```
252 | <?xml version="1.0" encoding="UTF-8"?>
253 | <rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
254 |     <channel>
255 |         <title>PreDB</title>
256 |         <link>https://predb.ovh/</link>
257 |         <description></description>
258 |         <pubDate>Sun, 15 Dec 2019 21:23:59 +0100</pubDate>
259 |         <item>
260 |             <title>Le.Steppe.Dell.Asia.La.Piu.Grande.Steppa.Della.Terra.iTALiAN.HDTV.x264-iDiB</title>
261 |             <link>https://predb.ovh/?id=9643108</link>
262 |             <description>Cat:TV-SD-X264 | Genre: | Size:0MB | Files:0 | ID:9643108</description>
263 |             <pubDate>Sun, 15 Dec 2019 21:23:49 +0100</pubDate>
264 |         </item>
265 |         <item>
266 |             <title>Le.Steppe.Dell.Asia.La.Piu.Grande.Steppa.Della.Terra.iTALiAN.720p.HDTV.x264-iDiB</title>
267 |             <link>https://predb.ovh/?id=9643107</link>
268 |             <description>Cat:TV-HD-X264 | Genre: | Size:0MB | Files:0 | ID:9643107</description>
269 |             <pubDate>Sun, 15 Dec 2019 21:23:48 +0100</pubDate>
270 |         </item>
271 |     </channel>
272 | </rss>
273 | ```
274 | 
275 | ### GET /teams
276 | 
277 | Teams stats with first and latest pre, and total recorded pre.
278 | 
279 | This is currently hard limited to 1000 results, may be subject to change if necessary.
280 | 
281 | - Cache : 3600 seconds
282 | - Usage : Teams statistics
283 | 
284 | #### Parameters
285 | 
286 | None
287 | 
288 | #### Response
289 | 
290 | ##### Data
291 | 
292 | | json key | type      | content                       |
293 | | -------- | --------- | ----------------------------- |
294 | | rowCount | int       | Count of rows returned        |
295 | | offset   | int       | Row count offset requested    |
296 | | reqCount | int       | Row count requested           |
297 | | total    | int       | Total matching rows           |
298 | | time     | float     | Request internal duration     |
299 | | rows     | []team    | Array of [teams](#team)       |
300 | 
301 | ##### Team
302 | 
303 | | json key  | type      | content                               |
304 | | --------- | --------- | ------------------------------------- |
305 | | team      | string    | Release group name                    |
306 | | firstPre  | int       | First recorded release pre timestamp  |
307 | | latestPre | int       | Latest recorded release pre timestamp |
308 | | count     | int       | Total team pre count                  |
309 | 
310 | #### Example
311 | 
312 | - [https://predb.ovh/api/v1/teams](https://predb.ovh/api/v1/teams)
313 | 
314 | ```
315 | {
316 |     "status": "success",
317 |     "message": "",
318 |     "data": {
319 |         "rowCount": 1000,
320 |         "rows": [
321 |             {
322 |                 "team": "KTR",
323 |                 "firstPre": 1203561700,
324 |                 "latestPre": 1598341396,
325 |                 "count": 307933
326 |             },
327 |             {
328 |                 "snip": "Content snipped for lisibility"
329 |             },
330 |         ],
331 |         "time": 5.465514844
332 |     }
333 | }
334 | ```
335 | 
336 | ### GET /stats
337 | 
338 | Basic stats about internal database health.
339 | 
340 | - Cache : 60 seconds
341 | - Usage : Keep track of current database status and response times
342 | 
343 | #### Parameters
344 | 
345 | None
346 | 
347 | #### Response
348 | 
349 | ##### Data
350 | 
351 | | json key | type   | content                             |
352 | | -------- | ------ | ----------------------------------- |
353 | | total    | int    | Total indexed releases count        |
354 | | date     | string | Current RFC3339 timestamp of server |
355 | | time     | int    | Full index scan duration            |
356 | 
357 | #### Example
358 | 
359 | - [https://predb.ovh/api/v1/stats](https://predb.ovh/api/v1/stats)
360 | 
361 | ```
362 | {
363 |     "status": "success",
364 |     "message": "",
365 |     "data": {
366 |         "total": 7751280,
367 |         "date": "2017-05-04T09:49:42.527504724+02:00",
368 |         "time": 0.893255737
369 |     }
370 | }
371 | ```
372 | 
373 | ### GET /ws
374 | 
375 | A websocket endpoint, sending near realtime updates. To use this, you need to bind using
376 | a [Websocket](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_client_applications).
377 | 
378 | - Cache : None
379 | - Usage : Realtime updates
380 | 
381 | #### Parameters
382 | 
383 | There is no parameter, and any input on the websocket is discarded
384 | 
385 | #### Response
386 | 
387 | There is no response per se, but a range of frames sent over time
388 | 
389 | ##### Frame
390 | 
391 | Each frame represents an action towards a specific release.
392 | 
393 | | json key | type    | content             |
394 | | -------- | ------- | ------------------- |
395 | | action   | string  | Action type         |
396 | | row      | release | [Release](#release) |
397 | 
398 | ##### Action types
399 | 
400 | Known action types
401 | 
402 | | action   | context                                 |
403 | | -------- | --------------------------------------- |
404 | | insert   | First release pre                       |
405 | | update   | Any release field update                |
406 | | delete   | Erroneous release (should never happen) |
407 | | nuke     | Release nuked by net                    |
408 | | unnuke   | Release unnuked by net                  |
409 | | modnuke  | Nuke reason modified by net             |
410 | | delpre   | Pre deleted by net                      |
411 | | undelpre | Pre undeleted by net                    |
412 | 
413 | #### Example
414 | 
415 | - [https://predb.ovh/api/v1/ws](https://predb.ovh/api/v1/ws)
416 | 
417 | ```
418 | {
419 |     "action": "insert",
420 |     "row": {
421 |         "id": 7814727,
422 |         "name": "Doom_Squad-Countdown_To_Doomsday_II-WEB-2016-ESG",
423 |         "team": "ESG",
424 |         "cat": "MP3",
425 |         "genre": "",
426 |         "url": "",
427 |         "size": 0,
428 |         "files": 0,
429 |         "preAt": 1493884429,
430 |         "nuke": null
431 |     }
432 | }
433 | ```
434 | 


--------------------------------------------------------------------------------