├── LICENSE ├── README.md ├── TODO.md ├── cmd ├── nntpchan │ └── main.go └── nntpserver │ └── main.go ├── lib ├── admin │ ├── doc.go │ └── server.go ├── api │ ├── api.go │ ├── doc.go │ └── server.go ├── config │ ├── article.go │ ├── cache.go │ ├── config.go │ ├── database.go │ ├── doc.go │ ├── feed.go │ ├── frontend.go │ ├── hook.go │ ├── middleware.go │ ├── nntp.go │ ├── proxy.go │ ├── ssl.go │ ├── store.go │ └── webhooks.go ├── crypto │ ├── doc.go │ ├── hash.go │ ├── nacl.go │ ├── nacl │ │ ├── box.go │ │ ├── buffer.go │ │ ├── key.go │ │ ├── nacl.go │ │ ├── rand.go │ │ ├── sign.go │ │ ├── stream.go │ │ └── verfiy.go │ ├── nacl_test.go │ ├── rand.go │ ├── sig.go │ ├── sign.go │ └── verify.go ├── database │ ├── database.go │ ├── doc.go │ └── postgres.go ├── frontend │ ├── captcha.go │ ├── doc.go │ ├── frontend.go │ ├── http.go │ ├── middleware.go │ ├── overchan.go │ ├── post.go │ └── webhooks.go ├── model │ ├── article.go │ ├── attachment.go │ ├── board.go │ ├── boardpage.go │ ├── doc.go │ ├── misc.go │ ├── post.go │ └── thread.go ├── network │ ├── dial.go │ ├── doc.go │ ├── i2p.go │ ├── socks.go │ └── std.go ├── nntp │ ├── acceptor.go │ ├── auth.go │ ├── client.go │ ├── codes.go │ ├── commands.go │ ├── common.go │ ├── common_test.go │ ├── conn.go │ ├── conn_v1.go │ ├── dial.go │ ├── doc.go │ ├── event_test.go │ ├── filter.go │ ├── hook.go │ ├── hooks.go │ ├── message │ │ ├── article.go │ │ ├── attachment.go │ │ ├── doc.go │ │ └── header.go │ ├── mode.go │ ├── multi.go │ ├── policy.go │ ├── server.go │ ├── state.go │ └── streaming.go ├── srnd │ └── doc.go ├── store │ ├── doc.go │ ├── fs.go │ ├── log.go │ ├── null.go │ └── store.go ├── thumbnail │ ├── doc.go │ ├── exec.go │ ├── multi.go │ ├── thumb.go │ └── thumb_test.go ├── util │ ├── cache.go │ ├── discard.go │ ├── hex.go │ ├── ip.go │ ├── nntp_login.go │ ├── post.go │ └── time.go └── webhooks │ ├── doc.go │ ├── http.go │ ├── multi.go │ └── webhooks.go └── srnd.go /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2017 Jeff Becker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # srndv2 3 | 4 | building: 5 | 6 | go get -u -v github.com/majestrate/srndv2/cmd/nntpchan 7 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## TODO LIST ## 4 | 5 | * OAUTH API for posting 6 | * sqlite database type 7 | * redis database type 8 | 9 | * static JSON files for http frontend 10 | * reprocess nntp articles admin function 11 | * thoroughly fix nntp sync deadlocks 12 | 13 | src/srnd/config.go 14 | 15 | Setup() method needs to catch errors. 16 | ReadConfig() : Should take config file name as argument. Should return error type if erorr occurs. 17 | 18 | func ReadConfig(filename string) (*SRNdConfig, error){} 19 | 20 | 21 | ----------- 22 | 23 | Validate() should also return errors if it fails: Needs to change to: 24 | 25 | func (self *SRNdConfig) Validate() error { 26 | 27 | } 28 | 29 | 30 | ------------------ 31 | 32 | src/srnd/database.go 33 | 34 | NewDatabase(par1,par2,..) should return Database and error 35 | 36 | Should change to: 37 | 38 | func NewDatabase(db_type, schema, host, port, user, password string) (Database,error) { 39 | -------------------------------------------------------------------------------- /cmd/nntpchan/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | log "github.com/Sirupsen/logrus" 5 | "github.com/majestrate/srndv2/lib/config" 6 | "github.com/majestrate/srndv2/lib/database" 7 | "github.com/majestrate/srndv2/lib/frontend" 8 | "github.com/majestrate/srndv2/lib/nntp" 9 | "github.com/majestrate/srndv2/lib/store" 10 | "github.com/majestrate/srndv2/lib/webhooks" 11 | "net" 12 | _ "net/http/pprof" 13 | "os" 14 | "os/signal" 15 | "syscall" 16 | "time" 17 | ) 18 | 19 | type runStatus struct { 20 | nntpListener net.Listener 21 | run bool 22 | done chan error 23 | } 24 | 25 | func (st *runStatus) Stop() { 26 | st.run = false 27 | if st.nntpListener != nil { 28 | st.nntpListener.Close() 29 | } 30 | st.nntpListener = nil 31 | log.Info("stopping daemon process") 32 | } 33 | 34 | func main() { 35 | st := &runStatus{ 36 | run: true, 37 | done: make(chan error), 38 | } 39 | log.Info("starting up nntpchan...") 40 | cfgFname := "nntpchan.json" 41 | conf, err := config.Ensure(cfgFname) 42 | if err != nil { 43 | log.Fatal(err) 44 | } 45 | 46 | if conf.Log == "debug" { 47 | log.SetLevel(log.DebugLevel) 48 | } 49 | 50 | sconfig := conf.Store 51 | 52 | if sconfig == nil { 53 | log.Fatal("no article storage configured") 54 | } 55 | 56 | nconfig := conf.NNTP 57 | 58 | if nconfig == nil { 59 | log.Fatal("no nntp server configured") 60 | } 61 | 62 | dconfig := conf.Database 63 | 64 | if dconfig == nil { 65 | log.Fatal("no database configured") 66 | } 67 | 68 | // create nntp server 69 | nserv := nntp.NewServer() 70 | nserv.Config = nconfig 71 | nserv.Feeds = conf.Feeds 72 | 73 | if nconfig.LoginsFile != "" { 74 | nserv.Auth = nntp.FlatfileAuth(nconfig.LoginsFile) 75 | } 76 | 77 | // create article storage 78 | nserv.Storage, err = store.NewFilesytemStorage(sconfig.Path, true) 79 | if err != nil { 80 | log.Fatal(err) 81 | } 82 | 83 | if conf.WebHooks != nil && len(conf.WebHooks) > 0 { 84 | // put webhooks into nntp server event hooks 85 | nserv.Hooks = webhooks.NewWebhooks(conf.WebHooks, nserv.Storage) 86 | } 87 | 88 | if conf.NNTPHooks != nil && len(conf.NNTPHooks) > 0 { 89 | var hooks nntp.MulitHook 90 | if nserv.Hooks != nil { 91 | hooks = append(hooks, nserv.Hooks) 92 | } 93 | for _, h := range conf.NNTPHooks { 94 | hooks = append(hooks, nntp.NewHook(h)) 95 | } 96 | nserv.Hooks = hooks 97 | } 98 | 99 | var db database.Database 100 | for _, fconf := range conf.Frontends { 101 | var f frontend.Frontend 102 | f, err = frontend.NewHTTPFrontend(fconf, db) 103 | if err == nil { 104 | go f.Serve() 105 | } 106 | } 107 | 108 | // start persisting feeds 109 | go nserv.PersistFeeds() 110 | 111 | // handle signals 112 | sigchnl := make(chan os.Signal, 1) 113 | signal.Notify(sigchnl, syscall.SIGHUP, os.Interrupt) 114 | go func() { 115 | for { 116 | s := <-sigchnl 117 | if s == syscall.SIGHUP { 118 | // handle SIGHUP 119 | conf, err := config.Ensure(cfgFname) 120 | if err == nil { 121 | log.Infof("reloading config: %s", cfgFname) 122 | nserv.ReloadServer(conf.NNTP) 123 | nserv.ReloadFeeds(conf.Feeds) 124 | } else { 125 | log.Errorf("failed to reload config: %s", err) 126 | } 127 | } else if s == os.Interrupt { 128 | // handle interrupted, clean close 129 | st.Stop() 130 | return 131 | } 132 | } 133 | }() 134 | go func() { 135 | var err error 136 | for st.run { 137 | var nl net.Listener 138 | naddr := conf.NNTP.Bind 139 | log.Infof("Bind nntp server to %s", naddr) 140 | nl, err = net.Listen("tcp", naddr) 141 | if err == nil { 142 | st.nntpListener = nl 143 | err = nserv.Serve(nl) 144 | if err != nil { 145 | nl.Close() 146 | log.Errorf("nntpserver.serve() %s", err.Error()) 147 | } 148 | } else { 149 | log.Errorf("nntp server net.Listen failed: %s", err.Error()) 150 | } 151 | time.Sleep(time.Second) 152 | } 153 | st.done <- err 154 | }() 155 | e := <-st.done 156 | if e != nil { 157 | log.Fatal(e) 158 | } 159 | log.Info("ended") 160 | } 161 | -------------------------------------------------------------------------------- /cmd/nntpserver/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // simple nntp server 4 | 5 | import ( 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/majestrate/srndv2/lib/config" 8 | "github.com/majestrate/srndv2/lib/nntp" 9 | "github.com/majestrate/srndv2/lib/store" 10 | "net" 11 | ) 12 | 13 | func main() { 14 | 15 | log.Info("starting NNTP server...") 16 | conf, err := config.Ensure("settings.json") 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | 21 | if conf.Log == "debug" { 22 | log.SetLevel(log.DebugLevel) 23 | } 24 | 25 | serv := &nntp.Server{ 26 | Config: conf.NNTP, 27 | Feeds: conf.Feeds, 28 | } 29 | serv.Storage, err = store.NewFilesytemStorage(conf.Store.Path, false) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | l, err := net.Listen("tcp", conf.NNTP.Bind) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | log.Info("listening on ", l.Addr()) 38 | err = serv.Serve(l) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /lib/admin/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // server admin panel 3 | // 4 | package admin 5 | -------------------------------------------------------------------------------- /lib/admin/server.go: -------------------------------------------------------------------------------- 1 | package admin 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type Server struct { 8 | } 9 | 10 | func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 11 | 12 | } 13 | 14 | func NewServer() *Server { 15 | return &Server{} 16 | } 17 | -------------------------------------------------------------------------------- /lib/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/model" 5 | ) 6 | 7 | // json api 8 | type API interface { 9 | MakePost(p model.Post) 10 | } 11 | -------------------------------------------------------------------------------- /lib/api/doc.go: -------------------------------------------------------------------------------- 1 | // json api 2 | package api 3 | -------------------------------------------------------------------------------- /lib/api/server.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | ) 7 | 8 | // api server 9 | type Server struct { 10 | } 11 | 12 | func (s *Server) HandlePing(w http.ResponseWriter, r *http.Request) { 13 | 14 | } 15 | 16 | // inject api routes 17 | func (s *Server) SetupRoutes(r *mux.Router) { 18 | // setup api pinger 19 | r.Path("/ping").HandlerFunc(s.HandlePing) 20 | } 21 | -------------------------------------------------------------------------------- /lib/config/article.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "regexp" 4 | 5 | // configration for local article policies 6 | type ArticleConfig struct { 7 | // explicitly allow these newsgroups (regexp) 8 | AllowGroups []string `json:"whitelist"` 9 | // explicitly disallow these newsgroups (regexp) 10 | DisallowGroups []string `json:"blacklist"` 11 | // only allow explicitly allowed groups 12 | ForceWhitelist bool `json:"force-whitelist"` 13 | // allow anonymous posts? 14 | AllowAnon bool `json:"anon"` 15 | // allow attachments? 16 | AllowAttachments bool `json:"attachments"` 17 | // allow anonymous attachments? 18 | AllowAnonAttachments bool `json:"anon-attachments"` 19 | } 20 | 21 | func (c *ArticleConfig) AllowGroup(group string) bool { 22 | 23 | for _, g := range c.DisallowGroups { 24 | r := regexp.MustCompile(g) 25 | if r.MatchString(group) && c.ForceWhitelist { 26 | // disallowed 27 | return false 28 | } 29 | } 30 | 31 | // check allowed groups first 32 | for _, g := range c.AllowGroups { 33 | r := regexp.MustCompile(g) 34 | if r.MatchString(g) { 35 | return true 36 | } 37 | } 38 | 39 | return !c.ForceWhitelist 40 | } 41 | 42 | // allow an article? 43 | func (c *ArticleConfig) Allow(msgid, group string, anon, attachment bool) bool { 44 | 45 | // check attachment policy 46 | if c.AllowGroup(group) { 47 | allow := true 48 | // no anon ? 49 | if anon && !c.AllowAnon { 50 | allow = false 51 | } 52 | // no attachments ? 53 | if allow && attachment && !c.AllowAttachments { 54 | allow = false 55 | } 56 | // no anon attachments ? 57 | if allow && attachment && anon && !c.AllowAnonAttachments { 58 | allow = false 59 | } 60 | return allow 61 | } else { 62 | return false 63 | } 64 | } 65 | 66 | var DefaultArticlePolicy = ArticleConfig{ 67 | AllowGroups: []string{"ctl", "overchan.test"}, 68 | DisallowGroups: []string{"overchan.cp"}, 69 | ForceWhitelist: false, 70 | AllowAnon: true, 71 | AllowAttachments: true, 72 | AllowAnonAttachments: false, 73 | } 74 | -------------------------------------------------------------------------------- /lib/config/cache.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // caching interface configuration 4 | type CacheConfig struct { 5 | // backend cache driver name 6 | Backend string `json:"backend"` 7 | // address for cache 8 | Addr string `json:"addr"` 9 | // username for login 10 | User string `json:"user"` 11 | // password for login 12 | Password string `json:"password"` 13 | } 14 | -------------------------------------------------------------------------------- /lib/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "os" 8 | ) 9 | 10 | // main configuration 11 | type Config struct { 12 | // nntp server configuration 13 | NNTP *NNTPServerConfig `json:"nntp"` 14 | // log level 15 | Log string `json:"log"` 16 | // article storage config 17 | Store *StoreConfig `json:"storage"` 18 | // web hooks to call 19 | WebHooks []*WebhookConfig `json:"webhooks"` 20 | // external scripts to call 21 | NNTPHooks []*NNTPHookConfig `json:"nntphooks"` 22 | // database backend configuration 23 | Database *DatabaseConfig `json:"db"` 24 | // list of feeds to add on runtime 25 | Feeds []*FeedConfig `json:"feeds"` 26 | // frontend config 27 | Frontends []*FrontendConfig `json:"frontends"` 28 | // unexported fields ... 29 | 30 | // absolute filepath to configuration 31 | fpath string 32 | } 33 | 34 | // default configuration 35 | var DefaultConfig = Config{ 36 | Store: &DefaultStoreConfig, 37 | NNTP: &DefaultNNTPConfig, 38 | Database: &DefaultDatabaseConfig, 39 | WebHooks: []*WebhookConfig{DefaultWebHookConfig}, 40 | NNTPHooks: []*NNTPHookConfig{DefaultNNTPHookConfig}, 41 | Feeds: DefaultFeeds, 42 | Frontends: []*FrontendConfig{&DefaultFrontendConfig}, 43 | Log: "debug", 44 | } 45 | 46 | // reload configuration 47 | func (c *Config) Reload() (err error) { 48 | var b []byte 49 | b, err = ioutil.ReadFile(c.fpath) 50 | if err == nil { 51 | err = json.Unmarshal(b, c) 52 | } 53 | return 54 | } 55 | 56 | // ensure that a config file exists 57 | // creates one if it does not exist 58 | func Ensure(fname string) (cfg *Config, err error) { 59 | _, err = os.Stat(fname) 60 | if os.IsNotExist(err) { 61 | err = nil 62 | var d []byte 63 | d, err = json.Marshal(&DefaultConfig) 64 | if err == nil { 65 | b := new(bytes.Buffer) 66 | err = json.Indent(b, d, "", " ") 67 | if err == nil { 68 | err = ioutil.WriteFile(fname, b.Bytes(), 0600) 69 | } 70 | } 71 | } 72 | if err == nil { 73 | cfg, err = Load(fname) 74 | } 75 | return 76 | } 77 | 78 | // load configuration file 79 | func Load(fname string) (cfg *Config, err error) { 80 | cfg = new(Config) 81 | cfg.fpath = fname 82 | err = cfg.Reload() 83 | if err != nil { 84 | cfg = nil 85 | } 86 | return 87 | } 88 | -------------------------------------------------------------------------------- /lib/config/database.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type DatabaseConfig struct { 4 | // url or address for database connector 5 | Addr string `json:"addr"` 6 | // password to use 7 | Password string `json:"password"` 8 | // username to use 9 | Username string `json:"username"` 10 | // type of database to use 11 | Type string `json:"type"` 12 | } 13 | 14 | var DefaultDatabaseConfig = DatabaseConfig{ 15 | Type: "postgres", 16 | Addr: "/var/run/postgresql", 17 | Password: "", 18 | } 19 | -------------------------------------------------------------------------------- /lib/config/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // package for parsing config files 3 | // 4 | package config 5 | -------------------------------------------------------------------------------- /lib/config/feed.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // configuration for 1 nntp feed 4 | type FeedConfig struct { 5 | // feed's policy, filters articles 6 | Policy *ArticleConfig `json:"policy"` 7 | // remote server's address 8 | Addr string `json:"addr"` 9 | // proxy server config 10 | Proxy *ProxyConfig `json:"proxy"` 11 | // nntp username to log in with 12 | Username string `json:"username"` 13 | // nntp password to use when logging in 14 | Password string `json:"password"` 15 | // do we want to use tls? 16 | TLS bool `json:"tls"` 17 | // the name of this feed 18 | Name string `json:"name"` 19 | // how often to pull articles from the server in minutes 20 | // 0 for never 21 | PullInterval int `json:"pull"` 22 | } 23 | 24 | var DuummyFeed = FeedConfig{ 25 | Policy: &DefaultArticlePolicy, 26 | Addr: "nntp.dummy.tld:1119", 27 | Proxy: &DefaultTorProxy, 28 | Name: "dummy", 29 | } 30 | 31 | var DefaultFeeds = []*FeedConfig{ 32 | &DuummyFeed, 33 | } 34 | -------------------------------------------------------------------------------- /lib/config/frontend.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type FrontendConfig struct { 4 | // bind to address 5 | BindAddr string `json:"bind"` 6 | // frontend cache 7 | Cache *CacheConfig `json:"cache"` 8 | // frontend ssl settings 9 | SSL *SSLSettings `json:"ssl"` 10 | // static files directory 11 | Static string `json:"static_dir"` 12 | // http middleware configuration 13 | Middleware *MiddlewareConfig `json:"middleware"` 14 | } 15 | 16 | // default Frontend Configuration 17 | var DefaultFrontendConfig = FrontendConfig{ 18 | BindAddr: "127.0.0.1:18888", 19 | Static: "./files/static/", 20 | Middleware: &DefaultMiddlewareConfig, 21 | } 22 | -------------------------------------------------------------------------------- /lib/config/hook.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // config for external callback for nntp articles 4 | type NNTPHookConfig struct { 5 | // name of hook 6 | Name string `json:"name"` 7 | // executable script path to be called with arguments: /path/to/article 8 | Exec string `json:"exec"` 9 | } 10 | 11 | // default dummy hook 12 | var DefaultNNTPHookConfig = &NNTPHookConfig{ 13 | Name: "dummy", 14 | Exec: "/bin/true", 15 | } 16 | -------------------------------------------------------------------------------- /lib/config/middleware.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // configuration for http middleware 4 | type MiddlewareConfig struct { 5 | // middleware type, currently just 1 is available: overchan 6 | Type string `json:"type"` 7 | // directory for our html templates 8 | Templates string `json:"templates_dir"` 9 | } 10 | 11 | var DefaultMiddlewareConfig = MiddlewareConfig{ 12 | Type: "overchan", 13 | Templates: "./files/templates/overchan/", 14 | } 15 | -------------------------------------------------------------------------------- /lib/config/nntp.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type NNTPServerConfig struct { 4 | // address to bind to 5 | Bind string `json:"bind"` 6 | // name of the nntp server 7 | Name string `json:"name"` 8 | // default inbound article policy 9 | Article *ArticleConfig `json:"policy"` 10 | // do we allow anonymous NNTP sync? 11 | AnonNNTP bool `json:"anon-nntp"` 12 | // ssl settings for nntp 13 | SSL *SSLSettings 14 | // file with login credentials 15 | LoginsFile string `json:"authfile"` 16 | } 17 | 18 | var DefaultNNTPConfig = NNTPServerConfig{ 19 | AnonNNTP: false, 20 | Bind: "0.0.0.0:1119", 21 | Name: "nntp.server.tld", 22 | Article: &DefaultArticlePolicy, 23 | LoginsFile: "", 24 | } 25 | -------------------------------------------------------------------------------- /lib/config/proxy.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // proxy configuration 4 | type ProxyConfig struct { 5 | Type string `json:"type"` 6 | Addr string `json:"addr"` 7 | } 8 | 9 | // default tor proxy 10 | var DefaultTorProxy = ProxyConfig{ 11 | Type: "socks", 12 | Addr: "127.0.0.1:9050", 13 | } 14 | -------------------------------------------------------------------------------- /lib/config/ssl.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // settings for setting up ssl 4 | type SSLSettings struct { 5 | // path to ssl private key 6 | SSLKeyFile string `json:"key"` 7 | // path to ssl certificate signed by CA 8 | SSLCertFile string `json:"cert"` 9 | // domain name to use for ssl 10 | DomainName string `json:"fqdn"` 11 | } 12 | -------------------------------------------------------------------------------- /lib/config/store.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type StoreConfig struct { 4 | // path to article directory 5 | Path string `json:"path"` 6 | } 7 | 8 | var DefaultStoreConfig = StoreConfig{ 9 | Path: "storage", 10 | } 11 | -------------------------------------------------------------------------------- /lib/config/webhooks.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // configuration for a single web hook 4 | type WebhookConfig struct { 5 | // user provided name for this hook 6 | Name string `json:"name"` 7 | // callback URL for webhook 8 | URL string `json:"url"` 9 | // dialect to use when calling webhook 10 | Dialect string `json:"dialect"` 11 | } 12 | 13 | var DefaultWebHookConfig = &WebhookConfig{ 14 | Name: "vichan", 15 | Dialect: "vichan", 16 | URL: "http://localhost/webhook.php", 17 | } 18 | -------------------------------------------------------------------------------- /lib/crypto/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // nntpchan crypto package 3 | // wraps all external crypro libs 4 | // 5 | package crypto 6 | -------------------------------------------------------------------------------- /lib/crypto/hash.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "github.com/dchest/blake256" 5 | ) 6 | 7 | // common hash function is blake2 8 | var Hash = blake256.New 9 | -------------------------------------------------------------------------------- /lib/crypto/nacl.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "crypto/sha512" 5 | "hash" 6 | 7 | "github.com/majestrate/srndv2/lib/crypto/nacl" 8 | ) 9 | 10 | type fuckyNacl struct { 11 | k []byte 12 | hash hash.Hash 13 | } 14 | 15 | func (fucky *fuckyNacl) Write(d []byte) (int, error) { 16 | return fucky.hash.Write(d) 17 | } 18 | 19 | func (fucky *fuckyNacl) Sign() (s Signature) { 20 | h := fucky.hash.Sum(nil) 21 | if h == nil { 22 | panic("fuck.hash.Sum == nil") 23 | } 24 | kp := nacl.LoadSignKey(fucky.k) 25 | defer kp.Free() 26 | sk := kp.Secret() 27 | sig := nacl.CryptoSignFucky(h, sk) 28 | if sig == nil { 29 | panic("fucky signer's call to nacl.CryptoSignFucky returned nil") 30 | } 31 | s = Signature(sig) 32 | fucky.resetState() 33 | return 34 | } 35 | 36 | // reset inner state so we can reuse this fuckyNacl for another operation 37 | func (fucky *fuckyNacl) resetState() { 38 | fucky.hash = sha512.New() 39 | } 40 | 41 | func (fucky *fuckyNacl) Verify(sig Signature) (valid bool) { 42 | h := fucky.hash.Sum(nil) 43 | if h == nil { 44 | panic("fuck.hash.Sum == nil") 45 | } 46 | valid = nacl.CryptoVerifyFucky(h, sig, fucky.k) 47 | fucky.resetState() 48 | return 49 | } 50 | 51 | func createFucky(k []byte) *fuckyNacl { 52 | return &fuckyNacl{ 53 | k: k, 54 | hash: sha512.New(), 55 | } 56 | } 57 | 58 | // create a standard signer given a secret key 59 | func CreateSigner(sk []byte) Signer { 60 | return createFucky(sk) 61 | } 62 | 63 | // create a standard verifier given a public key 64 | func CreateVerifier(pk []byte) Verifer { 65 | return createFucky(pk) 66 | } 67 | 68 | // get the public component given the secret key 69 | func ToPublic(sk []byte) (pk []byte) { 70 | kp := nacl.LoadSignKey(sk) 71 | defer kp.Free() 72 | pk = kp.Public() 73 | return 74 | } 75 | 76 | // create a standard keypair 77 | func GenKeypair() (pk, sk []byte) { 78 | kp := nacl.GenSignKeypair() 79 | defer kp.Free() 80 | pk = kp.Public() 81 | sk = kp.Seed() 82 | return 83 | } 84 | -------------------------------------------------------------------------------- /lib/crypto/nacl/box.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | // #cgo freebsd CFLAGS: -I/usr/local/include 4 | // #cgo freebsd LDFLAGS: -L/usr/local/lib 5 | // #cgo LDFLAGS: -lsodium 6 | // #include 7 | import "C" 8 | 9 | import ( 10 | "errors" 11 | ) 12 | 13 | // encrypts a message to a user given their public key is known 14 | // returns an encrypted box 15 | func CryptoBox(msg, nounce, pk, sk []byte) ([]byte, error) { 16 | msgbuff := NewBuffer(msg) 17 | defer msgbuff.Free() 18 | 19 | // check sizes 20 | if len(pk) != int(C.crypto_box_publickeybytes()) { 21 | err := errors.New("len(pk) != crypto_box_publickey_bytes") 22 | return nil, err 23 | } 24 | if len(sk) != int(C.crypto_box_secretkeybytes()) { 25 | err := errors.New("len(sk) != crypto_box_secretkey_bytes") 26 | return nil, err 27 | } 28 | if len(nounce) != int(C.crypto_box_macbytes()) { 29 | err := errors.New("len(nounce) != crypto_box_macbytes()") 30 | return nil, err 31 | } 32 | 33 | pkbuff := NewBuffer(pk) 34 | defer pkbuff.Free() 35 | skbuff := NewBuffer(sk) 36 | defer skbuff.Free() 37 | nouncebuff := NewBuffer(nounce) 38 | defer nouncebuff.Free() 39 | 40 | resultbuff := malloc(msgbuff.size + nouncebuff.size) 41 | defer resultbuff.Free() 42 | res := C.crypto_box_easy(resultbuff.uchar(), msgbuff.uchar(), C.ulonglong(msgbuff.size), nouncebuff.uchar(), pkbuff.uchar(), skbuff.uchar()) 43 | if res != 0 { 44 | err := errors.New("crypto_box_easy failed") 45 | return nil, err 46 | } 47 | return resultbuff.Bytes(), nil 48 | } 49 | 50 | // open an encrypted box 51 | func CryptoBoxOpen(box, nounce, sk, pk []byte) ([]byte, error) { 52 | boxbuff := NewBuffer(box) 53 | defer boxbuff.Free() 54 | 55 | // check sizes 56 | if len(pk) != int(C.crypto_box_publickeybytes()) { 57 | err := errors.New("len(pk) != crypto_box_publickey_bytes") 58 | return nil, err 59 | } 60 | if len(sk) != int(C.crypto_box_secretkeybytes()) { 61 | err := errors.New("len(sk) != crypto_box_secretkey_bytes") 62 | return nil, err 63 | } 64 | if len(nounce) != int(C.crypto_box_macbytes()) { 65 | err := errors.New("len(nounce) != crypto_box_macbytes()") 66 | return nil, err 67 | } 68 | 69 | pkbuff := NewBuffer(pk) 70 | defer pkbuff.Free() 71 | skbuff := NewBuffer(sk) 72 | defer skbuff.Free() 73 | nouncebuff := NewBuffer(nounce) 74 | defer nouncebuff.Free() 75 | resultbuff := malloc(boxbuff.size - nouncebuff.size) 76 | defer resultbuff.Free() 77 | 78 | // decrypt 79 | res := C.crypto_box_open_easy(resultbuff.uchar(), boxbuff.uchar(), C.ulonglong(boxbuff.size), nouncebuff.uchar(), pkbuff.uchar(), skbuff.uchar()) 80 | if res != 0 { 81 | return nil, errors.New("crypto_box_open_easy failed") 82 | } 83 | // return result 84 | return resultbuff.Bytes(), nil 85 | } 86 | 87 | // generate a new nounce 88 | func NewBoxNounce() []byte { 89 | return RandBytes(NounceLen()) 90 | } 91 | 92 | // length of a nounce 93 | func NounceLen() int { 94 | return int(C.crypto_box_macbytes()) 95 | } 96 | -------------------------------------------------------------------------------- /lib/crypto/nacl/buffer.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | // #cgo freebsd CFLAGS: -I/usr/local/include 4 | // #cgo freebsd LDFLAGS: -L/usr/local/lib 5 | // #cgo LDFLAGS: -lsodium 6 | // #include 7 | // 8 | // unsigned char * deref_uchar(void * ptr) { return (unsigned char*) ptr; } 9 | // 10 | import "C" 11 | 12 | import ( 13 | "encoding/hex" 14 | "reflect" 15 | "unsafe" 16 | ) 17 | 18 | // wrapper arround malloc/free 19 | type Buffer struct { 20 | ptr unsafe.Pointer 21 | length C.int 22 | size C.size_t 23 | } 24 | 25 | // wrapper arround nacl.malloc 26 | func Malloc(size int) *Buffer { 27 | if size > 0 { 28 | return malloc(C.size_t(size)) 29 | } 30 | return nil 31 | } 32 | 33 | // does not check for negatives 34 | func malloc(size C.size_t) *Buffer { 35 | ptr := C.malloc(size) 36 | C.sodium_memzero(ptr, size) 37 | buffer := &Buffer{ptr: ptr, size: size, length: C.int(size)} 38 | return buffer 39 | } 40 | 41 | // create a new buffer copying from a byteslice 42 | func NewBuffer(buff []byte) *Buffer { 43 | buffer := Malloc(len(buff)) 44 | if buffer == nil { 45 | return nil 46 | } 47 | if copy(buffer.Data(), buff) != len(buff) { 48 | return nil 49 | } 50 | return buffer 51 | } 52 | 53 | func (self *Buffer) uchar() *C.uchar { 54 | return C.deref_uchar(self.ptr) 55 | } 56 | 57 | func (self *Buffer) Length() int { 58 | return int(self.length) 59 | } 60 | 61 | // get immutable byte slice 62 | func (self *Buffer) Bytes() []byte { 63 | buff := make([]byte, self.Length()) 64 | copy(buff, self.Data()) 65 | return buff 66 | } 67 | 68 | // get underlying byte slice 69 | func (self *Buffer) Data() []byte { 70 | hdr := reflect.SliceHeader{ 71 | Data: uintptr(self.ptr), 72 | Len: self.Length(), 73 | Cap: self.Length(), 74 | } 75 | return *(*[]byte)(unsafe.Pointer(&hdr)) 76 | } 77 | 78 | func (self *Buffer) String() string { 79 | return hex.EncodeToString(self.Data()) 80 | } 81 | 82 | // zero out memory and then free 83 | func (self *Buffer) Free() { 84 | C.sodium_memzero(self.ptr, self.size) 85 | C.free(self.ptr) 86 | } 87 | -------------------------------------------------------------------------------- /lib/crypto/nacl/key.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | // #cgo freebsd CFLAGS: -I/usr/local/include 4 | // #cgo freebsd LDFLAGS: -L/usr/local/lib 5 | // #cgo LDFLAGS: -lsodium 6 | // #include 7 | import "C" 8 | 9 | import ( 10 | "encoding/hex" 11 | "errors" 12 | "fmt" 13 | ) 14 | 15 | type KeyPair struct { 16 | pk *Buffer 17 | sk *Buffer 18 | } 19 | 20 | // free this keypair from memory 21 | func (self *KeyPair) Free() { 22 | self.pk.Free() 23 | self.sk.Free() 24 | } 25 | 26 | func (self *KeyPair) Secret() []byte { 27 | return self.sk.Bytes() 28 | } 29 | 30 | func (self *KeyPair) Public() []byte { 31 | return self.pk.Bytes() 32 | } 33 | 34 | func (self *KeyPair) Seed() []byte { 35 | seed_len := C.crypto_sign_seedbytes() 36 | return self.sk.Bytes()[:seed_len] 37 | } 38 | 39 | // generate a keypair 40 | func GenSignKeypair() *KeyPair { 41 | sk_len := C.crypto_sign_secretkeybytes() 42 | sk := malloc(sk_len) 43 | pk_len := C.crypto_sign_publickeybytes() 44 | pk := malloc(pk_len) 45 | res := C.crypto_sign_keypair(pk.uchar(), sk.uchar()) 46 | if res == 0 { 47 | return &KeyPair{pk, sk} 48 | } 49 | pk.Free() 50 | sk.Free() 51 | return nil 52 | } 53 | 54 | // get public key from secret key 55 | func GetSignPubkey(sk []byte) ([]byte, error) { 56 | sk_len := C.crypto_sign_secretkeybytes() 57 | if C.size_t(len(sk)) != sk_len { 58 | return nil, errors.New(fmt.Sprintf("nacl.GetSignPubkey() invalid secret key size %d != %d", len(sk), sk_len)) 59 | } 60 | 61 | pk_len := C.crypto_sign_publickeybytes() 62 | pkbuff := malloc(pk_len) 63 | defer pkbuff.Free() 64 | 65 | skbuff := NewBuffer(sk) 66 | defer skbuff.Free() 67 | //XXX: hack 68 | res := C.crypto_sign_seed_keypair(pkbuff.uchar(), skbuff.uchar(), skbuff.uchar()) 69 | 70 | if res != 0 { 71 | return nil, errors.New(fmt.Sprintf("nacl.GetSignPubkey() failed to get public key from secret key: %d", res)) 72 | } 73 | 74 | return pkbuff.Bytes(), nil 75 | } 76 | 77 | // make keypair from seed 78 | func LoadSignKey(seed []byte) *KeyPair { 79 | seed_len := C.crypto_sign_seedbytes() 80 | if C.size_t(len(seed)) != seed_len { 81 | return nil 82 | } 83 | seedbuff := NewBuffer(seed) 84 | defer seedbuff.Free() 85 | pk_len := C.crypto_sign_publickeybytes() 86 | sk_len := C.crypto_sign_secretkeybytes() 87 | pkbuff := malloc(pk_len) 88 | skbuff := malloc(sk_len) 89 | res := C.crypto_sign_seed_keypair(pkbuff.uchar(), skbuff.uchar(), seedbuff.uchar()) 90 | if res != 0 { 91 | pkbuff.Free() 92 | skbuff.Free() 93 | return nil 94 | } 95 | return &KeyPair{pkbuff, skbuff} 96 | } 97 | 98 | func GenBoxKeypair() *KeyPair { 99 | sk_len := C.crypto_box_secretkeybytes() 100 | sk := malloc(sk_len) 101 | pk_len := C.crypto_box_publickeybytes() 102 | pk := malloc(pk_len) 103 | res := C.crypto_box_keypair(pk.uchar(), sk.uchar()) 104 | if res == 0 { 105 | return &KeyPair{pk, sk} 106 | } 107 | pk.Free() 108 | sk.Free() 109 | return nil 110 | } 111 | 112 | // get public key from secret key 113 | func GetBoxPubkey(sk []byte) []byte { 114 | sk_len := C.crypto_box_seedbytes() 115 | if C.size_t(len(sk)) != sk_len { 116 | return nil 117 | } 118 | 119 | pk_len := C.crypto_box_publickeybytes() 120 | pkbuff := malloc(pk_len) 121 | defer pkbuff.Free() 122 | 123 | skbuff := NewBuffer(sk) 124 | defer skbuff.Free() 125 | 126 | // compute the public key 127 | C.crypto_scalarmult_base(pkbuff.uchar(), skbuff.uchar()) 128 | 129 | return pkbuff.Bytes() 130 | } 131 | 132 | // load keypair from secret key 133 | func LoadBoxKey(sk []byte) *KeyPair { 134 | pk := GetBoxPubkey(sk) 135 | if pk == nil { 136 | return nil 137 | } 138 | pkbuff := NewBuffer(pk) 139 | skbuff := NewBuffer(sk) 140 | return &KeyPair{pkbuff, skbuff} 141 | } 142 | 143 | // make keypair from seed 144 | func SeedBoxKey(seed []byte) *KeyPair { 145 | seed_len := C.crypto_box_seedbytes() 146 | if C.size_t(len(seed)) != seed_len { 147 | return nil 148 | } 149 | seedbuff := NewBuffer(seed) 150 | defer seedbuff.Free() 151 | pk_len := C.crypto_box_publickeybytes() 152 | sk_len := C.crypto_box_secretkeybytes() 153 | pkbuff := malloc(pk_len) 154 | skbuff := malloc(sk_len) 155 | res := C.crypto_box_seed_keypair(pkbuff.uchar(), skbuff.uchar(), seedbuff.uchar()) 156 | if res != 0 { 157 | pkbuff.Free() 158 | skbuff.Free() 159 | return nil 160 | } 161 | return &KeyPair{pkbuff, skbuff} 162 | } 163 | 164 | func (self *KeyPair) String() string { 165 | return fmt.Sprintf("pk=%s sk=%s", hex.EncodeToString(self.pk.Data()), hex.EncodeToString(self.sk.Data())) 166 | } 167 | 168 | func CryptoSignPublicLen() int { 169 | return int(C.crypto_sign_publickeybytes()) 170 | } 171 | 172 | func CryptoSignSecretLen() int { 173 | return int(C.crypto_sign_secretkeybytes()) 174 | } 175 | 176 | func CryptoSignSeedLen() int { 177 | return int(C.crypto_sign_seedbytes()) 178 | } 179 | -------------------------------------------------------------------------------- /lib/crypto/nacl/nacl.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | // #cgo freebsd CFLAGS: -I/usr/local/include 4 | // #cgo freebsd LDFLAGS: -L/usr/local/lib 5 | // #cgo LDFLAGS: -lsodium 6 | // #include 7 | import "C" 8 | 9 | import ( 10 | "log" 11 | ) 12 | 13 | // return how many bytes overhead does CryptoBox have 14 | func CryptoBoxOverhead() int { 15 | return int(C.crypto_box_macbytes()) 16 | } 17 | 18 | // size of crypto_box public keys 19 | func CryptoBoxPubKeySize() int { 20 | return int(C.crypto_box_publickeybytes()) 21 | } 22 | 23 | // size of crypto_box private keys 24 | func CryptoBoxPrivKeySize() int { 25 | return int(C.crypto_box_secretkeybytes()) 26 | } 27 | 28 | // size of crypto_sign public keys 29 | func CryptoSignPubKeySize() int { 30 | return int(C.crypto_sign_publickeybytes()) 31 | } 32 | 33 | // size of crypto_sign private keys 34 | func CryptoSignPrivKeySize() int { 35 | return int(C.crypto_sign_secretkeybytes()) 36 | } 37 | 38 | // initialize sodium 39 | func init() { 40 | status := C.sodium_init() 41 | if status == -1 { 42 | log.Fatalf("failed to initialize libsodium status=%d", status) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lib/crypto/nacl/rand.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | // #cgo freebsd CFLAGS: -I/usr/local/include 4 | // #cgo freebsd LDFLAGS: -L/usr/local/lib 5 | // #cgo LDFLAGS: -lsodium 6 | // #include 7 | import "C" 8 | 9 | func randbytes(size C.size_t) *Buffer { 10 | 11 | buff := malloc(size) 12 | C.randombytes_buf(buff.ptr, size) 13 | return buff 14 | 15 | } 16 | 17 | func RandBytes(size int) []byte { 18 | if size > 0 { 19 | buff := randbytes(C.size_t(size)) 20 | defer buff.Free() 21 | return buff.Bytes() 22 | } 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /lib/crypto/nacl/sign.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | // #cgo freebsd CFLAGS: -I/usr/local/include 4 | // #cgo freebsd LDFLAGS: -L/usr/local/lib 5 | // #cgo LDFLAGS: -lsodium 6 | // #include 7 | import "C" 8 | 9 | // sign data detached with secret key sk 10 | func CryptoSignDetached(msg, sk []byte) []byte { 11 | msgbuff := NewBuffer(msg) 12 | defer msgbuff.Free() 13 | skbuff := NewBuffer(sk) 14 | defer skbuff.Free() 15 | if skbuff.size != C.crypto_sign_bytes() { 16 | return nil 17 | } 18 | 19 | // allocate the signature buffer 20 | sig := malloc(C.crypto_sign_bytes()) 21 | defer sig.Free() 22 | // compute signature 23 | siglen := C.ulonglong(0) 24 | res := C.crypto_sign_detached(sig.uchar(), &siglen, msgbuff.uchar(), C.ulonglong(msgbuff.size), skbuff.uchar()) 25 | if res == 0 && siglen == C.ulonglong(C.crypto_sign_bytes()) { 26 | // return copy of signature buffer 27 | return sig.Bytes() 28 | } 29 | // failure to sign 30 | return nil 31 | } 32 | 33 | // sign data with secret key sk 34 | // return detached sig 35 | // this uses crypto_sign instead pf crypto_sign_detached 36 | func CryptoSignFucky(msg, sk []byte) []byte { 37 | msgbuff := NewBuffer(msg) 38 | defer msgbuff.Free() 39 | skbuff := NewBuffer(sk) 40 | defer skbuff.Free() 41 | if skbuff.size != C.crypto_sign_bytes() { 42 | return nil 43 | } 44 | 45 | // allocate the signed message buffer 46 | sig := malloc(C.crypto_sign_bytes() + msgbuff.size) 47 | defer sig.Free() 48 | // compute signature 49 | siglen := C.ulonglong(0) 50 | res := C.crypto_sign(sig.uchar(), &siglen, msgbuff.uchar(), C.ulonglong(msgbuff.size), skbuff.uchar()) 51 | if res == 0 { 52 | // return copy of signature inside the signed message 53 | offset := int(C.crypto_sign_bytes()) 54 | return sig.Bytes()[:offset] 55 | } 56 | // failure to sign 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /lib/crypto/nacl/stream.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "net" 8 | "time" 9 | ) 10 | 11 | // TOY encrypted authenticated stream protocol like tls 12 | 13 | var BadHandshake = errors.New("Bad handshake") 14 | var ShortWrite = errors.New("short write") 15 | var ShortRead = errors.New("short read") 16 | var Closed = errors.New("socket closed") 17 | 18 | // write boxes at 512 bytes at a time 19 | const DefaultMTU = 512 20 | 21 | // wrapper arround crypto_box 22 | // provides an authenticated encrypted stream 23 | // this is a TOY 24 | type CryptoStream struct { 25 | // underlying stream to write on 26 | stream io.ReadWriteCloser 27 | // secret key seed 28 | key *KeyPair 29 | // public key of who we expect on the other end 30 | remote_pk []byte 31 | tx_nonce []byte 32 | rx_nonce []byte 33 | // box size 34 | mtu int 35 | } 36 | 37 | func (cs *CryptoStream) Close() (err error) { 38 | if cs.key != nil { 39 | cs.key.Free() 40 | cs.key = nil 41 | } 42 | return cs.stream.Close() 43 | } 44 | 45 | // implements io.Writer 46 | func (cs *CryptoStream) Write(data []byte) (n int, err error) { 47 | // let's split it up 48 | for n < len(data) && err == nil { 49 | if n+cs.mtu < len(data) { 50 | err = cs.writeSegment(data[n : n+cs.mtu]) 51 | n += cs.mtu 52 | } else { 53 | err = cs.writeSegment(data[n:]) 54 | if err == nil { 55 | n = len(data) 56 | } 57 | } 58 | } 59 | return 60 | } 61 | 62 | func (cs *CryptoStream) public() (p []byte) { 63 | p = cs.key.Public() 64 | return 65 | } 66 | 67 | func (cs *CryptoStream) secret() (s []byte) { 68 | s = cs.key.Secret() 69 | return 70 | } 71 | 72 | // read 1 segment 73 | func (cs *CryptoStream) readSegment() (s []byte, err error) { 74 | var stream_read int 75 | var seg []byte 76 | nl := NounceLen() 77 | msg := make([]byte, cs.mtu+nl) 78 | stream_read, err = cs.stream.Read(msg) 79 | seg, err = CryptoBoxOpen(msg[:stream_read], cs.rx_nonce, cs.secret(), cs.remote_pk) 80 | if err == nil { 81 | copy(cs.rx_nonce, seg[:nl]) 82 | s = seg[nl:] 83 | } 84 | return 85 | } 86 | 87 | // write 1 segment encrypted 88 | // update nounce 89 | func (cs *CryptoStream) writeSegment(data []byte) (err error) { 90 | var segment []byte 91 | nl := NounceLen() 92 | msg := make([]byte, len(data)+nl) 93 | // generate next nounce 94 | nextNounce := NewBoxNounce() 95 | copy(msg, nextNounce) 96 | copy(msg[nl:], data) 97 | // encrypt segment with current nounce 98 | segment, err = CryptoBox(data, cs.tx_nonce, cs.remote_pk, cs.secret()) 99 | var n int 100 | n, err = cs.stream.Write(segment) 101 | if n != len(segment) { 102 | // short write? 103 | err = ShortWrite 104 | return 105 | } 106 | // update nounce 107 | copy(cs.tx_nonce, nextNounce) 108 | return 109 | } 110 | 111 | // implements io.Reader 112 | func (cs *CryptoStream) Read(data []byte) (n int, err error) { 113 | var seg []byte 114 | seg, err = cs.readSegment() 115 | if err == nil { 116 | if len(seg) <= len(data) { 117 | copy(data, seg) 118 | n = len(seg) 119 | } else { 120 | // too big? 121 | err = ShortRead 122 | } 123 | } 124 | return 125 | } 126 | 127 | // version 0 protocol magic 128 | var protocol_magic = []byte("BENIS|00") 129 | 130 | // verify that a handshake is signed right and is in the correct format etc 131 | func verifyHandshake(hs, pk []byte) (valid bool) { 132 | ml := len(protocol_magic) 133 | // valid handshake? 134 | if bytes.Equal(hs[0:ml], protocol_magic) { 135 | // check pk 136 | pl := CryptoSignPublicLen() 137 | nl := NounceLen() 138 | if bytes.Equal(pk, hs[ml:ml+pl]) { 139 | // check signature 140 | msg := hs[0 : ml+pl+nl] 141 | sig := hs[ml+pl+nl:] 142 | valid = CryptoVerifyFucky(msg, sig, pk) 143 | } 144 | } 145 | return 146 | } 147 | 148 | // get claimed public key from handshake 149 | func getPubkey(hs []byte) (pk []byte) { 150 | ml := len(protocol_magic) 151 | pl := CryptoSignPublicLen() 152 | pk = hs[ml : ml+pl] 153 | return 154 | } 155 | 156 | func (cs *CryptoStream) genHandshake() (d []byte) { 157 | // protocol magic string version 00 158 | // Benis Encrypted Network Information Stream 159 | // :-DDDDD meme crypto 160 | d = append(d, protocol_magic...) 161 | // our public key 162 | d = append(d, cs.public()...) 163 | // nounce 164 | cs.tx_nonce = NewBoxNounce() 165 | d = append(d, cs.tx_nonce...) 166 | // sign protocol magic string, nounce and pubkey 167 | sig := CryptoSignFucky(d, cs.secret()) 168 | // if sig is nil we'll just die 169 | d = append(d, sig...) 170 | return 171 | } 172 | 173 | // extract nounce from handshake 174 | func getNounce(hs []byte) (n []byte) { 175 | ml := len(protocol_magic) 176 | pl := CryptoSignPublicLen() 177 | nl := NounceLen() 178 | n = hs[ml+pl : ml+pl+nl] 179 | return 180 | } 181 | 182 | // initiate protocol handshake 183 | func (cs *CryptoStream) Handshake() (err error) { 184 | // send them our info 185 | hs := cs.genHandshake() 186 | var n int 187 | n, err = cs.stream.Write(hs) 188 | if n != len(hs) { 189 | err = ShortWrite 190 | return 191 | } 192 | // read thier info 193 | buff := make([]byte, len(hs)) 194 | _, err = io.ReadFull(cs.stream, buff) 195 | 196 | if cs.remote_pk == nil { 197 | // inbound 198 | pk := getPubkey(buff) 199 | cs.remote_pk = make([]byte, len(pk)) 200 | copy(cs.remote_pk, pk) 201 | } 202 | 203 | if !verifyHandshake(buff, cs.remote_pk) { 204 | // verification failed 205 | err = BadHandshake 206 | return 207 | } 208 | cs.rx_nonce = make([]byte, NounceLen()) 209 | copy(cs.rx_nonce, getNounce(buff)) 210 | return 211 | } 212 | 213 | // create a client 214 | func Client(stream io.ReadWriteCloser, local_sk, remote_pk []byte) (c *CryptoStream) { 215 | c = &CryptoStream{ 216 | stream: stream, 217 | mtu: DefaultMTU, 218 | } 219 | c.remote_pk = make([]byte, len(remote_pk)) 220 | copy(c.remote_pk, remote_pk) 221 | c.key = LoadSignKey(local_sk) 222 | if c.key == nil { 223 | return nil 224 | } 225 | return c 226 | } 227 | 228 | type CryptoConn struct { 229 | stream *CryptoStream 230 | conn net.Conn 231 | } 232 | 233 | func (cc *CryptoConn) Close() (err error) { 234 | err = cc.stream.Close() 235 | return 236 | } 237 | 238 | func (cc *CryptoConn) Write(d []byte) (n int, err error) { 239 | return cc.stream.Write(d) 240 | } 241 | 242 | func (cc *CryptoConn) Read(d []byte) (n int, err error) { 243 | return cc.stream.Read(d) 244 | } 245 | 246 | func (cc *CryptoConn) LocalAddr() net.Addr { 247 | return cc.conn.LocalAddr() 248 | } 249 | 250 | func (cc *CryptoConn) RemoteAddr() net.Addr { 251 | return cc.conn.RemoteAddr() 252 | } 253 | 254 | func (cc *CryptoConn) SetDeadline(t time.Time) (err error) { 255 | return cc.conn.SetDeadline(t) 256 | } 257 | 258 | func (cc *CryptoConn) SetReadDeadline(t time.Time) (err error) { 259 | return cc.conn.SetReadDeadline(t) 260 | } 261 | 262 | func (cc *CryptoConn) SetWriteDeadline(t time.Time) (err error) { 263 | return cc.conn.SetWriteDeadline(t) 264 | } 265 | 266 | type CryptoListener struct { 267 | l net.Listener 268 | handshake chan net.Conn 269 | accepted chan *CryptoConn 270 | trust func(pk []byte) bool 271 | key *KeyPair 272 | } 273 | 274 | func (cl *CryptoListener) Close() (err error) { 275 | err = cl.l.Close() 276 | close(cl.accepted) 277 | close(cl.handshake) 278 | cl.key.Free() 279 | cl.key = nil 280 | return 281 | } 282 | 283 | func (cl *CryptoListener) acceptInbound() { 284 | for { 285 | c, err := cl.l.Accept() 286 | if err == nil { 287 | cl.handshake <- c 288 | } else { 289 | return 290 | } 291 | } 292 | } 293 | 294 | func (cl *CryptoListener) runChans() { 295 | for { 296 | select { 297 | case c := <-cl.handshake: 298 | go func() { 299 | s := &CryptoStream{ 300 | stream: c, 301 | mtu: DefaultMTU, 302 | key: cl.key, 303 | } 304 | err := s.Handshake() 305 | if err == nil { 306 | // we gud handshake was okay 307 | if cl.trust(s.remote_pk) { 308 | // the key is trusted okay 309 | cl.accepted <- &CryptoConn{stream: s, conn: c} 310 | } else { 311 | // not trusted, close connection 312 | s.Close() 313 | } 314 | } 315 | }() 316 | } 317 | } 318 | } 319 | 320 | // accept inbound authenticated and trusted connections 321 | func (cl *CryptoListener) Accept() (c net.Conn, err error) { 322 | var ok bool 323 | c, ok = <-cl.accepted 324 | if !ok { 325 | err = Closed 326 | } 327 | return 328 | } 329 | 330 | // create a listener 331 | func Server(l net.Listener, local_sk []byte, trust func(pk []byte) bool) (s *CryptoListener) { 332 | s = &CryptoListener{ 333 | l: l, 334 | trust: trust, 335 | handshake: make(chan net.Conn), 336 | accepted: make(chan *CryptoConn), 337 | } 338 | s.key = LoadSignKey(local_sk) 339 | go s.runChans() 340 | go s.acceptInbound() 341 | return 342 | } 343 | -------------------------------------------------------------------------------- /lib/crypto/nacl/verfiy.go: -------------------------------------------------------------------------------- 1 | package nacl 2 | 3 | // #cgo freebsd CFLAGS: -I/usr/local/include 4 | // #cgo freebsd LDFLAGS: -L/usr/local/lib 5 | // #cgo LDFLAGS: -lsodium 6 | // #include 7 | import "C" 8 | 9 | // verify a fucky detached sig 10 | func CryptoVerifyFucky(msg, sig, pk []byte) bool { 11 | var smsg []byte 12 | smsg = append(smsg, sig...) 13 | smsg = append(smsg, msg...) 14 | return CryptoVerify(smsg, pk) 15 | } 16 | 17 | // verify a signed message 18 | func CryptoVerify(smsg, pk []byte) bool { 19 | smsg_buff := NewBuffer(smsg) 20 | defer smsg_buff.Free() 21 | pk_buff := NewBuffer(pk) 22 | defer pk_buff.Free() 23 | 24 | if pk_buff.size != C.crypto_sign_publickeybytes() { 25 | return false 26 | } 27 | mlen := C.ulonglong(0) 28 | msg := malloc(C.size_t(len(smsg))) 29 | defer msg.Free() 30 | smlen := C.ulonglong(smsg_buff.size) 31 | return C.crypto_sign_open(msg.uchar(), &mlen, smsg_buff.uchar(), smlen, pk_buff.uchar()) != -1 32 | } 33 | 34 | // verfiy a detached signature 35 | // return true on valid otherwise false 36 | func CryptoVerifyDetached(msg, sig, pk []byte) bool { 37 | msg_buff := NewBuffer(msg) 38 | defer msg_buff.Free() 39 | sig_buff := NewBuffer(sig) 40 | defer sig_buff.Free() 41 | pk_buff := NewBuffer(pk) 42 | defer pk_buff.Free() 43 | 44 | if pk_buff.size != C.crypto_sign_publickeybytes() { 45 | return false 46 | } 47 | 48 | // invalid sig size 49 | if sig_buff.size != C.crypto_sign_bytes() { 50 | return false 51 | } 52 | return C.crypto_sign_verify_detached(sig_buff.uchar(), msg_buff.uchar(), C.ulonglong(len(msg)), pk_buff.uchar()) == 0 53 | } 54 | -------------------------------------------------------------------------------- /lib/crypto/nacl_test.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | func TestNaclToPublic(t *testing.T) { 11 | pk, sk := GenKeypair() 12 | t_pk := ToPublic(sk) 13 | if !bytes.Equal(pk, t_pk) { 14 | t.Logf("%q != %q", pk, t_pk) 15 | t.Fail() 16 | } 17 | } 18 | 19 | func TestNaclSignVerify(t *testing.T) { 20 | var msg [1024]byte 21 | pk, sk := GenKeypair() 22 | io.ReadFull(rand.Reader, msg[:]) 23 | 24 | signer := CreateSigner(sk) 25 | signer.Write(msg[:]) 26 | sig := signer.Sign() 27 | 28 | verifier := CreateVerifier(pk) 29 | verifier.Write(msg[:]) 30 | if !verifier.Verify(sig) { 31 | t.Logf("%q is invalid signature and is %dB long", sig, len(sig)) 32 | t.Fail() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /lib/crypto/rand.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/crypto/nacl" 5 | ) 6 | 7 | // generate random bytes 8 | var RandBytes = nacl.RandBytes 9 | -------------------------------------------------------------------------------- /lib/crypto/sig.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import "io" 4 | 5 | // a detached signature 6 | type Signature []byte 7 | 8 | type SigEncoder interface { 9 | // encode a signature to an io.Writer 10 | // return error if one occurrened while writing out signature 11 | Encode(sig Signature, w io.Writer) error 12 | // encode a signature to a string 13 | EncodeString(sig Signature) string 14 | } 15 | 16 | // a decoder of signatures 17 | type SigDecoder interface { 18 | // decode signature from io.Reader 19 | // reads all data until io.EOF 20 | // returns singaure or error if an error occured while reading 21 | Decode(r io.Reader) (Signature, error) 22 | // decode a signature from string 23 | // returns signature or error if an error ocurred while decoding 24 | DecodeString(str string) (Signature, error) 25 | } 26 | -------------------------------------------------------------------------------- /lib/crypto/sign.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import "io" 4 | 5 | // 6 | // provides generic signing interface for producing detached signatures 7 | // call Write() to feed data to be signed, call Sign() to generate 8 | // a detached signature 9 | // 10 | type Signer interface { 11 | io.Writer 12 | // generate detached Signature from previously fed body via Write() 13 | Sign() Signature 14 | } 15 | -------------------------------------------------------------------------------- /lib/crypto/verify.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import "io" 4 | 5 | // provides generic signature 6 | // call Write() to feed in message body 7 | // once the entire body has been fed in via Write() call Verify() with detached 8 | // signature to verify the detached signature against the previously fed body 9 | type Verifer interface { 10 | io.Writer 11 | // verify detached signature from body previously fed via Write() 12 | // return true if the detached signature is valid given the body 13 | Verify(sig Signature) bool 14 | } 15 | -------------------------------------------------------------------------------- /lib/database/database.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "errors" 5 | "github.com/majestrate/srndv2/lib/config" 6 | "github.com/majestrate/srndv2/lib/model" 7 | "strings" 8 | ) 9 | 10 | // 11 | type Database interface { 12 | ThreadByMessageID(msgid string) (*model.Thread, error) 13 | ThreadByHash(hash string) (*model.Thread, error) 14 | BoardPage(newsgroup string, pageno, perpage int) (*model.BoardPage, error) 15 | } 16 | 17 | // get new database connector from configuration 18 | func NewDBFromConfig(c *config.DatabaseConfig) (db Database, err error) { 19 | dbtype := strings.ToLower(c.Type) 20 | if dbtype == "postgres" { 21 | db, err = createPostgresDatabase(c.Addr, c.Username, c.Password) 22 | } else { 23 | err = errors.New("no such database driver: " + c.Type) 24 | } 25 | return 26 | } 27 | -------------------------------------------------------------------------------- /lib/database/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // database driver 3 | // 4 | package database 5 | -------------------------------------------------------------------------------- /lib/database/postgres.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/model" 5 | ) 6 | 7 | type PostgresDB struct { 8 | } 9 | 10 | func (db *PostgresDB) ThreadByMessageID(msgid string) (thread *model.Thread, err error) { 11 | 12 | return 13 | } 14 | 15 | func (db *PostgresDB) ThreadByHash(hash string) (thread *model.Thread, err error) { 16 | 17 | return 18 | } 19 | 20 | func (db *PostgresDB) BoardPage(newsgroup string, pageno, perpage int) (page *model.BoardPage, err error) { 21 | 22 | return 23 | } 24 | 25 | func createPostgresDatabase(addr, user, passwd string) (p *PostgresDB, err error) { 26 | 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /lib/frontend/captcha.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/dchest/captcha" 8 | "github.com/gorilla/mux" 9 | "github.com/gorilla/sessions" 10 | "github.com/majestrate/srndv2/lib/config" 11 | "net/http" 12 | ) 13 | 14 | // server of captchas 15 | // implements frontend.Middleware 16 | type CaptchaServer struct { 17 | h int 18 | w int 19 | store *sessions.CookieStore 20 | prefix string 21 | sessionName string 22 | } 23 | 24 | // create new captcha server using existing session store 25 | func NewCaptchaServer(w, h int, prefix string, store *sessions.CookieStore) *CaptchaServer { 26 | return &CaptchaServer{ 27 | h: h, 28 | w: w, 29 | prefix: prefix, 30 | store: store, 31 | sessionName: "captcha", 32 | } 33 | } 34 | 35 | func (cs *CaptchaServer) Reload(c *config.MiddlewareConfig) { 36 | 37 | } 38 | 39 | func (cs *CaptchaServer) SetupRoutes(m *mux.Router) { 40 | m.Path("/new").HandlerFunc(cs.NewCaptcha) 41 | m.Path("/img/{f}").Handler(captcha.Server(cs.w, cs.h)) 42 | m.Path("/verify.json").HandlerFunc(cs.VerifyCaptcha) 43 | } 44 | 45 | // return true if this session has solved the last captcha given provided solution, otherwise false 46 | func (cs *CaptchaServer) CheckSession(w http.ResponseWriter, r *http.Request, solution string) (bool, error) { 47 | s, err := cs.store.Get(r, cs.sessionName) 48 | if err == nil { 49 | id, ok := s.Values["captcha_id"] 50 | if ok { 51 | return captcha.VerifyString(id.(string), solution), nil 52 | } 53 | } 54 | return false, err 55 | } 56 | 57 | // verify a captcha 58 | func (cs *CaptchaServer) VerifyCaptcha(w http.ResponseWriter, r *http.Request) { 59 | dec := json.NewDecoder(r.Body) 60 | defer r.Body.Close() 61 | // request 62 | req := make(map[string]string) 63 | // response 64 | resp := make(map[string]interface{}) 65 | resp["solved"] = false 66 | // decode request 67 | err := dec.Decode(req) 68 | if err == nil { 69 | // decode okay 70 | id, ok := req["id"] 71 | if ok { 72 | // we have id 73 | solution, ok := req["solution"] 74 | if ok { 75 | // we have solution and id 76 | resp["solved"] = captcha.VerifyString(id, solution) 77 | } else { 78 | // we don't have solution 79 | err = errors.New("no captcha solution provided") 80 | } 81 | } else { 82 | // we don't have id 83 | err = errors.New("no captcha id provided") 84 | } 85 | } 86 | if err != nil { 87 | // error happened 88 | resp["error"] = err.Error() 89 | } 90 | // send reply 91 | w.Header().Set("Content-Type", "text/json; encoding=UTF-8") 92 | enc := json.NewEncoder(w) 93 | enc.Encode(resp) 94 | } 95 | 96 | // generate a new captcha 97 | func (cs *CaptchaServer) NewCaptcha(w http.ResponseWriter, r *http.Request) { 98 | // obtain session 99 | sess, err := cs.store.Get(r, cs.sessionName) 100 | if err != nil { 101 | // failed to obtain session 102 | http.Error(w, err.Error(), http.StatusInternalServerError) 103 | return 104 | } 105 | // new captcha 106 | id := captcha.New() 107 | // do we want to interpret as json? 108 | use_json := r.URL.Query().Get("t") == "json" 109 | // image url 110 | url := fmt.Sprintf("%simg/%s.png", cs.prefix, id) 111 | if use_json { 112 | // send json 113 | enc := json.NewEncoder(w) 114 | enc.Encode(map[string]string{"id": id, "url": url}) 115 | } else { 116 | // set captcha id 117 | sess.Values["captcha_id"] = id 118 | // save session 119 | sess.Save(r, w) 120 | // rediect to image 121 | http.Redirect(w, r, url, http.StatusFound) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /lib/frontend/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // nntpchan frontend 3 | // allows posting to nntpchan network via various implementations 4 | // 5 | package frontend 6 | -------------------------------------------------------------------------------- /lib/frontend/frontend.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/config" 5 | "github.com/majestrate/srndv2/lib/database" 6 | "github.com/majestrate/srndv2/lib/model" 7 | "github.com/majestrate/srndv2/lib/nntp" 8 | ) 9 | 10 | // a frontend that displays nntp posts and allows posting 11 | type Frontend interface { 12 | 13 | // run mainloop 14 | Serve() 15 | 16 | // do we accept this inbound post? 17 | AllowPost(p model.PostReference) bool 18 | 19 | // trigger a manual regen of indexes for a root post 20 | Regen(p model.PostReference) 21 | 22 | // implements nntp.EventHooks 23 | GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) 24 | 25 | // implements nntp.EventHooks 26 | SentArticleVia(msgid nntp.MessageID, feedname string) 27 | 28 | // reload config 29 | Reload(c *config.FrontendConfig) 30 | } 31 | 32 | // create a new http frontend give frontend config 33 | func NewHTTPFrontend(c *config.FrontendConfig, db database.Database) (f Frontend, err error) { 34 | 35 | var mid Middleware 36 | if c.Middleware != nil { 37 | // middleware configured 38 | mid, err = OverchanMiddleware(c.Middleware, db) 39 | } 40 | 41 | if err == nil { 42 | // create http frontend only if no previous errors 43 | f, err = createHttpFrontend(c, mid, db) 44 | } 45 | return 46 | } 47 | -------------------------------------------------------------------------------- /lib/frontend/http.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "fmt" 5 | log "github.com/Sirupsen/logrus" 6 | "github.com/gorilla/mux" 7 | "github.com/majestrate/srndv2/lib/admin" 8 | "github.com/majestrate/srndv2/lib/api" 9 | "github.com/majestrate/srndv2/lib/config" 10 | "github.com/majestrate/srndv2/lib/database" 11 | "github.com/majestrate/srndv2/lib/model" 12 | "github.com/majestrate/srndv2/lib/nntp" 13 | "net/http" 14 | "time" 15 | ) 16 | 17 | // http frontend server 18 | // provides glue layer between nntp and middleware 19 | type httpFrontend struct { 20 | // bind address 21 | addr string 22 | // http mux 23 | httpmux *mux.Router 24 | // admin panel 25 | adminPanel *admin.Server 26 | // static files path 27 | staticDir string 28 | // http middleware 29 | middleware Middleware 30 | // api server 31 | apiserve *api.Server 32 | // database driver 33 | db database.Database 34 | } 35 | 36 | // reload http frontend 37 | // reloads middleware 38 | func (f *httpFrontend) Reload(c *config.FrontendConfig) { 39 | if f.middleware == nil { 40 | if c.Middleware != nil { 41 | var err error 42 | // no middleware set, create middleware 43 | f.middleware, err = OverchanMiddleware(c.Middleware, f.db) 44 | if err != nil { 45 | log.Errorf("overchan middleware reload failed: %s", err.Error()) 46 | } 47 | 48 | } 49 | } else { 50 | // middleware exists 51 | // do middleware reload 52 | f.middleware.Reload(c.Middleware) 53 | } 54 | 55 | } 56 | 57 | // serve http requests from net.Listener 58 | func (f *httpFrontend) Serve() { 59 | // serve http 60 | for { 61 | err := http.ListenAndServe(f.addr, f.httpmux) 62 | if err != nil { 63 | log.Errorf("failed to listen and serve with frontend: %s", err) 64 | } 65 | time.Sleep(time.Second) 66 | } 67 | } 68 | 69 | // serve robots.txt page 70 | func (f *httpFrontend) serveRobots(w http.ResponseWriter, r *http.Request) { 71 | fmt.Fprintf(w, "User-Agent: *\nDisallow: /\n") 72 | } 73 | 74 | func (f *httpFrontend) AllowPost(p model.PostReference) bool { 75 | // TODO: implement 76 | return true 77 | } 78 | 79 | func (f *httpFrontend) Regen(p model.PostReference) { 80 | // TODO: implement 81 | } 82 | 83 | func (f *httpFrontend) GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) { 84 | // TODO: implement 85 | } 86 | 87 | func (f *httpFrontend) SentArticleVia(msgid nntp.MessageID, feedname string) { 88 | // TODO: implement 89 | } 90 | 91 | func createHttpFrontend(c *config.FrontendConfig, mid Middleware, db database.Database) (f *httpFrontend, err error) { 92 | f = new(httpFrontend) 93 | // set db 94 | // db.Ensure() called elsewhere 95 | f.db = db 96 | 97 | // set bind address 98 | f.addr = c.BindAddr 99 | 100 | // set up mux 101 | f.httpmux = mux.NewRouter() 102 | 103 | // set up admin panel 104 | f.adminPanel = admin.NewServer() 105 | 106 | // set static files dir 107 | f.staticDir = c.Static 108 | 109 | // set middleware 110 | f.middleware = mid 111 | 112 | // set up routes 113 | 114 | if f.adminPanel != nil { 115 | // route up admin panel 116 | f.httpmux.PathPrefix("/admin/").Handler(f.adminPanel) 117 | } 118 | 119 | if f.middleware != nil { 120 | // route up middleware 121 | f.middleware.SetupRoutes(f.httpmux) 122 | } 123 | 124 | if f.apiserve != nil { 125 | // route up api 126 | f.apiserve.SetupRoutes(f.httpmux.PathPrefix("/api/").Subrouter()) 127 | } 128 | 129 | // route up robots.txt 130 | f.httpmux.Path("/robots.txt").HandlerFunc(f.serveRobots) 131 | 132 | // route up static files 133 | f.httpmux.PathPrefix("/static/").Handler(http.FileServer(http.Dir(f.staticDir))) 134 | 135 | return 136 | } 137 | -------------------------------------------------------------------------------- /lib/frontend/middleware.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "github.com/majestrate/srndv2/lib/config" 6 | ) 7 | 8 | // http middleware 9 | type Middleware interface { 10 | // set up routes 11 | SetupRoutes(m *mux.Router) 12 | // reload with new configuration 13 | Reload(c *config.MiddlewareConfig) 14 | } 15 | -------------------------------------------------------------------------------- /lib/frontend/overchan.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | log "github.com/Sirupsen/logrus" 5 | "github.com/gorilla/mux" 6 | "github.com/gorilla/sessions" 7 | "github.com/majestrate/srndv2/lib/config" 8 | "github.com/majestrate/srndv2/lib/database" 9 | "html/template" 10 | "net/http" 11 | "path/filepath" 12 | "strconv" 13 | ) 14 | 15 | // standard overchan imageboard middleware 16 | type overchanMiddleware struct { 17 | templ *template.Template 18 | captcha *CaptchaServer 19 | store *sessions.CookieStore 20 | db database.Database 21 | } 22 | 23 | func (m *overchanMiddleware) SetupRoutes(mux *mux.Router) { 24 | // setup front page handler 25 | mux.Path("/").HandlerFunc(m.ServeIndex) 26 | // setup thread handler 27 | mux.Path("/t/{id}/").HandlerFunc(m.ServeThread) 28 | // setup board page handler 29 | mux.Path("/b/{name}/").HandlerFunc(m.ServeBoardPage) 30 | // setup posting endpoint 31 | mux.Path("/post") 32 | // create captcha 33 | captchaPrefix := "/captcha/" 34 | m.captcha = NewCaptchaServer(200, 400, captchaPrefix, m.store) 35 | // setup captcha endpoint 36 | m.captcha.SetupRoutes(mux.PathPrefix(captchaPrefix).Subrouter()) 37 | } 38 | 39 | // reload middleware 40 | func (m *overchanMiddleware) Reload(c *config.MiddlewareConfig) { 41 | // reload templates 42 | templ, err := template.ParseGlob(filepath.Join(c.Templates, "*.tmpl")) 43 | if err == nil { 44 | log.Infof("middleware reloaded templates") 45 | m.templ = templ 46 | } else { 47 | log.Errorf("middleware reload failed: %s", err.Error()) 48 | } 49 | } 50 | 51 | func (m *overchanMiddleware) ServeBoardPage(w http.ResponseWriter, r *http.Request) { 52 | param := mux.Vars(r) 53 | board := param["name"] 54 | page := r.URL.Query().Get("q") 55 | pageno, err := strconv.Atoi(page) 56 | if err == nil { 57 | var obj interface{} 58 | obj, err = m.db.BoardPage(board, pageno, 10) 59 | if err == nil { 60 | m.serveTemplate(w, r, "board.html.tmpl", obj) 61 | } else { 62 | m.serveTemplate(w, r, "error.html.tmpl", err) 63 | } 64 | } else { 65 | // 404 66 | http.NotFound(w, r) 67 | } 68 | } 69 | 70 | // serve cached thread 71 | func (m *overchanMiddleware) ServeThread(w http.ResponseWriter, r *http.Request) { 72 | param := mux.Vars(r) 73 | obj, err := m.db.ThreadByHash(param["id"]) 74 | if err == nil { 75 | m.serveTemplate(w, r, "thread.html.tmpl", obj) 76 | } else { 77 | m.serveTemplate(w, r, "error.html.tmpl", err) 78 | } 79 | } 80 | 81 | // serve index page 82 | func (m *overchanMiddleware) ServeIndex(w http.ResponseWriter, r *http.Request) { 83 | m.serveTemplate(w, r, "index.html.tmpl", nil) 84 | } 85 | 86 | // serve a template 87 | func (m *overchanMiddleware) serveTemplate(w http.ResponseWriter, r *http.Request, tname string, obj interface{}) { 88 | t := m.templ.Lookup(tname) 89 | if t == nil { 90 | log.WithFields(log.Fields{ 91 | "template": tname, 92 | }).Warning("template not found") 93 | http.NotFound(w, r) 94 | } else { 95 | err := t.Execute(w, obj) 96 | if err != nil { 97 | // error getting model 98 | log.WithFields(log.Fields{ 99 | "error": err, 100 | "template": tname, 101 | }).Warning("failed to render template") 102 | } 103 | } 104 | } 105 | 106 | // create standard overchan middleware 107 | func OverchanMiddleware(c *config.MiddlewareConfig, db database.Database) (m Middleware, err error) { 108 | om := new(overchanMiddleware) 109 | om.templ, err = template.ParseGlob(filepath.Join(c.Templates, "*.tmpl")) 110 | om.db = db 111 | if err == nil { 112 | m = om 113 | } 114 | return 115 | } 116 | -------------------------------------------------------------------------------- /lib/frontend/post.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | -------------------------------------------------------------------------------- /lib/frontend/webhooks.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | -------------------------------------------------------------------------------- /lib/model/article.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Article struct { 4 | Subject string 5 | Name string 6 | Header map[string][]string 7 | Text string 8 | Attachments []Attachment 9 | MessageID string 10 | Newsgroup string 11 | Reference string 12 | Path string 13 | Posted int64 14 | Addr string 15 | } 16 | -------------------------------------------------------------------------------- /lib/model/attachment.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Attachment struct { 4 | Path string 5 | Name string 6 | Mime string 7 | Hash string 8 | // only filled for api 9 | Body string 10 | } 11 | -------------------------------------------------------------------------------- /lib/model/board.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Board struct { 4 | } 5 | -------------------------------------------------------------------------------- /lib/model/boardpage.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type BoardPage struct { 4 | Name string 5 | Page int 6 | Pages int 7 | Threads []Thread 8 | } 9 | -------------------------------------------------------------------------------- /lib/model/doc.go: -------------------------------------------------------------------------------- 1 | // MVC models 2 | package model 3 | -------------------------------------------------------------------------------- /lib/model/misc.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ArticleHeader map[string][]string 8 | 9 | // a ( MessageID , newsgroup ) tuple 10 | type ArticleEntry [2]string 11 | 12 | func (self ArticleEntry) Newsgroup() string { 13 | return self[1] 14 | } 15 | 16 | func (self ArticleEntry) MessageID() string { 17 | return self[0] 18 | } 19 | 20 | // a ( time point, post count ) tuple 21 | type PostEntry [2]int64 22 | 23 | func (self PostEntry) Time() time.Time { 24 | return time.Unix(self[0], 0) 25 | } 26 | 27 | func (self PostEntry) Count() int64 { 28 | return self[1] 29 | } 30 | -------------------------------------------------------------------------------- /lib/model/post.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Tripcode string 8 | 9 | type Post struct { 10 | MessageID string 11 | Newsgroup string 12 | Attachments []Attachment 13 | Subject string 14 | Posted time.Time 15 | PostedAt uint64 16 | Name string 17 | Tripcode Tripcode 18 | } 19 | 20 | // ( message-id, references, newsgroup ) 21 | type PostReference [3]string 22 | 23 | func (r PostReference) MessageID() string { 24 | return r[0] 25 | } 26 | 27 | func (r PostReference) References() string { 28 | return r[1] 29 | } 30 | func (r PostReference) Newsgroup() string { 31 | return r[2] 32 | } 33 | -------------------------------------------------------------------------------- /lib/model/thread.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Thread struct { 4 | Root *Post 5 | Replies []*Post 6 | } 7 | -------------------------------------------------------------------------------- /lib/network/dial.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "errors" 5 | "github.com/majestrate/srndv2/lib/config" 6 | "net" 7 | "strings" 8 | ) 9 | 10 | // operation timed out 11 | var ErrTimeout = errors.New("timeout") 12 | 13 | // the operation was reset abruptly 14 | var ErrReset = errors.New("reset") 15 | 16 | // the operation was actively refused 17 | var ErrRefused = errors.New("refused") 18 | 19 | // generic dialer 20 | // dials out to a remote address 21 | // returns a net.Conn and nil on success 22 | // returns nil and error if an error happens while dialing 23 | type Dialer interface { 24 | Dial(remote string) (net.Conn, error) 25 | } 26 | 27 | // create a new dialer from configuration 28 | func NewDialer(conf *config.ProxyConfig) (d Dialer) { 29 | d = StdDialer 30 | if conf != nil { 31 | proxyType := strings.ToLower(conf.Type) 32 | if proxyType == "socks" || proxyType == "socks4a" { 33 | d = SocksDialer(conf.Addr) 34 | } 35 | } 36 | return 37 | } 38 | -------------------------------------------------------------------------------- /lib/network/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // network utilities 3 | // 4 | package network 5 | -------------------------------------------------------------------------------- /lib/network/i2p.go: -------------------------------------------------------------------------------- 1 | package network 2 | -------------------------------------------------------------------------------- /lib/network/socks.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "errors" 5 | log "github.com/Sirupsen/logrus" 6 | "io" 7 | "net" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | type socksDialer struct { 13 | socksAddr string 14 | dialer Dialer 15 | } 16 | 17 | // try dialing out via socks proxy 18 | func (sd *socksDialer) Dial(remote string) (c net.Conn, err error) { 19 | log.WithFields(log.Fields{ 20 | "addr": remote, 21 | "socks": sd.socksAddr, 22 | }).Debug("dailing out to socks proxy") 23 | c, err = sd.dialer.Dial(sd.socksAddr) 24 | if err == nil { 25 | // dailed out to socks proxy good 26 | remote_addr := remote 27 | // generate request 28 | idx := strings.LastIndex(remote_addr, ":") 29 | if idx == -1 { 30 | err = errors.New("invalid address: " + remote_addr) 31 | return 32 | } 33 | var port uint64 34 | addr := remote_addr[:idx] 35 | port, err = strconv.ParseUint(remote_addr[idx+1:], 10, 16) 36 | if port >= 25536 { 37 | err = errors.New("bad proxy port") 38 | c.Close() 39 | c = nil 40 | return 41 | } else if err != nil { 42 | c.Close() 43 | return 44 | } 45 | var proxy_port uint16 46 | proxy_port = uint16(port) 47 | proxy_ident := "srndproxy" 48 | req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8 49 | 50 | req := make([]byte, req_len) 51 | // pack request 52 | req[0] = '\x04' 53 | req[1] = '\x01' 54 | req[2] = byte(proxy_port & 0xff00 >> 8) 55 | req[3] = byte(proxy_port & 0x00ff) 56 | req[7] = '\x01' 57 | idx = 8 58 | 59 | proxy_ident_b := []byte(proxy_ident) 60 | addr_b := []byte(addr) 61 | 62 | var bi int 63 | for bi = range proxy_ident_b { 64 | req[idx] = proxy_ident_b[bi] 65 | idx += 1 66 | } 67 | idx += 1 68 | for bi = range addr_b { 69 | req[idx] = addr_b[bi] 70 | idx += 1 71 | } 72 | log.WithFields(log.Fields{ 73 | "addr": remote, 74 | "socks": sd.socksAddr, 75 | "req": req, 76 | }).Debug("write socks request") 77 | n := 0 78 | n, err = c.Write(req) 79 | if err == nil && n == len(req) { 80 | // wrote request okay 81 | resp := make([]byte, 8) 82 | _, err = io.ReadFull(c, resp) 83 | if err == nil { 84 | // got reply okay 85 | if resp[1] == '\x5a' { 86 | // successful socks connection 87 | log.WithFields(log.Fields{ 88 | "addr": remote, 89 | "socks": sd.socksAddr, 90 | }).Debug("socks proxy connection successful") 91 | } else { 92 | // unsucessful socks connect 93 | log.WithFields(log.Fields{ 94 | "addr": remote, 95 | "socks": sd.socksAddr, 96 | "code": resp[1], 97 | }).Warn("connect via socks proxy failed") 98 | c.Close() 99 | c = nil 100 | } 101 | } else { 102 | // error reading reply 103 | log.WithFields(log.Fields{ 104 | "addr": remote, 105 | "socks": sd.socksAddr, 106 | }).Error("failed to read socks response ", err) 107 | c.Close() 108 | c = nil 109 | } 110 | } else { 111 | if err == nil { 112 | err = errors.New("short write") 113 | } 114 | 115 | // error writing request 116 | log.WithFields(log.Fields{ 117 | "addr": remote, 118 | "socks": sd.socksAddr, 119 | }).Error("failed to write socks request ", err) 120 | c.Close() 121 | c = nil 122 | 123 | } 124 | } else { 125 | // dail fail 126 | log.WithFields(log.Fields{ 127 | "addr": remote, 128 | "socks": sd.socksAddr, 129 | }).Error("Cannot connect to socks proxy ", err) 130 | } 131 | return 132 | } 133 | 134 | // create a socks dialer that dials out via socks proxy at address 135 | func SocksDialer(addr string) Dialer { 136 | return &socksDialer{ 137 | socksAddr: addr, 138 | dialer: StdDialer, 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /lib/network/std.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | type stdDialer struct { 8 | } 9 | 10 | func (sd *stdDialer) Dial(addr string) (c net.Conn, err error) { 11 | return net.Dial("tcp", addr) 12 | } 13 | 14 | var StdDialer = &stdDialer{} 15 | -------------------------------------------------------------------------------- /lib/nntp/acceptor.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/nntp/message" 5 | ) 6 | 7 | const ( 8 | // accepted article 9 | ARTICLE_ACCEPT = iota 10 | // reject article, don't send again 11 | ARTICLE_REJECT 12 | // defer article, send later 13 | ARTICLE_DEFER 14 | // reject + ban 15 | ARTICLE_BAN 16 | ) 17 | 18 | type PolicyStatus int 19 | 20 | const PolicyAccept = PolicyStatus(ARTICLE_ACCEPT) 21 | const PolicyReject = PolicyStatus(ARTICLE_REJECT) 22 | const PolicyDefer = PolicyStatus(ARTICLE_DEFER) 23 | const PolicyBan = PolicyStatus(ARTICLE_BAN) 24 | 25 | func (s PolicyStatus) String() string { 26 | switch int(s) { 27 | case ARTICLE_ACCEPT: 28 | return "ACCEPTED" 29 | case ARTICLE_REJECT: 30 | return "REJECTED" 31 | case ARTICLE_DEFER: 32 | return "DEFERRED" 33 | case ARTICLE_BAN: 34 | return "BANNED" 35 | default: 36 | return "[invalid policy status]" 37 | } 38 | } 39 | 40 | // is this an accept code? 41 | func (s PolicyStatus) Accept() bool { 42 | return s == ARTICLE_ACCEPT 43 | } 44 | 45 | // is this a defer code? 46 | func (s PolicyStatus) Defer() bool { 47 | return s == ARTICLE_DEFER 48 | } 49 | 50 | // is this a ban code 51 | func (s PolicyStatus) Ban() bool { 52 | return s == ARTICLE_BAN 53 | } 54 | 55 | // is this a reject code? 56 | func (s PolicyStatus) Reject() bool { 57 | return s == ARTICLE_BAN || s == ARTICLE_REJECT 58 | } 59 | 60 | // type defining a policy that determines if we want to accept/reject/defer an 61 | // incoming article 62 | type ArticleAcceptor interface { 63 | // check article given an article header 64 | CheckHeader(hdr message.Header) PolicyStatus 65 | // check article given a message id 66 | CheckMessageID(msgid MessageID) PolicyStatus 67 | // get max article size in bytes 68 | MaxArticleSize() int64 69 | } 70 | -------------------------------------------------------------------------------- /lib/nntp/auth.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | // defines server side authentication mechanism 11 | type ServerAuth interface { 12 | // check plaintext login 13 | // returns nil on success otherwise error if one occurs during authentication 14 | // returns true if authentication was successful and an error if a network io error happens 15 | CheckLogin(username, passwd string) (bool, error) 16 | } 17 | 18 | type FlatfileAuth string 19 | 20 | func (fname FlatfileAuth) CheckLogin(username, passwd string) (found bool, err error) { 21 | cred := fmt.Sprintf("%s:%s", username, passwd) 22 | var f *os.File 23 | f, err = os.Open(string(fname)) 24 | if err == nil { 25 | defer f.Close() 26 | r := bufio.NewReader(f) 27 | for err == nil { 28 | var line string 29 | line, err = r.ReadString(10) 30 | line = strings.Trim(line, "\r\n") 31 | if line == cred { 32 | found = true 33 | break 34 | } 35 | } 36 | } 37 | return 38 | } 39 | -------------------------------------------------------------------------------- /lib/nntp/client.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "errors" 5 | "github.com/majestrate/srndv2/lib/nntp/message" 6 | ) 7 | 8 | var ErrArticleNotFound = errors.New("article not found") 9 | var ErrPostRejected = errors.New("post rejected") 10 | 11 | // an nntp client 12 | // obtains articles from remote nntp server 13 | type Client interface { 14 | // obtain article by message id 15 | // returns an article and nil if obtained 16 | // returns nil and an error if an error occured while obtaining the article, 17 | // error is ErrArticleNotFound if the remote server doesn't have that article 18 | Article(msgid MessageID) (*message.Article, error) 19 | 20 | // check if the remote server has an article given its message-id 21 | // return true and nil if the server has the article 22 | // return false and nil if the server doesn't have the article 23 | // returns false and error if an error occured while checking 24 | Check(msgid MessageID) (bool, error) 25 | 26 | // check if the remote server carries a newsgroup 27 | // return true and nil if the server carries this newsgroup 28 | // return false and nil if the server doesn't carry this newsgroup 29 | // returns false and error if an error occured while checking 30 | NewsgroupExists(group Newsgroup) (bool, error) 31 | 32 | // return true and nil if posting is allowed 33 | // return false and nil if posting is not allowed 34 | // return false and error if an error occured 35 | PostingAllowed() (bool, error) 36 | 37 | // post an nntp article to remote server 38 | // returns nil on success 39 | // returns error if an error ocurred during post 40 | // returns ErrPostRejected if the remote server rejected our post 41 | Post(a *message.Article) error 42 | 43 | // connect to remote server 44 | // returns nil on success 45 | // returns error if one occurs during dial or handshake 46 | Connect(d Dialer) error 47 | 48 | // send quit and disconnects from server 49 | // blocks until done 50 | Quit() 51 | } 52 | -------------------------------------------------------------------------------- /lib/nntp/codes.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | // 1xx codes 4 | 5 | // help info follows 6 | const RPL_Help = "100" 7 | 8 | // capabilities info follows 9 | const RPL_Capabilities = "101" 10 | 11 | // server date time follows 12 | const RPL_Date = "111" 13 | 14 | // 2xx codes 15 | 16 | // posting is allowed 17 | const RPL_PostingAllowed = "200" 18 | 19 | // posting is not allowed 20 | const RPL_PostingNotAllowed = "201" 21 | 22 | // streaming mode enabled 23 | const RPL_PostingStreaming = "203" 24 | 25 | // reply to QUIT command, we will close the connection 26 | const RPL_Quit = "205" 27 | 28 | // reply for GROUP and LISTGROUP commands 29 | const RPL_Group = "211" 30 | 31 | // info list follows 32 | const RPL_List = "215" 33 | 34 | // index follows 35 | const RPL_Index = "218" 36 | 37 | // article follows 38 | const RPL_Article = "220" 39 | 40 | // article headers follows 41 | const RPL_ArticleHeaders = "221" 42 | 43 | // article body follows 44 | const RPL_ArticleBody = "222" 45 | 46 | // selected article exists 47 | const RPL_ArticleSelectedExists = "223" 48 | 49 | // overview info follows 50 | const RPL_Overview = "224" 51 | 52 | // list of article heards follows 53 | const RPL_HeadersList = "225" 54 | 55 | // list of new articles follows 56 | const RPL_NewArticles = "230" 57 | 58 | // list of newsgroups followes 59 | const RPL_NewsgroupList = "231" 60 | 61 | // article was transfered okay by IHAVE command 62 | const RPL_TransferOkay = "235" 63 | 64 | // article is not found by CHECK and we want it 65 | const RPL_StreamingAccept = "238" 66 | 67 | // article was transfered via TAKETHIS successfully 68 | const RPL_StreamingTransfered = "239" 69 | 70 | // article was transfered by POST command successfully 71 | const RPL_PostReceived = "240" 72 | 73 | // AUTHINFO SIMPLE accepted 74 | const RPL_AuthInfoAccepted = "250" 75 | 76 | // authentication creds have been accepted 77 | const RPL_AuthAccepted = "281" 78 | 79 | // binary content follows 80 | const RPL_Binary = "288" 81 | 82 | // line sent for posting allowed 83 | const Line_PostingAllowed = RPL_PostingAllowed + " Posting Allowed" 84 | 85 | // line sent for posting not allowed 86 | const Line_PostingNotAllowed = RPL_PostingNotAllowed + " Posting Not Allowed" 87 | 88 | // 3xx codes 89 | 90 | // article is accepted via IHAVE 91 | const RPL_TransferAccepted = "335" 92 | 93 | // article was accepted via POST 94 | const RPL_PostAccepted = "340" 95 | 96 | // continue with authorization 97 | const RPL_ContinueAuthorization = "350" 98 | 99 | // more authentication info required 100 | const RPL_MoreAuth = "381" 101 | 102 | // continue with tls handshake 103 | const RPL_TLSContinue = "382" 104 | 105 | // 4xx codes 106 | 107 | // server says servive is not avaiable on initial connection 108 | const RPL_NotAvaiable = "400" 109 | 110 | // server is in the wrong mode 111 | const RPL_WrongMode = "401" 112 | 113 | // generic fault prevent action from being taken 114 | const RPL_GenericError = "403" 115 | 116 | // newsgroup does not exist 117 | const RPL_NoSuchGroup = "411" 118 | 119 | // no newsgroup has been selected 120 | const RPL_NoGroupSelected = "412" 121 | 122 | // no tin style index available 123 | const RPL_NoIndex = "418" 124 | 125 | // current article number is invalid 126 | const RPL_NoArticleNum = "420" 127 | 128 | // no next article in this group (NEXT) 129 | const RPL_NoNextArticle = "421" 130 | 131 | // no previous article in this group (LAST) 132 | const RPL_NoPrevArticle = "422" 133 | 134 | // no article in specified range 135 | const RPL_NoArticleRange = "423" 136 | 137 | // no article with that message-id 138 | const RPL_NoArticleMsgID = "430" 139 | 140 | // defer article asked by CHECK comamnd 141 | const RPL_StreamingDefer = "431" 142 | 143 | // article is not wanted (1st stage of IHAVE) 144 | const RPL_TransferNotWanted = "435" 145 | 146 | // article was not sent defer sending (either stage of IHAVE) 147 | const RPL_TransferDefer = "436" 148 | 149 | // reject transfer do not retry (2nd stage IHAVE) 150 | const RPL_TransferReject = "437" 151 | 152 | // reject article and don't ask again (CHECK command) 153 | const RPL_StreamingReject = "438" 154 | 155 | // article transfer via streaming failed (TAKETHIS) 156 | const RPL_StreamingFailed = "439" 157 | 158 | // posting not permitted (1st stage of POST command) 159 | const RPL_PostingNotPermitted = "440" 160 | 161 | // posting failed (2nd stage of POST command) 162 | const RPL_PostingFailed = "441" 163 | 164 | // authorization required 165 | const RPL_AuthorizeRequired = "450" 166 | 167 | // authorization rejected 168 | const RPL_AuthorizeRejected = "452" 169 | 170 | // command unavaibale until client has authenticated 171 | const RPL_AuthenticateRequired = "480" 172 | 173 | // authentication creds rejected 174 | const RPL_AuthenticateRejected = "482" 175 | 176 | // command unavailable until connection is encrypted 177 | const RPL_EncryptionRequired = "483" 178 | 179 | // 5xx codes 180 | 181 | // got an unknown command 182 | const RPL_UnknownCommand = "500" 183 | 184 | // got a command with invalid syntax 185 | const RPL_SyntaxError = "501" 186 | 187 | // fatal error happened and connection will close 188 | const RPL_GenericFatal = "502" 189 | 190 | // feature is not supported 191 | const RPL_FeatureNotSupported = "503" 192 | 193 | // message encoding is bad 194 | const RPL_EncodingError = "504" 195 | 196 | // starttls can not be done 197 | const RPL_TLSRejected = "580" 198 | 199 | // line sent on invalid mode 200 | const Line_InvalidMode = RPL_SyntaxError + " Invalid Mode Selected" 201 | 202 | // line sent on successful streaming 203 | const Line_StreamingAllowed = RPL_PostingStreaming + " aw yeh streamit brah" 204 | 205 | // send this when we handle a QUIT command 206 | const Line_RPLQuit = RPL_Quit + " bai" 207 | -------------------------------------------------------------------------------- /lib/nntp/commands.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | type Command string 4 | 5 | func (c Command) String() string { 6 | return string(c) 7 | } 8 | 9 | // command to list newsgroups 10 | const CMD_Newsgroups = Command("NEWSGROUPS 0 0 GMT") 11 | 12 | // create group command for a newsgroup 13 | func CMD_Group(g Newsgroup) Command { 14 | return Command("GROUP " + g.String()) 15 | } 16 | 17 | const CMD_XOver = Command("XOVER 0") 18 | 19 | func CMD_Article(msgid MessageID) Command { 20 | return Command("ARTICLE " + msgid.String()) 21 | } 22 | 23 | func CMD_Head(msgid MessageID) Command { 24 | return Command("HEAD " + msgid.String()) 25 | } 26 | 27 | const CMD_Capabilities = Command("CAPABILITIES") 28 | -------------------------------------------------------------------------------- /lib/nntp/common.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/crypto" 5 | 6 | "crypto/sha1" 7 | "fmt" 8 | "io" 9 | "regexp" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | var exp_valid_message_id = regexp.MustCompilePOSIX(`^<[a-zA-Z0-9$.]{2,128}@[a-zA-Z0-9\-.]{2,63}>$`) 15 | 16 | type MessageID string 17 | 18 | // return true if this message id is well formed, otherwise return false 19 | func (msgid MessageID) Valid() bool { 20 | return exp_valid_message_id.Copy().MatchString(msgid.String()) 21 | } 22 | 23 | // get message id as string 24 | func (msgid MessageID) String() string { 25 | return string(msgid) 26 | } 27 | 28 | // compute long form hash of message id 29 | func (msgid MessageID) LongHash() string { 30 | return fmt.Sprintf("%x", sha1.Sum([]byte(msgid))) 31 | } 32 | 33 | // compute truncated form of message id hash 34 | func (msgid MessageID) ShortHash() string { 35 | return strings.ToLower(msgid.LongHash()[:18]) 36 | } 37 | 38 | // compute blake2 hash of message id 39 | func (msgid MessageID) Blake2Hash() string { 40 | h := crypto.Hash() 41 | io.WriteString(h, msgid.String()) 42 | return strings.ToLower(fmt.Sprintf("%x", h.Sum(nil))) 43 | } 44 | 45 | // generate a new message id given name of server 46 | func GenMessageID(name string) MessageID { 47 | r := crypto.RandBytes(4) 48 | t := time.Now() 49 | return MessageID(fmt.Sprintf("<%x$%d@%s>", r, t.Unix(), name)) 50 | } 51 | 52 | var exp_valid_newsgroup = regexp.MustCompilePOSIX(`^[a-zA-Z0-9.]{1,128}$`) 53 | 54 | // an nntp newsgroup 55 | type Newsgroup string 56 | 57 | // return true if this newsgroup is well formed otherwise false 58 | func (g Newsgroup) Valid() bool { 59 | return exp_valid_newsgroup.Copy().MatchString(g.String()) 60 | } 61 | 62 | // get newsgroup as string 63 | func (g Newsgroup) String() string { 64 | return string(g) 65 | } 66 | 67 | // (message-id, newsgroup) tuple 68 | type ArticleEntry [2]string 69 | 70 | func (e ArticleEntry) MessageID() MessageID { 71 | return MessageID(e[0]) 72 | } 73 | 74 | func (e ArticleEntry) Newsgroup() Newsgroup { 75 | return Newsgroup(e[1]) 76 | } 77 | -------------------------------------------------------------------------------- /lib/nntp/common_test.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestGenMessageID(t *testing.T) { 8 | msgid := GenMessageID("test.tld") 9 | t.Logf("generated id %s", msgid) 10 | if !msgid.Valid() { 11 | t.Logf("invalid generated message-id %s", msgid) 12 | t.Fail() 13 | } 14 | msgid = GenMessageID("<><><>") 15 | t.Logf("generated id %s", msgid) 16 | if msgid.Valid() { 17 | t.Logf("generated valid message-id when it should've been invalid %s", msgid) 18 | t.Fail() 19 | } 20 | } 21 | 22 | func TestMessageIDHash(t *testing.T) { 23 | msgid := GenMessageID("test.tld") 24 | lh := msgid.LongHash() 25 | sh := msgid.ShortHash() 26 | bh := msgid.Blake2Hash() 27 | t.Logf("long=%s short=%s blake2=%s", lh, sh, bh) 28 | } 29 | 30 | func TestValidNewsgroup(t *testing.T) { 31 | g := Newsgroup("overchan.test") 32 | if !g.Valid() { 33 | t.Logf("%s is invalid?", g) 34 | t.Fail() 35 | } 36 | } 37 | 38 | func TestInvalidNewsgroup(t *testing.T) { 39 | g := Newsgroup("asd.asd.asd.&&&") 40 | if g.Valid() { 41 | t.Logf("%s should be invalid", g) 42 | t.Fail() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lib/nntp/conn.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | // an nntp connection 4 | type Conn interface { 5 | 6 | // negotiate an nntp session on this connection 7 | // returns nil if we negitated successfully 8 | // returns ErrAuthRejected if the remote server rejected any authentication 9 | // we sent or another error if one occured while negotiating 10 | Negotiate(stream bool) error 11 | 12 | // obtain connection state 13 | GetState() *ConnState 14 | 15 | // retutrn true if posting is allowed 16 | // return false if posting is not allowed 17 | PostingAllowed() bool 18 | 19 | // handle inbound non-streaming connection 20 | // call event hooks on event 21 | ProcessInbound(hooks EventHooks) 22 | 23 | // does this connection want to do nntp streaming? 24 | WantsStreaming() bool 25 | 26 | // what mode are we in? 27 | // returns mode in all caps 28 | Mode() Mode 29 | 30 | // initiate nntp streaming 31 | // after calling this the caller MUST call StreamAndQuit() 32 | // returns a channel for message ids, true if caller sends on the channel or 33 | // returns nil and ErrStreamingNotAllowed if streaming is not allowed on this 34 | // connection or another error if one occurs while trying to start streaming 35 | StartStreaming() (chan ArticleEntry, error) 36 | 37 | // stream articles and quit when the channel obtained by StartStreaming() is 38 | // closed, after which this nntp connection is no longer open 39 | StreamAndQuit() 40 | 41 | // is this nntp connection open? 42 | IsOpen() bool 43 | 44 | // send quit command and close connection 45 | Quit() 46 | 47 | // download all articles in a newsgroup 48 | // returns error if a network error occurs 49 | DownloadGroup(g Newsgroup) error 50 | 51 | // get list of active newsgroups 52 | ListNewsgroups() ([]Newsgroup, error) 53 | } 54 | -------------------------------------------------------------------------------- /lib/nntp/conn_v1.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "crypto/tls" 7 | "encoding/base64" 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | log "github.com/Sirupsen/logrus" 12 | "github.com/majestrate/srndv2/lib/config" 13 | "github.com/majestrate/srndv2/lib/nntp/message" 14 | "github.com/majestrate/srndv2/lib/store" 15 | "github.com/majestrate/srndv2/lib/util" 16 | "io" 17 | "mime" 18 | "mime/multipart" 19 | "net" 20 | "net/textproto" 21 | "os" 22 | "strings" 23 | ) 24 | 25 | // handles 1 line of input from a connection 26 | type lineHandlerFunc func(c *v1Conn, line string, hooks EventHooks) error 27 | 28 | // base nntp connection 29 | type v1Conn struct { 30 | // buffered connection 31 | C *textproto.Conn 32 | 33 | // unexported fields ... 34 | 35 | // connection state (mutable) 36 | state ConnState 37 | // tls connection if tls is established 38 | tlsConn *tls.Conn 39 | // tls config for this connection, nil if we don't support tls 40 | tlsConfig *tls.Config 41 | // has this connection authenticated yet? 42 | authenticated bool 43 | // the username logged in with if it has authenticated via user/pass 44 | username string 45 | // underlying network socket 46 | conn net.Conn 47 | // server's name 48 | serverName string 49 | // article acceptor checks if we want articles 50 | acceptor ArticleAcceptor 51 | // headerIO for read/write of article header 52 | hdrio *message.HeaderIO 53 | // article storage 54 | storage store.Storage 55 | // event callbacks 56 | hooks EventHooks 57 | // inbound connection authenticator 58 | auth ServerAuth 59 | // command handlers 60 | cmds map[string]lineHandlerFunc 61 | } 62 | 63 | // json representation of this connection 64 | // format is: 65 | // { 66 | // "state" : (connection state object), 67 | // "authed" : bool, 68 | // "tls" : (tls info or null if plaintext connection) 69 | // } 70 | func (c *v1Conn) MarshalJSON() ([]byte, error) { 71 | j := make(map[string]interface{}) 72 | j["state"] = c.state 73 | j["authed"] = c.authenticated 74 | if c.tlsConn == nil { 75 | j["tls"] = nil 76 | } else { 77 | j["tls"] = c.tlsConn.ConnectionState() 78 | } 79 | return json.Marshal(j) 80 | } 81 | 82 | // get the current state of our connection (immutable) 83 | func (c *v1Conn) GetState() (state *ConnState) { 84 | return &ConnState{ 85 | FeedName: c.state.FeedName, 86 | ConnName: c.state.ConnName, 87 | HostName: c.state.HostName, 88 | Mode: c.state.Mode, 89 | Group: c.state.Group, 90 | Article: c.state.Article, 91 | Policy: &FeedPolicy{ 92 | Whitelist: c.state.Policy.Whitelist, 93 | Blacklist: c.state.Policy.Blacklist, 94 | AllowAnonPosts: c.state.Policy.AllowAnonPosts, 95 | AllowAnonAttachments: c.state.Policy.AllowAnonAttachments, 96 | AllowAttachments: c.state.Policy.AllowAttachments, 97 | UntrustedRequiresPoW: c.state.Policy.UntrustedRequiresPoW, 98 | }, 99 | } 100 | } 101 | 102 | func (c *v1Conn) Group() string { 103 | return c.state.Group.String() 104 | } 105 | 106 | func (c *v1Conn) IsOpen() bool { 107 | return c.state.Open 108 | } 109 | 110 | func (c *v1Conn) Mode() Mode { 111 | return c.state.Mode 112 | } 113 | 114 | // is posting allowed rignt now? 115 | func (c *v1Conn) PostingAllowed() bool { 116 | return c.Authed() 117 | } 118 | 119 | // process incoming commands 120 | // call event hooks as needed 121 | func (c *v1Conn) Process(hooks EventHooks) { 122 | var err error 123 | var line string 124 | for err == nil { 125 | line, err = c.readline() 126 | if len(line) == 0 { 127 | // eof (proably?) 128 | c.Close() 129 | return 130 | } 131 | 132 | uline := strings.ToUpper(line) 133 | parts := strings.Split(uline, " ") 134 | handler, ok := c.cmds[parts[0]] 135 | if ok { 136 | // we know the command 137 | err = handler(c, line, hooks) 138 | } else { 139 | // we don't know the command 140 | err = c.printfLine("%s Unknown Command: %s", RPL_UnknownCommand, line) 141 | } 142 | } 143 | } 144 | 145 | type v1OBConn struct { 146 | C v1Conn 147 | supports_stream bool 148 | streamChnl chan ArticleEntry 149 | conf *config.FeedConfig 150 | } 151 | 152 | func (c *v1OBConn) IsOpen() bool { 153 | return c.IsOpen() 154 | } 155 | 156 | func (c *v1OBConn) Mode() Mode { 157 | return c.Mode() 158 | } 159 | 160 | func (c *v1OBConn) DownloadGroup(g Newsgroup) (err error) { 161 | err = c.C.printfLine(CMD_Group(g).String()) 162 | if err == nil { 163 | var line string 164 | line, err = c.C.readline() 165 | if strings.HasPrefix(line, RPL_NoSuchGroup) { 166 | // group does not exist 167 | // don't error this is not a network io error 168 | return 169 | } 170 | // send XOVER 171 | err = c.C.printfLine(CMD_XOver.String()) 172 | if err == nil { 173 | line, err = c.C.readline() 174 | if err == nil { 175 | if !strings.HasPrefix(line, RPL_Overview) { 176 | // bad response 177 | // not a network io error, don't error 178 | return 179 | } 180 | var msgids []MessageID 181 | // read reply 182 | for err == nil && line != "." { 183 | line, err = c.C.readline() 184 | parts := strings.Split(line, "\t") 185 | if len(parts) != 6 { 186 | // incorrect size 187 | continue 188 | } 189 | m := MessageID(parts[4]) 190 | r := MessageID(parts[5]) 191 | if c.C.acceptor == nil { 192 | // no acceptor take it if store doesn't have it 193 | if c.C.storage.HasArticle(m.String()) == store.ErrNoSuchArticle { 194 | msgids = append(msgids, m) 195 | } 196 | } else { 197 | // check if thread is banned 198 | if c.C.acceptor.CheckMessageID(r).Ban() { 199 | continue 200 | } 201 | // check if message is wanted 202 | if c.C.acceptor.CheckMessageID(m).Accept() { 203 | msgids = append(msgids, m) 204 | } 205 | } 206 | } 207 | var accepted []MessageID 208 | 209 | for _, msgid := range msgids { 210 | 211 | if err != nil { 212 | return // io error 213 | } 214 | 215 | if !msgid.Valid() { 216 | // invalid message id 217 | continue 218 | } 219 | // get message header 220 | err = c.C.printfLine(CMD_Head(msgid).String()) 221 | if err == nil { 222 | line, err = c.C.readline() 223 | if err == nil { 224 | if !strings.HasPrefix(line, RPL_ArticleHeaders) { 225 | // bad response 226 | continue 227 | } 228 | // read message header 229 | dr := c.C.C.DotReader() 230 | var hdr message.Header 231 | hdr, err = c.C.hdrio.ReadHeader(dr) 232 | if err == nil { 233 | if c.C.acceptor == nil { 234 | accepted = append(accepted, msgid) 235 | } else if c.C.acceptor.CheckHeader(hdr).Accept() { 236 | accepted = append(accepted, msgid) 237 | } 238 | } 239 | } 240 | } 241 | } 242 | // download wanted messages 243 | for _, msgid := range accepted { 244 | if err != nil { 245 | // io error 246 | return 247 | } 248 | // request message 249 | err = c.C.printfLine(CMD_Article(msgid).String()) 250 | if err == nil { 251 | line, err = c.C.readline() 252 | if err == nil { 253 | if !strings.HasPrefix(line, RPL_Article) { 254 | // bad response 255 | continue 256 | } 257 | // read article 258 | _, err = c.C.readArticle(false, c.C.hooks) 259 | if err == nil { 260 | // we read it okay 261 | } 262 | } 263 | } 264 | } 265 | } 266 | } 267 | } 268 | return 269 | } 270 | 271 | func (c *v1OBConn) ListNewsgroups() (groups []Newsgroup, err error) { 272 | err = c.C.printfLine(CMD_Newsgroups.String()) 273 | if err == nil { 274 | var line string 275 | line, err = c.C.readline() 276 | if err == nil { 277 | if !strings.HasPrefix(line, RPL_NewsgroupList) { 278 | // bad stuff 279 | err = errors.New("invalid reply for NEWSGROUPS command: " + line) 280 | return 281 | } 282 | for err == nil && line != "." { 283 | line, err = c.C.readline() 284 | if err == nil { 285 | parts := strings.Split(line, " ") 286 | if len(parts) != 4 { 287 | // bad format 288 | continue 289 | } 290 | groups = append(groups, Newsgroup(parts[0])) 291 | } 292 | } 293 | } 294 | } 295 | return 296 | } 297 | 298 | // negioate outbound connection 299 | func (c *v1OBConn) Negotiate(stream bool) (err error) { 300 | var line string 301 | // discard first line 302 | _, err = c.C.readline() 303 | if err == nil { 304 | // request capabilities 305 | err = c.C.printfLine(CMD_Capabilities.String()) 306 | dr := c.C.C.DotReader() 307 | var b bytes.Buffer 308 | _, err = io.Copy(&b, dr) 309 | if err == nil { 310 | // try login if specified 311 | if c.conf.Username != "" && c.conf.Password != "" { 312 | err = c.C.printfLine("AUTHINFO USER %s", c.conf.Username) 313 | if err != nil { 314 | return 315 | } 316 | line, err = c.C.readline() 317 | if strings.HasPrefix(line, RPL_MoreAuth) { 318 | err = c.C.printfLine("AUTHINFO PASS %s", c.conf.Password) 319 | if err != nil { 320 | return 321 | } 322 | line, err = c.C.readline() 323 | if err != nil { 324 | return 325 | } 326 | if strings.HasPrefix(line, RPL_AuthAccepted) { 327 | log.WithFields(log.Fields{ 328 | "name": c.conf.Name, 329 | "user": c.conf.Username, 330 | }).Info("authentication accepted") 331 | } else { 332 | // not accepted? 333 | err = errors.New(line) 334 | } 335 | } else { 336 | // bad user? 337 | err = errors.New(line) 338 | } 339 | } 340 | if err == nil { 341 | if stream { 342 | // set mode stream 343 | err = c.C.printfLine(ModeStream.String()) 344 | if err == nil { 345 | line, err = c.C.readline() 346 | if err == nil && !strings.HasPrefix(line, RPL_PostingStreaming) { 347 | err = errors.New("streaiming not allowed") 348 | } 349 | } 350 | } 351 | } 352 | } 353 | } 354 | return 355 | } 356 | 357 | func (c *v1OBConn) PostingAllowed() bool { 358 | return c.C.PostingAllowed() 359 | } 360 | 361 | func (c *v1OBConn) ProcessInbound(hooks EventHooks) { 362 | 363 | } 364 | 365 | func (c *v1OBConn) WantsStreaming() bool { 366 | return c.supports_stream 367 | } 368 | 369 | func (c *v1OBConn) StreamAndQuit() { 370 | for { 371 | e, ok := <-c.streamChnl 372 | if ok { 373 | // do CHECK 374 | msgid := e.MessageID() 375 | if !msgid.Valid() { 376 | log.WithFields(log.Fields{ 377 | "pkg": "nntp-conn", 378 | "state": c.C.state, 379 | "msgid": msgid, 380 | }).Warn("Dropping stream event with invalid message-id") 381 | continue 382 | } 383 | // send line 384 | err := c.C.printfLine("%s %s", stream_CHECK, msgid) 385 | if err == nil { 386 | // read response 387 | var line string 388 | line, err = c.C.readline() 389 | ev := StreamEvent(line) 390 | if ev.Valid() { 391 | cmd := ev.Command() 392 | if cmd == RPL_StreamingAccept { 393 | // accepted to send 394 | 395 | // check if we really have it in storage 396 | err = c.C.storage.HasArticle(msgid.String()) 397 | if err == nil { 398 | var r io.ReadCloser 399 | r, err = c.C.storage.OpenArticle(msgid.String()) 400 | if err == nil { 401 | log.WithFields(log.Fields{ 402 | "pkg": "nntp-conn", 403 | "state": c.C.state, 404 | "msgid": msgid, 405 | }).Debug("article accepted will send via TAKETHIS now") 406 | _ = c.C.printfLine("%s %s", stream_TAKETHIS, msgid) 407 | br := bufio.NewReader(r) 408 | n := int64(0) 409 | for err == nil { 410 | var line string 411 | line, err = br.ReadString(10) 412 | if err == io.EOF { 413 | err = nil 414 | break 415 | } 416 | line = strings.Trim(line, "\r") 417 | line = strings.Trim(line, "\n") 418 | err = c.C.printfLine(line) 419 | n += int64(len(line)) 420 | } 421 | r.Close() 422 | err = c.C.printfLine(".") 423 | if err == nil { 424 | // successful takethis sent 425 | log.WithFields(log.Fields{ 426 | "pkg": "nntp-conn", 427 | "state": c.C.state, 428 | "msgid": msgid, 429 | "bytes": n, 430 | }).Debug("article transfer done") 431 | // read response 432 | line, err = c.C.readline() 433 | ev := StreamEvent(line) 434 | if ev.Valid() { 435 | // valid reply 436 | cmd := ev.Command() 437 | if cmd == RPL_StreamingTransfered { 438 | // successful transfer 439 | log.WithFields(log.Fields{ 440 | "feed": c.C.state.FeedName, 441 | "msgid": msgid, 442 | "bytes": n, 443 | }).Debug("Article Transferred") 444 | // call hooks 445 | if c.C.hooks != nil { 446 | c.C.hooks.SentArticleVia(msgid, c.C.state.FeedName) 447 | } 448 | } else { 449 | // failed transfer 450 | log.WithFields(log.Fields{ 451 | "feed": c.C.state.FeedName, 452 | "msgid": msgid, 453 | "bytes": n, 454 | }).Debug("Article Rejected") 455 | } 456 | } 457 | } else { 458 | log.WithFields(log.Fields{ 459 | "feed": c.C.state.FeedName, 460 | "msgid": msgid, 461 | }).Errorf("failed to transfer: %s", err.Error()) 462 | } 463 | } 464 | } else { 465 | log.WithFields(log.Fields{ 466 | "pkg": "nntp-conn", 467 | "state": c.C.state, 468 | "msgid": msgid, 469 | }).Warn("article not in storage, not sending") 470 | } 471 | } 472 | } else { 473 | // invalid reply 474 | log.WithFields(log.Fields{ 475 | "pkg": "nntp-conn", 476 | "state": c.C.state, 477 | "msgid": msgid, 478 | "line": line, 479 | }).Error("invalid streaming response") 480 | // close 481 | return 482 | } 483 | } else { 484 | log.WithFields(log.Fields{ 485 | "pkg": "nntp-conn", 486 | "state": c.C.state, 487 | "msgid": msgid, 488 | }).Error("streaming error during CHECK", err) 489 | return 490 | } 491 | } else { 492 | // channel closed 493 | return 494 | } 495 | } 496 | } 497 | 498 | func (c *v1OBConn) Quit() { 499 | c.C.printfLine("QUIT yo") 500 | c.C.readline() 501 | c.C.Close() 502 | } 503 | 504 | func (c *v1OBConn) StartStreaming() (chnl chan ArticleEntry, err error) { 505 | if c.streamChnl == nil { 506 | c.streamChnl = make(chan ArticleEntry) 507 | 508 | } 509 | chnl = c.streamChnl 510 | return 511 | } 512 | 513 | func (c *v1OBConn) GetState() *ConnState { 514 | return c.GetState() 515 | } 516 | 517 | // create a new connection from an established connection 518 | func newOutboundConn(c net.Conn, s *Server, conf *config.FeedConfig) Conn { 519 | 520 | sname := s.Name() 521 | 522 | if len(sname) == 0 { 523 | sname = "nntp.anon.tld" 524 | } 525 | storage := s.Storage 526 | if storage == nil { 527 | storage = store.NewNullStorage() 528 | } 529 | return &v1OBConn{ 530 | conf: conf, 531 | C: v1Conn{ 532 | hooks: s, 533 | state: ConnState{ 534 | FeedName: conf.Name, 535 | HostName: conf.Addr, 536 | Open: true, 537 | }, 538 | serverName: sname, 539 | storage: storage, 540 | C: textproto.NewConn(c), 541 | conn: c, 542 | hdrio: message.NewHeaderIO(), 543 | }, 544 | } 545 | } 546 | 547 | type v1IBConn struct { 548 | C v1Conn 549 | } 550 | 551 | func (c *v1IBConn) DownloadGroup(g Newsgroup) error { 552 | return nil 553 | } 554 | 555 | func (c *v1IBConn) ListNewsgroups() (groups []Newsgroup, err error) { 556 | return 557 | } 558 | 559 | func (c *v1IBConn) GetState() *ConnState { 560 | return c.C.GetState() 561 | } 562 | 563 | // negotiate an inbound connection 564 | func (c *v1IBConn) Negotiate(stream bool) (err error) { 565 | var line string 566 | if c.PostingAllowed() { 567 | line = Line_PostingAllowed 568 | } else { 569 | line = Line_PostingNotAllowed 570 | } 571 | err = c.C.printfLine(line) 572 | return 573 | } 574 | 575 | func (c *v1IBConn) PostingAllowed() bool { 576 | return c.C.PostingAllowed() 577 | } 578 | 579 | func (c *v1IBConn) IsOpen() bool { 580 | return c.C.IsOpen() 581 | } 582 | 583 | func (c *v1IBConn) Quit() { 584 | // inbound connections quit without warning 585 | log.WithFields(log.Fields{ 586 | "pkg": "nntp-ibconn", 587 | "addr": c.C.conn.RemoteAddr(), 588 | }).Info("closing inbound connection") 589 | c.C.Close() 590 | } 591 | 592 | // is this connection authenticated? 593 | func (c *v1Conn) Authed() bool { 594 | return c.tlsConn != nil || c.authenticated 595 | } 596 | 597 | // unconditionally close connection 598 | func (c *v1Conn) Close() { 599 | if c.tlsConn == nil { 600 | // tls is not on 601 | c.C.Close() 602 | } else { 603 | // tls is on 604 | // we should close tls cleanly 605 | c.tlsConn.Close() 606 | } 607 | c.state.Open = false 608 | } 609 | 610 | func (c *v1IBConn) WantsStreaming() bool { 611 | return c.C.state.Mode.Is(MODE_STREAM) 612 | } 613 | 614 | func (c *v1Conn) printfLine(format string, args ...interface{}) error { 615 | log.WithFields(log.Fields{ 616 | "pkg": "nntp-conn", 617 | "version": 1, 618 | "state": &c.state, 619 | "io": "send", 620 | }).Debugf(format, args...) 621 | return c.C.PrintfLine(format, args...) 622 | } 623 | 624 | func (c *v1Conn) readline() (line string, err error) { 625 | line, err = c.C.ReadLine() 626 | log.WithFields(log.Fields{ 627 | "pkg": "nntp-conn", 628 | "version": 1, 629 | "state": &c.state, 630 | "io": "recv", 631 | }).Debug(line) 632 | return 633 | } 634 | 635 | // handle switching nntp modes for inbound connection 636 | func switchModeInbound(c *v1Conn, line string, hooks EventHooks) (err error) { 637 | cmd := ModeCommand(line) 638 | m := c.Mode() 639 | if cmd.Is(ModeReader) { 640 | if m.Is(MODE_STREAM) { 641 | // we need to stop streaming 642 | } 643 | var line string 644 | if c.PostingAllowed() { 645 | line = Line_PostingAllowed 646 | } else { 647 | line = Line_PostingNotAllowed 648 | } 649 | err = c.printfLine(line) 650 | if err == nil { 651 | c.state.Mode = MODE_READER 652 | } 653 | } else if cmd.Is(ModeStream) { 654 | // we want to switch to streaming mode 655 | err = c.printfLine(Line_StreamingAllowed) 656 | if err == nil { 657 | c.state.Mode = MODE_STREAM 658 | } 659 | } else { 660 | err = c.printfLine(Line_InvalidMode) 661 | } 662 | return 663 | } 664 | 665 | // handle quit command 666 | func quitConnection(c *v1Conn, line string, hooks EventHooks) (err error) { 667 | log.WithFields(log.Fields{ 668 | "pkg": "nntp-conn", 669 | "version": "1", 670 | "state": &c.state, 671 | }).Debug("quit requested") 672 | err = c.printfLine(Line_RPLQuit) 673 | c.Close() 674 | return 675 | } 676 | 677 | // send our capabailities 678 | func sendCapabilities(c *v1Conn, line string, hooks EventHooks) (err error) { 679 | var caps []string 680 | 681 | caps = append(caps, "MODE-READER", "IMPLEMENTATION nntpchand", "STREAMING") 682 | if c.tlsConfig != nil { 683 | caps = append(caps, "STARTTLS") 684 | } 685 | 686 | err = c.printfLine("%s We can do things", RPL_Capabilities) 687 | if err == nil { 688 | for _, l := range caps { 689 | err = c.printfLine(l) 690 | if err != nil { 691 | log.WithFields(log.Fields{ 692 | "pkg": "nntp-conn", 693 | "version": "1", 694 | "state": &c.state, 695 | }).Error(err) 696 | } 697 | } 698 | err = c.printfLine(".") 699 | } 700 | return 701 | } 702 | 703 | // read an article via dotreader 704 | func (c *v1Conn) readArticle(newpost bool, hooks EventHooks) (ps PolicyStatus, err error) { 705 | store_r, store_w := io.Pipe() 706 | article_r, article_w := io.Pipe() 707 | article_body_r, article_body_w := io.Pipe() 708 | 709 | accept_chnl := make(chan PolicyStatus) 710 | store_info_chnl := make(chan ArticleEntry) 711 | store_result_chnl := make(chan error) 712 | 713 | hdr_chnl := make(chan message.Header) 714 | 715 | log.WithFields(log.Fields{ 716 | "pkg": "nntp-conn", 717 | }).Debug("start reading") 718 | done_chnl := make(chan PolicyStatus) 719 | go func() { 720 | var err error 721 | dr := c.C.DotReader() 722 | var buff [1024]byte 723 | var n int64 724 | n, err = io.CopyBuffer(article_w, dr, buff[:]) 725 | log.WithFields(log.Fields{ 726 | "n": n, 727 | }).Debug("read from connection") 728 | if err != nil && err != io.EOF { 729 | article_w.CloseWithError(err) 730 | } else { 731 | article_w.Close() 732 | } 733 | st := <-accept_chnl 734 | close(accept_chnl) 735 | // get result from storage 736 | err2, ok := <-store_result_chnl 737 | if ok && err2 != io.EOF { 738 | err = err2 739 | } 740 | close(store_result_chnl) 741 | done_chnl <- st 742 | }() 743 | 744 | // parse message and store attachments in bg 745 | go func(msgbody io.ReadCloser) { 746 | defer msgbody.Close() 747 | hdr, ok := <-hdr_chnl 748 | if !ok { 749 | return 750 | } 751 | // all text in this post 752 | // txt := new(bytes.Buffer) 753 | // the article itself 754 | // a := new(model.Article) 755 | var err error 756 | if hdr.IsMultipart() { 757 | var params map[string]string 758 | _, params, err = hdr.GetMediaType() 759 | if err == nil { 760 | boundary, ok := params["boundary"] 761 | if ok { 762 | part_r := multipart.NewReader(msgbody, boundary) 763 | for err == nil { 764 | var part *multipart.Part 765 | part, err = part_r.NextPart() 766 | if err == io.EOF { 767 | // we done 768 | break 769 | } else if err == nil { 770 | // we gots a part 771 | 772 | // get header 773 | part_hdr := part.Header 774 | 775 | // check for base64 encoding 776 | var part_body io.Reader 777 | if part_hdr.Get("Content-Transfer-Encoding") == "base64" { 778 | part_body = base64.NewDecoder(base64.StdEncoding, part) 779 | } else { 780 | part_body = part 781 | } 782 | 783 | // get content type 784 | content_type := part_hdr.Get("Content-Type") 785 | if len(content_type) == 0 { 786 | // assume text/plain 787 | content_type = "text/plain; charset=UTF8" 788 | } 789 | var part_type string 790 | // extract mime type 791 | part_type, _, err = mime.ParseMediaType(content_type) 792 | if err == nil { 793 | 794 | if part_type == "text/plain" { 795 | // if we are plaintext save it to the text buffer 796 | _, err = io.Copy(util.Discard, part_body) 797 | } else { 798 | var fpath string 799 | fname := part.FileName() 800 | fpath, err = c.storage.StoreAttachment(part_body, fname) 801 | if err == nil { 802 | // stored attachment good 803 | log.WithFields(log.Fields{ 804 | "pkg": "nntp-conn", 805 | "state": &c.state, 806 | "version": "1", 807 | "filename": fname, 808 | "filepath": fpath, 809 | }).Debug("attachment stored") 810 | } else { 811 | // failed to save attachment 812 | log.WithFields(log.Fields{ 813 | "pkg": "nntp-conn", 814 | "state": &c.state, 815 | "version": "1", 816 | }).Error("failed to save attachment ", err) 817 | } 818 | } 819 | } else { 820 | // cannot read part header 821 | log.WithFields(log.Fields{ 822 | "pkg": "nntp-conn", 823 | "state": &c.state, 824 | "version": "1", 825 | }).Error("bad attachment in multipart message ", err) 826 | } 827 | err = nil 828 | part.Close() 829 | } else if err != io.EOF { 830 | // error reading part 831 | log.WithFields(log.Fields{ 832 | "pkg": "nntp-conn", 833 | "state": &c.state, 834 | "version": "1", 835 | }).Error("error reading part ", err) 836 | } 837 | } 838 | } 839 | } 840 | } else if hdr.IsSigned() { 841 | // signed message 842 | 843 | // discard for now 844 | _, err = io.Copy(util.Discard, msgbody) 845 | } else { 846 | // plaintext message 847 | var n int64 848 | n, err = io.Copy(util.Discard, msgbody) 849 | log.WithFields(log.Fields{ 850 | "bytes": n, 851 | "pkg": "nntp-conn", 852 | }).Debug("text body copied") 853 | } 854 | if err != nil && err != io.EOF { 855 | log.WithFields(log.Fields{ 856 | "pkg": "nntp-conn", 857 | "state": &c.state, 858 | }).Error("error handing message body", err) 859 | } 860 | }(article_body_r) 861 | 862 | // store function 863 | go func(r io.ReadCloser) { 864 | e, ok := <-store_info_chnl 865 | if !ok { 866 | // failed to get info 867 | // don't read anything 868 | r.Close() 869 | store_result_chnl <- io.EOF 870 | return 871 | } 872 | msgid := e.MessageID() 873 | if msgid.Valid() { 874 | // valid message-id 875 | log.WithFields(log.Fields{ 876 | "pkg": "nntp-conn", 877 | "msgid": msgid, 878 | "version": "1", 879 | "state": &c.state, 880 | }).Debug("storing article") 881 | 882 | fpath, err := c.storage.StoreArticle(r, msgid.String(), e.Newsgroup().String()) 883 | r.Close() 884 | if err == nil { 885 | log.WithFields(log.Fields{ 886 | "pkg": "nntp-conn", 887 | "msgid": msgid, 888 | "version": "1", 889 | "state": &c.state, 890 | }).Debug("stored article okay to ", fpath) 891 | // we got the article 892 | if hooks != nil { 893 | hooks.GotArticle(msgid, e.Newsgroup()) 894 | } 895 | store_result_chnl <- io.EOF 896 | log.Debugf("store informed") 897 | } else { 898 | // error storing article 899 | log.WithFields(log.Fields{ 900 | "pkg": "nntp-conn", 901 | "msgid": msgid, 902 | "state": &c.state, 903 | "version": "1", 904 | }).Error("failed to store article ", err) 905 | io.Copy(util.Discard, r) 906 | store_result_chnl <- err 907 | } 908 | } else { 909 | // invalid message-id 910 | // discard 911 | log.WithFields(log.Fields{ 912 | "pkg": "nntp-conn", 913 | "msgid": msgid, 914 | "state": &c.state, 915 | "version": "1", 916 | }).Warn("store will discard message with invalid message-id") 917 | io.Copy(util.Discard, r) 918 | store_result_chnl <- nil 919 | r.Close() 920 | } 921 | }(store_r) 922 | 923 | // acceptor function 924 | go func(r io.ReadCloser, out_w, body_w io.WriteCloser) { 925 | var w io.WriteCloser 926 | defer r.Close() 927 | status := PolicyAccept 928 | hdr, err := c.hdrio.ReadHeader(r) 929 | if err == nil { 930 | // append path 931 | hdr.AppendPath(c.serverName) 932 | // get message-id 933 | var msgid MessageID 934 | if newpost { 935 | // new post 936 | // generate it 937 | msgid = GenMessageID(c.serverName) 938 | hdr.Set("Message-ID", msgid.String()) 939 | } else { 940 | // not a new post, get from header 941 | msgid = MessageID(hdr.MessageID()) 942 | if msgid.Valid() { 943 | // check store fo existing article 944 | err = c.storage.HasArticle(msgid.String()) 945 | if err == store.ErrNoSuchArticle { 946 | // we don't have the article 947 | status = PolicyAccept 948 | log.Infof("accept article %s", msgid) 949 | } else if err == nil { 950 | // we do have the article, reject it we don't need it again 951 | status = PolicyReject 952 | } else { 953 | // some other error happened 954 | log.WithFields(log.Fields{ 955 | "pkg": "nntp-conn", 956 | "state": c.state, 957 | }).Error("failed to check store for article ", err) 958 | } 959 | err = nil 960 | } else { 961 | // bad article 962 | status = PolicyBan 963 | } 964 | } 965 | // check the header if we have an acceptor and the previous checks are good 966 | if status.Accept() && c.acceptor != nil { 967 | status = c.acceptor.CheckHeader(hdr) 968 | } 969 | if status.Accept() { 970 | // we have accepted the article 971 | // store to disk 972 | w = out_w 973 | } else { 974 | // we have not accepted the article 975 | // discard 976 | w = util.Discard 977 | out_w.Close() 978 | } 979 | store_info_chnl <- ArticleEntry{msgid.String(), hdr.Newsgroup()} 980 | hdr_chnl <- hdr 981 | // close the channel for headers 982 | close(hdr_chnl) 983 | // write header out to storage 984 | err = c.hdrio.WriteHeader(hdr, w) 985 | if err == nil { 986 | mw := io.MultiWriter(body_w, w) 987 | // we wrote header 988 | var n int64 989 | if c.acceptor == nil { 990 | // write the rest of the body 991 | // we don't care about article size 992 | log.WithFields(log.Fields{}).Debug("copying body") 993 | var buff [128]byte 994 | n, err = io.CopyBuffer(mw, r, buff[:]) 995 | } else { 996 | // we care about the article size 997 | max := c.acceptor.MaxArticleSize() 998 | var n int64 999 | // copy it out 1000 | n, err = io.CopyN(mw, r, max) 1001 | if err == nil { 1002 | if n < max { 1003 | // under size limit 1004 | // we gud 1005 | log.WithFields(log.Fields{ 1006 | "pkg": "nntp-conn", 1007 | "bytes": n, 1008 | "state": &c.state, 1009 | }).Debug("body fits") 1010 | } else { 1011 | // too big, discard the rest 1012 | _, err = io.Copy(util.Discard, r) 1013 | // ... and ban it 1014 | status = PolicyBan 1015 | } 1016 | } 1017 | } 1018 | log.WithFields(log.Fields{ 1019 | "pkg": "nntp-conn", 1020 | "bytes": n, 1021 | "state": &c.state, 1022 | }).Debug("body wrote") 1023 | // TODO: inform store to delete article and attachments 1024 | } else { 1025 | // error writing header 1026 | log.WithFields(log.Fields{ 1027 | "msgid": msgid, 1028 | }).Error("error writing header ", err) 1029 | } 1030 | } else { 1031 | // error reading header 1032 | // possibly a read error? 1033 | status = PolicyDefer 1034 | } 1035 | // close info channel for store 1036 | close(store_info_chnl) 1037 | w.Close() 1038 | // close body pipe 1039 | body_w.Close() 1040 | // inform result 1041 | log.Debugf("status %s", status) 1042 | accept_chnl <- status 1043 | log.Debugf("informed") 1044 | }(article_r, store_w, article_body_w) 1045 | 1046 | ps = <-done_chnl 1047 | close(done_chnl) 1048 | log.Debug("read article done") 1049 | return 1050 | } 1051 | 1052 | // handle IHAVE command 1053 | func nntpRecvArticle(c *v1Conn, line string, hooks EventHooks) (err error) { 1054 | parts := strings.Split(line, " ") 1055 | if len(parts) == 2 { 1056 | msgid := MessageID(parts[1]) 1057 | if msgid.Valid() { 1058 | // valid message-id 1059 | err = c.printfLine("%s send article to be transfered", RPL_TransferAccepted) 1060 | // read in article 1061 | if err == nil { 1062 | var status PolicyStatus 1063 | status, err = c.readArticle(false, hooks) 1064 | if err == nil { 1065 | // we read in article 1066 | if status.Accept() { 1067 | // accepted 1068 | err = c.printfLine("%s transfer wuz gud", RPL_TransferOkay) 1069 | } else if status.Defer() { 1070 | // deferred 1071 | err = c.printfLine("%s transfer defer", RPL_TransferDefer) 1072 | } else if status.Reject() { 1073 | // rejected 1074 | err = c.printfLine("%s transfer rejected, don't send it again brah", RPL_TransferReject) 1075 | } 1076 | } else { 1077 | // could not transfer article 1078 | err = c.printfLine("%s transfer failed; try again later", RPL_TransferDefer) 1079 | } 1080 | } 1081 | } else { 1082 | // invalid message-id 1083 | err = c.printfLine("%s article not wanted", RPL_TransferNotWanted) 1084 | } 1085 | } else { 1086 | // invaldi syntax 1087 | err = c.printfLine("%s invalid syntax", RPL_SyntaxError) 1088 | } 1089 | return 1090 | } 1091 | 1092 | // handle POST command 1093 | func nntpPostArticle(c *v1Conn, line string, hooks EventHooks) (err error) { 1094 | if c.PostingAllowed() { 1095 | if c.Mode().Is(MODE_READER) { 1096 | err = c.printfLine("%s go ahead yo", RPL_PostAccepted) 1097 | var status PolicyStatus 1098 | status, err = c.readArticle(true, hooks) 1099 | if err == nil { 1100 | // read okay 1101 | if status.Accept() { 1102 | err = c.printfLine("%s post was recieved", RPL_PostReceived) 1103 | } else { 1104 | err = c.printfLine("%s posting failed", RPL_PostingFailed) 1105 | } 1106 | } else { 1107 | log.WithFields(log.Fields{ 1108 | "pkg": "nntp-conn", 1109 | "state": &c.state, 1110 | "version": "1", 1111 | }).Error("POST failed ", err) 1112 | err = c.printfLine("%s post failed: %s", RPL_PostingFailed, err) 1113 | } 1114 | } else { 1115 | // not in reader mode 1116 | err = c.printfLine("%s not in reader mode", RPL_WrongMode) 1117 | } 1118 | } else { 1119 | err = c.printfLine("%s posting is disallowed", RPL_PostingNotPermitted) 1120 | } 1121 | return 1122 | } 1123 | 1124 | // handle streaming line 1125 | func streamingLine(c *v1Conn, line string, hooks EventHooks) (err error) { 1126 | ev := StreamEvent(line) 1127 | if c.Mode().Is(MODE_STREAM) { 1128 | if ev.Valid() { 1129 | // valid stream line 1130 | cmd := ev.Command() 1131 | msgid := ev.MessageID() 1132 | if cmd == stream_CHECK { 1133 | if c.acceptor == nil { 1134 | // no acceptor, we'll take them all 1135 | err = c.printfLine("%s %s", RPL_StreamingAccept, msgid) 1136 | } else { 1137 | status := PolicyAccept 1138 | if c.storage.HasArticle(msgid.String()) == nil { 1139 | // we have this article 1140 | status = PolicyReject 1141 | } 1142 | if status.Accept() && c.acceptor != nil { 1143 | status = c.acceptor.CheckMessageID(ev.MessageID()) 1144 | } 1145 | if status.Accept() { 1146 | // accepted 1147 | err = c.printfLine("%s %s", RPL_StreamingAccept, msgid) 1148 | } else if status.Defer() { 1149 | // deferred 1150 | err = c.printfLine("%s %s", RPL_StreamingDefer, msgid) 1151 | } else { 1152 | // rejected 1153 | err = c.printfLine("%s %s", RPL_StreamingReject, msgid) 1154 | } 1155 | } 1156 | } else if cmd == stream_TAKETHIS { 1157 | var status PolicyStatus 1158 | status, err = c.readArticle(false, hooks) 1159 | if status.Accept() { 1160 | // this article was accepted 1161 | err = c.printfLine("%s %s", RPL_StreamingTransfered, msgid) 1162 | } else { 1163 | // this article was not accepted 1164 | err = c.printfLine("%s %s", RPL_StreamingReject, msgid) 1165 | } 1166 | } 1167 | } else { 1168 | // invalid line 1169 | err = c.printfLine("%s Invalid syntax", RPL_SyntaxError) 1170 | } 1171 | } else { 1172 | if ev.MessageID().Valid() { 1173 | // not in streaming mode 1174 | err = c.printfLine("%s %s", RPL_StreamingDefer, ev.MessageID()) 1175 | } else { 1176 | // invalid message id 1177 | err = c.printfLine("%s Invalid Syntax", RPL_SyntaxError) 1178 | } 1179 | } 1180 | return 1181 | } 1182 | 1183 | func newsgroupList(c *v1Conn, line string, hooks EventHooks, rpl string) (err error) { 1184 | var groups []string 1185 | if c.storage == nil { 1186 | // no database driver available 1187 | // let's say we carry overchan.test for now 1188 | groups = append(groups, "overchan.test") 1189 | } else { 1190 | groups, err = c.storage.GetAllNewsgroups() 1191 | } 1192 | 1193 | if err == nil { 1194 | // we got newsgroups from the db 1195 | dw := c.C.DotWriter() 1196 | fmt.Fprintf(dw, "%s list of newsgroups follows\n", rpl) 1197 | for _, g := range groups { 1198 | hi := uint64(1) 1199 | lo := uint64(0) 1200 | if c.storage != nil { 1201 | hi, lo, err = c.storage.GetWatermark(g) 1202 | } 1203 | if err != nil { 1204 | // log error if it occurs 1205 | log.WithFields(log.Fields{ 1206 | "pkg": "nntp-conn", 1207 | "group": g, 1208 | "state": c.state, 1209 | }).Warn("cannot get high low water marks for LIST command") 1210 | 1211 | } else { 1212 | fmt.Fprintf(dw, "%s %d %d y", g, hi, lo) 1213 | } 1214 | } 1215 | // flush dotwriter 1216 | err = dw.Close() 1217 | } else { 1218 | // db error while getting newsgroup list 1219 | err = c.printfLine("%s cannot list newsgroups %s", RPL_GenericError, err.Error()) 1220 | } 1221 | return 1222 | } 1223 | 1224 | // handle inbound STARTTLS command 1225 | func upgradeTLS(c *v1Conn, line string, hooks EventHooks) (err error) { 1226 | if c.tlsConfig == nil { 1227 | err = c.printfLine("%s TLS not supported", RPL_TLSRejected) 1228 | } else { 1229 | err = c.printfLine("%s Continue with TLS Negotiation", RPL_TLSContinue) 1230 | if err == nil { 1231 | tconn := tls.Server(c.conn, c.tlsConfig) 1232 | err = tconn.Handshake() 1233 | if err == nil { 1234 | // successful tls handshake 1235 | c.tlsConn = tconn 1236 | c.C = textproto.NewConn(c.tlsConn) 1237 | } else { 1238 | // tls failed 1239 | log.WithFields(log.Fields{ 1240 | "pkg": "nntp-conn", 1241 | "addr": c.conn.RemoteAddr(), 1242 | "state": c.state, 1243 | }).Warn("TLS Handshake failed ", err) 1244 | // fall back to plaintext 1245 | err = nil 1246 | } 1247 | } 1248 | } 1249 | return 1250 | } 1251 | 1252 | // switch to another newsgroup 1253 | func switchNewsgroup(c *v1Conn, line string, hooks EventHooks) (err error) { 1254 | parts := strings.Split(line, " ") 1255 | var has bool 1256 | var group Newsgroup 1257 | if len(parts) == 2 { 1258 | group = Newsgroup(parts[1]) 1259 | if group.Valid() { 1260 | // correct format 1261 | if c.storage == nil { 1262 | has = false 1263 | } else { 1264 | has, err = c.storage.HasNewsgroup(group.String()) 1265 | } 1266 | } 1267 | } 1268 | if has { 1269 | // we have it 1270 | hi := uint64(1) 1271 | lo := uint64(0) 1272 | if c.storage != nil { 1273 | // check database for water marks 1274 | hi, lo, err = c.storage.GetWatermark(group.String()) 1275 | } 1276 | if err == nil { 1277 | // XXX: ensure hi > lo 1278 | err = c.printfLine("%s %d %d %d %s", RPL_Group, hi-lo, lo, hi, group.String()) 1279 | if err == nil { 1280 | // line was sent 1281 | c.state.Group = group 1282 | log.WithFields(log.Fields{ 1283 | "pkg": "nntp-conn", 1284 | "group": group, 1285 | "state": c.state, 1286 | }).Debug("switched newsgroups") 1287 | } 1288 | } else { 1289 | err = c.printfLine("%s error checking for newsgroup %s", RPL_GenericError, err.Error()) 1290 | } 1291 | } else if err != nil { 1292 | // error 1293 | err = c.printfLine("%s error checking for newsgroup %s", RPL_GenericError, err.Error()) 1294 | } else { 1295 | // incorrect format 1296 | err = c.printfLine("%s no such newsgroup", RPL_NoSuchGroup) 1297 | } 1298 | return 1299 | } 1300 | 1301 | func handleAuthInfo(c *v1Conn, line string, hooks EventHooks) (err error) { 1302 | subcmd := line[9:] 1303 | if strings.HasPrefix(strings.ToUpper(subcmd), "USER") { 1304 | c.username = subcmd[5:] 1305 | err = c.printfLine("%s password required", RPL_MoreAuth) 1306 | } else if strings.HasPrefix(strings.ToUpper(subcmd), "PASS") { 1307 | var success bool 1308 | if c.username == "" { 1309 | // out of order commands 1310 | c.printfLine("%s auth info sent out of order yo", RPL_GenericError) 1311 | return 1312 | } else if c.auth == nil { 1313 | // no auth mechanism, this will be set to true if anon nntp is enabled 1314 | success = c.authenticated 1315 | } else { 1316 | // check login 1317 | success, err = c.auth.CheckLogin(c.username, subcmd[5:]) 1318 | } 1319 | if success { 1320 | // login good 1321 | err = c.printfLine("%s login gud, proceed yo", RPL_AuthAccepted) 1322 | c.authenticated = true 1323 | } else if err == nil { 1324 | // login bad 1325 | err = c.printfLine("%s bad login", RPL_AuthenticateRejected) 1326 | } else { 1327 | // error 1328 | err = c.printfLine("%s error processing login: %s", RPL_GenericError, err.Error()) 1329 | } 1330 | } else { 1331 | err = c.printfLine("%s only USER/PASS accepted with AUTHINFO", RPL_SyntaxError) 1332 | } 1333 | return 1334 | } 1335 | 1336 | func handleXOVER(c *v1Conn, line string, hooks EventHooks) (err error) { 1337 | group := c.Group() 1338 | if group == "" { 1339 | err = c.printfLine("%s no group selected", RPL_NoGroupSelected) 1340 | return 1341 | } 1342 | if !Newsgroup(group).Valid() { 1343 | err = c.printfLine("%s Invalid Newsgroup format", RPL_GenericError) 1344 | return 1345 | } 1346 | err = c.printfLine("%s overview follows", RPL_Overview) 1347 | if err != nil { 1348 | return 1349 | } 1350 | chnl := make(chan string) 1351 | go func() { 1352 | c.storage.ForEachInGroup(group, chnl) 1353 | close(chnl) 1354 | }() 1355 | i := 0 1356 | for err == nil { 1357 | m, ok := <-chnl 1358 | if !ok { 1359 | break 1360 | } 1361 | msgid := MessageID(m) 1362 | if !msgid.Valid() { 1363 | continue 1364 | } 1365 | var f *os.File 1366 | f, err = c.storage.OpenArticle(m) 1367 | if f != nil { 1368 | h, e := c.hdrio.ReadHeader(f) 1369 | f.Close() 1370 | if e == nil { 1371 | i++ 1372 | err = c.printfLine("%.6d\t%s\t%s\t%s\t%s\t%s", i, h.Get("Subject", "None"), h.Get("From", "anon "), h.Get("Date", "???"), h.MessageID(), h.Reference()) 1373 | } 1374 | } 1375 | } 1376 | if err == nil { 1377 | err = c.printfLine(".") 1378 | } 1379 | return 1380 | } 1381 | 1382 | func handleArticle(c *v1Conn, line string, hooks EventHooks) (err error) { 1383 | msgid := MessageID(line[8:]) 1384 | if msgid.Valid() && c.storage.HasArticle(msgid.String()) == nil { 1385 | // valid id and we have it 1386 | var r io.ReadCloser 1387 | var buff [1024]byte 1388 | r, err = c.storage.OpenArticle(msgid.String()) 1389 | if err == nil { 1390 | err = c.printfLine("%s %s", RPL_Article, msgid) 1391 | for err == nil { 1392 | _, err = io.CopyBuffer(c.C.W, r, buff[:]) 1393 | } 1394 | if err == io.EOF { 1395 | err = nil 1396 | } 1397 | if err == nil { 1398 | err = c.printfLine(".") 1399 | } 1400 | r.Close() 1401 | return 1402 | } 1403 | } 1404 | // invalid id or we don't have it 1405 | err = c.printfLine("%s %s", RPL_NoArticleMsgID, msgid) 1406 | return 1407 | } 1408 | 1409 | // inbound streaming start 1410 | func (c *v1IBConn) StartStreaming() (chnl chan ArticleEntry, err error) { 1411 | if c.Mode().Is(MODE_STREAM) { 1412 | chnl = make(chan ArticleEntry) 1413 | } else { 1414 | err = ErrInvalidMode 1415 | } 1416 | return 1417 | } 1418 | 1419 | func (c *v1IBConn) Mode() Mode { 1420 | return c.C.Mode() 1421 | } 1422 | 1423 | func (c *v1IBConn) ProcessInbound(hooks EventHooks) { 1424 | c.C.Process(hooks) 1425 | } 1426 | 1427 | // inbound streaming handling 1428 | func (c *v1IBConn) StreamAndQuit() { 1429 | } 1430 | 1431 | func newInboundConn(s *Server, c net.Conn) Conn { 1432 | sname := s.Name() 1433 | storage := s.Storage 1434 | if storage == nil { 1435 | storage = store.NewNullStorage() 1436 | } 1437 | anon := false 1438 | if s.Config != nil { 1439 | anon = s.Config.AnonNNTP 1440 | } 1441 | return &v1IBConn{ 1442 | C: v1Conn{ 1443 | state: ConnState{ 1444 | FeedName: "inbound-feed", 1445 | HostName: c.RemoteAddr().String(), 1446 | Open: true, 1447 | }, 1448 | auth: s.Auth, 1449 | authenticated: anon, 1450 | serverName: sname, 1451 | storage: storage, 1452 | acceptor: s.Acceptor, 1453 | hdrio: message.NewHeaderIO(), 1454 | C: textproto.NewConn(c), 1455 | conn: c, 1456 | cmds: map[string]lineHandlerFunc{ 1457 | "STARTTLS": upgradeTLS, 1458 | "IHAVE": nntpRecvArticle, 1459 | "POST": nntpPostArticle, 1460 | "MODE": switchModeInbound, 1461 | "QUIT": quitConnection, 1462 | "CAPABILITIES": sendCapabilities, 1463 | "CHECK": streamingLine, 1464 | "TAKETHIS": streamingLine, 1465 | "LIST": func(c *v1Conn, line string, h EventHooks) error { 1466 | return newsgroupList(c, line, h, RPL_List) 1467 | }, 1468 | "NEWSGROUPS": func(c *v1Conn, line string, h EventHooks) error { 1469 | return newsgroupList(c, line, h, RPL_NewsgroupList) 1470 | }, 1471 | "GROUP": switchNewsgroup, 1472 | "AUTHINFO": handleAuthInfo, 1473 | "XOVER": handleXOVER, 1474 | "ARTICLE": handleArticle, 1475 | }, 1476 | }, 1477 | } 1478 | } 1479 | -------------------------------------------------------------------------------- /lib/nntp/dial.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "crypto/tls" 5 | "github.com/majestrate/srndv2/lib/network" 6 | ) 7 | 8 | // establishes an outbound nntp connection to a remote server 9 | type Dialer interface { 10 | // dial out with a dialer 11 | // if cfg is not nil, try to establish a tls connection with STARTTLS 12 | // returns a new nntp connection and nil on successful handshake and login 13 | // returns nil and an error if an error happened 14 | Dial(d network.Dialer, cfg *tls.Config) (*Conn, error) 15 | } 16 | -------------------------------------------------------------------------------- /lib/nntp/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // nntp client/server 3 | // 4 | package nntp 5 | -------------------------------------------------------------------------------- /lib/nntp/event_test.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestTAKETHISParse(t *testing.T) { 8 | msgid := GenMessageID("test.tld") 9 | ev := stream_cmd_TAKETHIS(msgid) 10 | t.Logf("event: %s", ev) 11 | if ev.MessageID() != msgid { 12 | t.Logf("%s != %s, event was %s", msgid, ev.MessageID(), ev) 13 | t.Fail() 14 | } 15 | if ev.Command() != "TAKETHIS" { 16 | t.Logf("%s != TAKETHIS, event was %s", ev.Command(), ev) 17 | t.Fail() 18 | } 19 | if !ev.Valid() { 20 | t.Logf("%s is invalid stream event", ev) 21 | t.Fail() 22 | } 23 | } 24 | 25 | func TestCHECKParse(t *testing.T) { 26 | msgid := GenMessageID("test.tld") 27 | ev := stream_cmd_CHECK(msgid) 28 | t.Logf("event: %s", ev) 29 | if ev.MessageID() != msgid { 30 | t.Logf("%s != %s, event was %s", msgid, ev.MessageID(), ev) 31 | t.Fail() 32 | } 33 | if ev.Command() != "CHECK" { 34 | t.Logf("%s != CHECK, event was %s", ev.Command(), ev) 35 | t.Fail() 36 | } 37 | if !ev.Valid() { 38 | t.Logf("%s is invalid stream event", ev) 39 | t.Fail() 40 | } 41 | } 42 | 43 | func TestInvalidStremEvent(t *testing.T) { 44 | str := "asd" 45 | ev := StreamEvent(str) 46 | t.Logf("invalid str=%s ev=%s", str, ev) 47 | if ev.Valid() { 48 | t.Logf("invalid CHECK command is valid? %s", ev) 49 | t.Fail() 50 | } 51 | 52 | str = "asd asd" 53 | ev = StreamEvent(str) 54 | t.Logf("invalid str=%s ev=%s", str, ev) 55 | 56 | if ev.Valid() { 57 | t.Logf("invalid CHECK command is valid? %s", ev) 58 | t.Fail() 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /lib/nntp/filter.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/nntp/message" 5 | "io" 6 | ) 7 | 8 | // defines interface for filtering an nntp article 9 | // filters can (and does) modify the article it operates on 10 | type ArticleFilter interface { 11 | // filter the article header 12 | // returns the modified Header and an error if one occurs 13 | FilterHeader(hdr message.Header) (message.Header, error) 14 | 15 | // reads the article's body and write the filtered version to an io.Writer 16 | // returns the number of bytes written to the io.Writer, true if the body was 17 | // modifed (or false if body is unchanged) and an error if one occurs 18 | FilterAndWriteBody(body io.Reader, wr io.Writer) (int64, bool, error) 19 | } 20 | -------------------------------------------------------------------------------- /lib/nntp/hook.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | log "github.com/Sirupsen/logrus" 5 | "github.com/majestrate/srndv2/lib/config" 6 | "os/exec" 7 | ) 8 | 9 | type Hook struct { 10 | cfg *config.NNTPHookConfig 11 | } 12 | 13 | func NewHook(cfg *config.NNTPHookConfig) *Hook { 14 | return &Hook{ 15 | cfg: cfg, 16 | } 17 | } 18 | 19 | func (h *Hook) GotArticle(msgid MessageID, group Newsgroup) { 20 | c := exec.Command(h.cfg.Exec, group.String(), msgid.String()) 21 | log.Infof("calling hook %s", h.cfg.Name) 22 | err := c.Run() 23 | if err != nil { 24 | log.Errorf("error in nntp hook %s: %s", h.cfg.Name, err.Error()) 25 | } 26 | } 27 | 28 | func (*Hook) SentArticleVia(msgid MessageID, feedname string) { 29 | 30 | } 31 | -------------------------------------------------------------------------------- /lib/nntp/hooks.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | // callback hooks fired on certain events 4 | type EventHooks interface { 5 | // called when we have obtained an article given its message-id 6 | GotArticle(msgid MessageID, group Newsgroup) 7 | // called when we have sent an article to a single remote feed 8 | SentArticleVia(msgid MessageID, feedname string) 9 | } 10 | -------------------------------------------------------------------------------- /lib/nntp/message/article.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | // an nntp article 4 | type Article struct { 5 | 6 | // the article's mime header 7 | Header Header 8 | 9 | // unexported fields ... 10 | 11 | } 12 | -------------------------------------------------------------------------------- /lib/nntp/message/attachment.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // attachment in an nntp article 8 | type Attachment struct { 9 | // mimetype 10 | Mime string 11 | // the filename 12 | FileName string 13 | // the fully decoded attachment body 14 | // must close when done 15 | Body io.ReadCloser 16 | } 17 | -------------------------------------------------------------------------------- /lib/nntp/message/doc.go: -------------------------------------------------------------------------------- 1 | // package for parsing, packing, signing, verifying nntp articles 2 | package message 3 | -------------------------------------------------------------------------------- /lib/nntp/message/header.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "io" 5 | "mime" 6 | "strings" 7 | ) 8 | 9 | // an nntp message header 10 | type Header map[string][]string 11 | 12 | // get message-id header 13 | func (self Header) MessageID() (v string) { 14 | for _, hdr := range []string{"MessageID", "Message-ID", "Message-Id", "message-id"} { 15 | v = self.Get(hdr, "") 16 | if v != "" { 17 | break 18 | } 19 | } 20 | return 21 | } 22 | 23 | func (self Header) Reference() (ref string) { 24 | return self.Get("Reference", self.MessageID()) 25 | } 26 | 27 | // extract media type from content-type header 28 | func (self Header) GetMediaType() (mediatype string, params map[string]string, err error) { 29 | return mime.ParseMediaType(self.Get("Content-Type", "text/plain")) 30 | } 31 | 32 | // is this header for a multipart message? 33 | func (self Header) IsMultipart() bool { 34 | return strings.HasPrefix(self.Get("Content-Type", "text/plain"), "multipart/mixed") 35 | } 36 | 37 | func (self Header) IsSigned() bool { 38 | return self.Get("X-Pubkey-Ed25519", "") != "" 39 | } 40 | 41 | func (self Header) Newsgroup() string { 42 | return self.Get("Newsgroups", "overchan.discard") 43 | } 44 | 45 | // do we have a key in this header? 46 | func (self Header) Has(key string) bool { 47 | _, ok := self[key] 48 | return ok 49 | } 50 | 51 | // set key value 52 | func (self Header) Set(key, val string) { 53 | self[key] = []string{val} 54 | } 55 | 56 | func (self Header) AppendPath(name string) { 57 | p := self.Get("Path", name) 58 | if p != name { 59 | p = name + "!" + p 60 | } 61 | self.Set("Path", p) 62 | } 63 | 64 | // append value to key 65 | func (self Header) Add(key, val string) { 66 | if self.Has(key) { 67 | self[key] = append(self[key], val) 68 | } else { 69 | self.Set(key, val) 70 | } 71 | } 72 | 73 | // get via key or return fallback value 74 | func (self Header) Get(key, fallback string) string { 75 | val, ok := self[key] 76 | if ok { 77 | str := "" 78 | for _, k := range val { 79 | str += k + ", " 80 | } 81 | return str[:len(str)-2] 82 | } else { 83 | return fallback 84 | } 85 | } 86 | 87 | // interface for types that can read an nntp header 88 | type HeaderReader interface { 89 | // blocking read an nntp header from an io.Reader 90 | // return the read header and nil on success 91 | // return nil and an error if an error occurred while reading 92 | ReadHeader(r io.Reader) (Header, error) 93 | } 94 | 95 | // interface for types that can write an nntp header 96 | type HeaderWriter interface { 97 | // blocking write an nntp header to an io.Writer 98 | // returns an error if one occurs otherwise nil 99 | WriteHeader(hdr Header, w io.Writer) error 100 | } 101 | 102 | // implements HeaderReader and HeaderWriter 103 | type HeaderIO struct { 104 | delim byte 105 | } 106 | 107 | // read header 108 | func (s *HeaderIO) ReadHeader(r io.Reader) (hdr Header, err error) { 109 | hdr = make(Header) 110 | var k, v string 111 | var buf [1]byte 112 | for err == nil { 113 | // read key 114 | for err == nil { 115 | _, err = r.Read(buf[:]) 116 | if err != nil { 117 | return 118 | } 119 | if buf[0] == 58 { // colin 120 | // consume space 121 | _, err = r.Read(buf[:]) 122 | for err == nil { 123 | _, err = r.Read(buf[:]) 124 | if buf[0] == s.delim { 125 | // got delimiter 126 | hdr.Add(k, v) 127 | k = "" 128 | v = "" 129 | break 130 | } else { 131 | v += string(buf[:]) 132 | } 133 | } 134 | break 135 | } else if buf[0] == s.delim { 136 | // done 137 | return 138 | } else { 139 | k += string(buf[:]) 140 | } 141 | } 142 | } 143 | return 144 | } 145 | 146 | // write header 147 | func (s *HeaderIO) WriteHeader(hdr Header, wr io.Writer) (err error) { 148 | for k, vs := range hdr { 149 | for _, v := range vs { 150 | var line []byte 151 | // key 152 | line = append(line, []byte(k)...) 153 | // ": " 154 | line = append(line, 58, 32) 155 | // value 156 | line = append(line, []byte(v)...) 157 | // delimiter 158 | line = append(line, s.delim) 159 | // write line 160 | _, err = wr.Write(line) 161 | if err != nil { 162 | return 163 | } 164 | } 165 | } 166 | _, err = wr.Write([]byte{s.delim}) 167 | return 168 | } 169 | 170 | func NewHeaderIO() *HeaderIO { 171 | return &HeaderIO{ 172 | delim: 10, 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /lib/nntp/mode.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | ) 7 | 8 | var ErrInvalidMode = errors.New("invalid mode set") 9 | 10 | // a mode set by an nntp client 11 | type Mode string 12 | 13 | // reader mode 14 | const MODE_READER = Mode("reader") 15 | 16 | // streaming mode 17 | const MODE_STREAM = Mode("stream") 18 | 19 | // mode is not set 20 | const MODE_UNSET = Mode("") 21 | 22 | // get as string 23 | func (m Mode) String() string { 24 | return strings.ToUpper(string(m)) 25 | } 26 | 27 | // is this a valid mode of operation? 28 | func (m Mode) Valid() bool { 29 | return m.Is(MODE_READER) || m.Is(MODE_STREAM) 30 | } 31 | 32 | // is this mode equal to another mode 33 | func (m Mode) Is(other Mode) bool { 34 | return m.String() == other.String() 35 | } 36 | 37 | // a switch mode command 38 | type ModeCommand string 39 | 40 | // get as string 41 | func (m ModeCommand) String() string { 42 | return strings.ToUpper(string(m)) 43 | } 44 | 45 | // is this mode command well formed? 46 | // does not check the actual mode sent. 47 | func (m ModeCommand) Valid() bool { 48 | s := m.String() 49 | return strings.Count(s, " ") == 1 && strings.HasPrefix(s, "MODE ") 50 | } 51 | 52 | // get the mode selected in this mode command 53 | func (m ModeCommand) Mode() Mode { 54 | return Mode(strings.Split(m.String(), " ")[1]) 55 | } 56 | 57 | // check if this mode command is equal to an existing one 58 | func (m ModeCommand) Is(cmd ModeCommand) bool { 59 | return m.String() == cmd.String() 60 | } 61 | 62 | // reader mode command 63 | const ModeReader = ModeCommand("mode reader") 64 | 65 | // streaming mode command 66 | const ModeStream = ModeCommand("mode stream") 67 | 68 | // line prefix for mode 69 | const LinePrefix_Mode = "MODE " 70 | -------------------------------------------------------------------------------- /lib/nntp/multi.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | // multiplexed event hook 4 | type MulitHook []EventHooks 5 | 6 | func (m MulitHook) GotArticle(msgid MessageID, group Newsgroup) { 7 | for _, h := range m { 8 | h.GotArticle(msgid, group) 9 | } 10 | } 11 | 12 | func (m MulitHook) SentArticleVia(msgid MessageID, feedname string) { 13 | for _, h := range m { 14 | h.SentArticleVia(msgid, feedname) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /lib/nntp/policy.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | // 4 | // a policy that governs whether we federate an article via a feed 5 | // 6 | type FeedPolicy struct { 7 | // list of whitelist regexps for newsgorups 8 | Whitelist []string `json:"whitelist"` 9 | // list of blacklist regexps for newsgroups 10 | Blacklist []string `json:"blacklist"` 11 | // are anon posts of any kind allowed? 12 | AllowAnonPosts bool `json:"anon"` 13 | // are anon posts with attachments allowed? 14 | AllowAnonAttachments bool `json:"anon_attachments"` 15 | // are any attachments allowed? 16 | AllowAttachments bool `json:"attachments"` 17 | // do we require Proof Of Work for untrusted connections? 18 | UntrustedRequiresPoW bool `json:"pow"` 19 | } 20 | 21 | // default feed policy to be used if not configured explicitly 22 | var DefaultFeedPolicy = &FeedPolicy{ 23 | Whitelist: []string{"ctl", "overchan.test"}, 24 | Blacklist: []string{`!^overchan\.`}, 25 | AllowAnonPosts: true, 26 | AllowAnonAttachments: false, 27 | UntrustedRequiresPoW: true, 28 | AllowAttachments: true, 29 | } 30 | -------------------------------------------------------------------------------- /lib/nntp/server.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | log "github.com/Sirupsen/logrus" 5 | "github.com/majestrate/srndv2/lib/config" 6 | "github.com/majestrate/srndv2/lib/network" 7 | "github.com/majestrate/srndv2/lib/store" 8 | "net" 9 | "time" 10 | ) 11 | 12 | // nntp outfeed state 13 | type nntpFeed struct { 14 | conn Conn 15 | send chan ArticleEntry 16 | conf *config.FeedConfig 17 | } 18 | 19 | // an nntp server 20 | type Server struct { 21 | // user callback 22 | Hooks EventHooks 23 | // filters to apply 24 | Filters []ArticleFilter 25 | // global article acceptor 26 | Acceptor ArticleAcceptor 27 | // article storage 28 | Storage store.Storage 29 | // nntp config 30 | Config *config.NNTPServerConfig 31 | // outfeeds to connect to 32 | Feeds []*config.FeedConfig 33 | // inbound authentiaction mechanism 34 | Auth ServerAuth 35 | // send to outbound feed channel 36 | send chan ArticleEntry 37 | // register inbound feed channel 38 | regis chan *nntpFeed 39 | // deregister inbound feed channel 40 | deregis chan *nntpFeed 41 | } 42 | 43 | func NewServer() *Server { 44 | return &Server{ 45 | // XXX: buffered? 46 | send: make(chan ArticleEntry), 47 | regis: make(chan *nntpFeed), 48 | deregis: make(chan *nntpFeed), 49 | } 50 | } 51 | 52 | // reload server configuration 53 | func (s *Server) ReloadServer(c *config.NNTPServerConfig) { 54 | 55 | } 56 | 57 | // reload feeds 58 | func (s *Server) ReloadFeeds(feeds []*config.FeedConfig) { 59 | 60 | } 61 | 62 | func (s *Server) GotArticle(msgid MessageID, group Newsgroup) { 63 | log.WithFields(log.Fields{ 64 | "pkg": "nntp-server", 65 | "msgid": msgid, 66 | "group": group, 67 | }).Info("obtained article") 68 | if s.Hooks != nil { 69 | s.Hooks.GotArticle(msgid, group) 70 | } 71 | // send to outbound feeds 72 | s.send <- ArticleEntry{msgid.String(), group.String()} 73 | } 74 | 75 | func (s *Server) SentArticleVia(msgid MessageID, feedname string) { 76 | log.WithFields(log.Fields{ 77 | "pkg": "nntp-server", 78 | "msgid": msgid, 79 | "feed": feedname, 80 | }).Info("article sent") 81 | if s.Hooks != nil { 82 | s.Hooks.SentArticleVia(msgid, feedname) 83 | } 84 | } 85 | 86 | func (s *Server) Name() string { 87 | if s.Config == nil || s.Config.Name == "" { 88 | return "nntp.anon.tld" 89 | } 90 | return s.Config.Name 91 | } 92 | 93 | // persist 1 feed forever 94 | func (s *Server) persist(cfg *config.FeedConfig) { 95 | delay := time.Second 96 | 97 | log.WithFields(log.Fields{ 98 | "name": cfg.Name, 99 | }).Debug("Persist Feed") 100 | for { 101 | dialer := network.NewDialer(cfg.Proxy) 102 | c, err := dialer.Dial(cfg.Addr) 103 | if err == nil { 104 | // successful connect 105 | delay = time.Second 106 | conn := newOutboundConn(c, s, cfg) 107 | err = conn.Negotiate(true) 108 | if err == nil { 109 | // negotiation good 110 | log.WithFields(log.Fields{ 111 | "name": cfg.Name, 112 | }).Debug("Negotitation good") 113 | // start streaming 114 | var chnl chan ArticleEntry 115 | chnl, err = conn.StartStreaming() 116 | if err == nil { 117 | // register new connection 118 | f := &nntpFeed{ 119 | conn: conn, 120 | send: chnl, 121 | conf: cfg, 122 | } 123 | s.regis <- f 124 | // start streaming 125 | conn.StreamAndQuit() 126 | // deregister 127 | s.deregis <- f 128 | continue 129 | } 130 | } else { 131 | log.WithFields(log.Fields{ 132 | "name": cfg.Name, 133 | }).Info("outbound nntp connection failed to negotiate ", err) 134 | } 135 | conn.Quit() 136 | } else { 137 | // failed dial, do exponential backoff up to 1 hour 138 | if delay <= time.Hour { 139 | delay *= 2 140 | } 141 | log.WithFields(log.Fields{ 142 | "name": cfg.Name, 143 | }).Info("feed backoff for ", delay) 144 | time.Sleep(delay) 145 | } 146 | } 147 | } 148 | 149 | // download all new posts from a remote server 150 | func (s *Server) downloadPosts(cfg *config.FeedConfig) error { 151 | dialer := network.NewDialer(cfg.Proxy) 152 | c, err := dialer.Dial(cfg.Addr) 153 | if err != nil { 154 | return err 155 | } 156 | conn := newOutboundConn(c, s, cfg) 157 | err = conn.Negotiate(false) 158 | if err != nil { 159 | conn.Quit() 160 | return err 161 | } 162 | groups, err := conn.ListNewsgroups() 163 | if err != nil { 164 | conn.Quit() 165 | return err 166 | } 167 | for _, g := range groups { 168 | if cfg.Policy != nil && cfg.Policy.AllowGroup(g.String()) { 169 | log.WithFields(log.Fields{ 170 | "group": g, 171 | "pkg": "nntp-server", 172 | }).Debug("downloading group") 173 | err = conn.DownloadGroup(g) 174 | if err != nil { 175 | conn.Quit() 176 | return err 177 | } 178 | } 179 | } 180 | conn.Quit() 181 | return nil 182 | } 183 | 184 | func (s *Server) periodicDownload(cfg *config.FeedConfig) { 185 | for cfg.PullInterval > 0 { 186 | err := s.downloadPosts(cfg) 187 | if err != nil { 188 | // report error 189 | log.WithFields(log.Fields{ 190 | "feed": cfg.Name, 191 | "pkg": "nntp-server", 192 | "error": err, 193 | }).Error("periodic download failed") 194 | } 195 | time.Sleep(time.Minute * time.Duration(cfg.PullInterval)) 196 | } 197 | } 198 | 199 | // persist all outbound feeds 200 | func (s *Server) PersistFeeds() { 201 | for _, f := range s.Feeds { 202 | go s.persist(f) 203 | go s.periodicDownload(f) 204 | } 205 | 206 | feeds := make(map[string]*nntpFeed) 207 | 208 | for { 209 | select { 210 | case e, ok := <-s.send: 211 | if !ok { 212 | break 213 | } 214 | msgid := e.MessageID().String() 215 | group := e.Newsgroup().String() 216 | // TODO: determine anon 217 | anon := false 218 | // TODO: determine attachments 219 | attachments := false 220 | 221 | for _, f := range feeds { 222 | if f.conf.Policy != nil && !f.conf.Policy.Allow(msgid, group, anon, attachments) { 223 | // not allowed in this feed 224 | continue 225 | } 226 | log.WithFields(log.Fields{ 227 | "name": f.conf.Name, 228 | "msgid": msgid, 229 | "group": group, 230 | }).Debug("sending article") 231 | f.send <- e 232 | } 233 | break 234 | case f, ok := <-s.regis: 235 | if ok { 236 | log.WithFields(log.Fields{ 237 | "name": f.conf.Name, 238 | }).Debug("register feed") 239 | feeds[f.conf.Name] = f 240 | } 241 | break 242 | case f, ok := <-s.deregis: 243 | if ok { 244 | log.WithFields(log.Fields{ 245 | "name": f.conf.Name, 246 | }).Debug("deregister feed") 247 | delete(feeds, f.conf.Name) 248 | } 249 | break 250 | } 251 | } 252 | } 253 | 254 | // serve connections from listener 255 | func (s *Server) Serve(l net.Listener) (err error) { 256 | log.WithFields(log.Fields{ 257 | "pkg": "nntp-server", 258 | "addr": l.Addr(), 259 | }).Debug("Serving") 260 | for err == nil { 261 | var c net.Conn 262 | c, err = l.Accept() 263 | if err == nil { 264 | // we got a new connection 265 | go s.handleInboundConnection(c) 266 | } else { 267 | log.WithFields(log.Fields{ 268 | "pkg": "nntp-server", 269 | }).Error("failed to accept inbound connection", err) 270 | } 271 | } 272 | return 273 | } 274 | 275 | // get the article policy for a connection given its state 276 | func (s *Server) getPolicyFor(state *ConnState) ArticleAcceptor { 277 | return s.Acceptor 278 | } 279 | 280 | // recv inbound streaming messages 281 | func (s *Server) recvInboundStream(chnl chan ArticleEntry) { 282 | for { 283 | e, ok := <-chnl 284 | if ok { 285 | s.GotArticle(e.MessageID(), e.Newsgroup()) 286 | } else { 287 | return 288 | } 289 | } 290 | } 291 | 292 | // process an inbound connection 293 | func (s *Server) handleInboundConnection(c net.Conn) { 294 | log.WithFields(log.Fields{ 295 | "pkg": "nntp-server", 296 | "addr": c.RemoteAddr(), 297 | }).Debug("handling inbound connection") 298 | var nc Conn 299 | nc = newInboundConn(s, c) 300 | err := nc.Negotiate(true) 301 | if err == nil { 302 | // do they want to stream? 303 | if nc.WantsStreaming() { 304 | // yeeeeeh let's stream 305 | var chnl chan ArticleEntry 306 | chnl, err = nc.StartStreaming() 307 | // for inbound we will recv messages 308 | go s.recvInboundStream(chnl) 309 | nc.StreamAndQuit() 310 | log.WithFields(log.Fields{ 311 | "pkg": "nntp-server", 312 | "addr": c.RemoteAddr(), 313 | }).Info("streaming finished") 314 | return 315 | } else { 316 | // handle non streaming commands 317 | nc.ProcessInbound(s) 318 | } 319 | } else { 320 | log.WithFields(log.Fields{ 321 | "pkg": "nntp-server", 322 | "addr": c.RemoteAddr(), 323 | }).Warn("failed to negotiate with inbound connection", err) 324 | c.Close() 325 | } 326 | } 327 | -------------------------------------------------------------------------------- /lib/nntp/state.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | // state of an nntp connection 4 | type ConnState struct { 5 | // name of parent feed 6 | FeedName string `json:"feedname"` 7 | // name of the connection 8 | ConnName string `json:"connname"` 9 | // hostname of remote connection 10 | HostName string `json:"hostname"` 11 | // current nntp mode 12 | Mode Mode `json:"mode"` 13 | // current selected nntp newsgroup 14 | Group Newsgroup `json:"newsgroup"` 15 | // current selected nntp article 16 | Article string `json:"article"` 17 | // parent feed's policy 18 | Policy *FeedPolicy `json:"feedpolicy"` 19 | // is this connection open? 20 | Open bool `json:"open"` 21 | } 22 | -------------------------------------------------------------------------------- /lib/nntp/streaming.go: -------------------------------------------------------------------------------- 1 | package nntp 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // an nntp stream event 9 | // these are pipelined between nntp servers 10 | type StreamEvent string 11 | 12 | func (ev StreamEvent) MessageID() MessageID { 13 | parts := strings.Split(string(ev), " ") 14 | if len(parts) > 1 { 15 | return MessageID(parts[1]) 16 | } 17 | return "" 18 | } 19 | 20 | func (ev StreamEvent) String() string { 21 | return string(ev) 22 | } 23 | 24 | func (ev StreamEvent) Command() string { 25 | return strings.Split(ev.String(), " ")[0] 26 | } 27 | 28 | func (ev StreamEvent) Valid() bool { 29 | return strings.Count(ev.String(), " ") == 1 && ev.MessageID().Valid() 30 | } 31 | 32 | var stream_TAKETHIS = "TAKETHIS" 33 | var stream_CHECK = "CHECK" 34 | 35 | func createStreamEvent(cmd string, msgid MessageID) StreamEvent { 36 | if msgid.Valid() { 37 | return StreamEvent(fmt.Sprintf("%s %s", cmd, msgid)) 38 | } else { 39 | return "" 40 | } 41 | } 42 | 43 | func stream_rpl_Accept(msgid MessageID) StreamEvent { 44 | return createStreamEvent(RPL_StreamingAccept, msgid) 45 | } 46 | 47 | func stream_rpl_Reject(msgid MessageID) StreamEvent { 48 | return createStreamEvent(RPL_StreamingReject, msgid) 49 | } 50 | 51 | func stream_rpl_Defer(msgid MessageID) StreamEvent { 52 | return createStreamEvent(RPL_StreamingDefer, msgid) 53 | } 54 | 55 | func stream_rpl_Failed(msgid MessageID) StreamEvent { 56 | return createStreamEvent(RPL_StreamingFailed, msgid) 57 | } 58 | 59 | func stream_cmd_TAKETHIS(msgid MessageID) StreamEvent { 60 | return createStreamEvent(stream_TAKETHIS, msgid) 61 | } 62 | 63 | func stream_cmd_CHECK(msgid MessageID) StreamEvent { 64 | return createStreamEvent(stream_CHECK, msgid) 65 | } 66 | -------------------------------------------------------------------------------- /lib/srnd/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // main package for srndv2 3 | // called from main 4 | // 5 | package srnd 6 | -------------------------------------------------------------------------------- /lib/store/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // nntp article storage 3 | // 4 | package store 5 | -------------------------------------------------------------------------------- /lib/store/fs.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/base32" 5 | "fmt" 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/majestrate/srndv2/lib/crypto" 8 | "io" 9 | "io/ioutil" 10 | "net/textproto" 11 | "os" 12 | "path/filepath" 13 | "strconv" 14 | "time" 15 | ) 16 | 17 | const HighWaterHeader = "X-High-Water" 18 | const LowWaterHeader = "X-Low-Water" 19 | 20 | // filesystem storage of nntp articles and attachments 21 | type FilesystemStorage struct { 22 | root string 23 | discardAttachments bool 24 | } 25 | 26 | func (fs FilesystemStorage) String() string { 27 | return fs.root 28 | } 29 | 30 | func (fs FilesystemStorage) NewsgroupsDir() string { 31 | return filepath.Join(fs.root, "newsgroups") 32 | } 33 | 34 | func (fs FilesystemStorage) metadataFileForNewsgroup(newsgroup string) string { 35 | return filepath.Join(fs.newsgroupDir(newsgroup), "metadata") 36 | } 37 | 38 | func (fs FilesystemStorage) GetWatermark(newsgroup string) (hi, lo uint64, err error) { 39 | var hdr textproto.MIMEHeader 40 | hdr, err = fs.getMetadataForNewsgroup(newsgroup) 41 | if err == nil { 42 | hi, err = strconv.ParseUint(hdr.Get(HighWaterHeader), 10, 64) 43 | if err == nil { 44 | lo, err = strconv.ParseUint(hdr.Get(LowWaterHeader), 10, 64) 45 | } 46 | } 47 | return 48 | } 49 | 50 | func (fs FilesystemStorage) getMetadataForNewsgroup(newsgroup string) (hdr textproto.MIMEHeader, err error) { 51 | var f *os.File 52 | fp := fs.metadataFileForNewsgroup(newsgroup) 53 | _, err = os.Stat(fp) 54 | if os.IsNotExist(err) { 55 | f, err = os.OpenFile(fp, os.O_RDWR|os.O_CREATE, 0600) 56 | if err == nil { 57 | h := make(textproto.MIMEHeader) 58 | h.Set(HighWaterHeader, "0") 59 | c := textproto.NewConn(f) 60 | for k := range hdr { 61 | for _, v := range h[k] { 62 | err = c.PrintfLine("%s: %s", k, v) 63 | if err != nil { 64 | c.Close() 65 | return 66 | } 67 | } 68 | } 69 | c.Close() 70 | } 71 | } 72 | f, err = os.OpenFile(fp, os.O_RDWR, 0600) 73 | if err == nil { 74 | c := textproto.NewConn(f) 75 | hdr, err = c.ReadMIMEHeader() 76 | c.Close() 77 | } 78 | return 79 | } 80 | 81 | func (fs FilesystemStorage) nextIDForNewsgroup(newsgroup string) (id uint64, err error) { 82 | id, _, err = fs.GetWatermark(newsgroup) 83 | 84 | if err == nil { 85 | id++ 86 | var hdr textproto.MIMEHeader 87 | hdr, err = fs.getMetadataForNewsgroup(newsgroup) 88 | if err == nil { 89 | hdr.Set(HighWaterHeader, fmt.Sprintf("%d", id)) 90 | var f *os.File 91 | f, err = os.OpenFile(fs.metadataFileForNewsgroup(newsgroup), os.O_WRONLY, 0600) 92 | if err == nil { 93 | c := textproto.NewConn(f) 94 | for k := range hdr { 95 | for _, v := range hdr[k] { 96 | err = c.PrintfLine("%s: %s", k, v) 97 | if err != nil { 98 | c.Close() 99 | return 100 | } 101 | } 102 | } 103 | c.Close() 104 | } 105 | } 106 | } 107 | return 108 | } 109 | 110 | func (fs FilesystemStorage) GetAllNewsgroups() (newsgroups []string, err error) { 111 | err = filepath.Walk(fs.NewsgroupsDir(), filepath.WalkFunc(func(f string, info os.FileInfo, e error) (er error) { 112 | if e != nil { 113 | er = e 114 | newsgroups = nil 115 | } 116 | if info.IsDir() && err == nil { 117 | newsgroups = append(newsgroups, f) 118 | } 119 | return er 120 | })) 121 | return 122 | } 123 | 124 | func (fs FilesystemStorage) HasNewsgroup(newsgroup string) (has bool, err error) { 125 | _, err = os.Stat(fs.newsgroupDir(newsgroup)) 126 | has = err == nil 127 | return 128 | } 129 | 130 | // ensure the filesystem storage exists and is well formed and read/writable 131 | func (fs FilesystemStorage) Ensure() (err error) { 132 | _, err = os.Stat(fs.String()) 133 | if os.IsNotExist(err) { 134 | // directory does not exist, create it 135 | err = os.Mkdir(fs.String(), 0755) 136 | if err != nil { 137 | log.WithFields(log.Fields{ 138 | "pkg": "fs-store", 139 | "filepath": fs.String(), 140 | }).Error("failed to ensure directory", err) 141 | // failed to create initial directory 142 | return 143 | } 144 | } 145 | 146 | // ensure subdirectories 147 | for _, subdir := range []string{"att", "thm", "articles", "tmp", "newsgroups"} { 148 | fpath := filepath.Join(fs.String(), subdir) 149 | _, err = os.Stat(fpath) 150 | if os.IsNotExist(err) { 151 | // make subdirectory 152 | err = os.Mkdir(fpath, 0755) 153 | if err != nil { 154 | log.WithFields(log.Fields{ 155 | "pkg": "fs-store", 156 | "filepath": fpath, 157 | }).Error("failed to ensure sub-directory", err) 158 | // failed to create subdirectory 159 | return 160 | } 161 | } 162 | } 163 | return 164 | } 165 | 166 | // get the temp file directory 167 | func (fs FilesystemStorage) TempDir() string { 168 | return filepath.Join(fs.String(), "tmp") 169 | } 170 | 171 | // get the directory path for attachments 172 | func (fs FilesystemStorage) AttachmentDir() string { 173 | return filepath.Join(fs.String(), "att") 174 | } 175 | 176 | // get the directory path for articles 177 | func (fs FilesystemStorage) ArticleDir() string { 178 | return filepath.Join(fs.String(), "articles") 179 | } 180 | 181 | // get a temporary file we can use for read/write that deletes itself on close 182 | func (fs FilesystemStorage) obtainTempFile() (f *os.File, err error) { 183 | fname := fmt.Sprintf("tempfile-%x-%d", crypto.RandBytes(4), time.Now().Unix()) 184 | log.WithFields(log.Fields{ 185 | "pkg": "fs-store", 186 | "filepath": fname, 187 | }).Debug("opening temp file") 188 | f, err = os.OpenFile(filepath.Join(fs.TempDir(), fname), os.O_RDWR|os.O_CREATE, 0400) 189 | return 190 | } 191 | 192 | // store an article from a reader to disk 193 | func (fs FilesystemStorage) StoreArticle(r io.Reader, msgid, newsgroup string) (fpath string, err error) { 194 | err = fs.HasArticle(msgid) 195 | if err == nil { 196 | // discard the body as we have it stored already 197 | _, err = io.Copy(ioutil.Discard, r) 198 | log.WithFields(log.Fields{ 199 | "pkg": "fs-store", 200 | "msgid": msgid, 201 | }).Debug("discard article") 202 | } else if err == ErrNoSuchArticle { 203 | log.WithFields(log.Fields{ 204 | "pkg": "fs-store", 205 | "msgid": msgid, 206 | }).Debug("storing article") 207 | // don't have an article with this message id, write it to disk 208 | var f *os.File 209 | fpath = filepath.Join(fs.ArticleDir(), msgid) 210 | f, err = os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) 211 | if err == nil { 212 | // file opened okay, defer the close 213 | defer f.Close() 214 | // write to disk 215 | log.WithFields(log.Fields{ 216 | "pkg": "fs-store", 217 | "msgid": msgid, 218 | }).Debug("writing to disk") 219 | var n int64 220 | n, err = io.Copy(f, r) 221 | if err == nil { 222 | log.WithFields(log.Fields{ 223 | "pkg": "fs-store", 224 | "msgid": msgid, 225 | "written": n, 226 | }).Debug("wrote article to disk") 227 | // symlink 228 | g := fs.newsgroupDir(newsgroup) 229 | _, e := os.Stat(g) 230 | if os.IsNotExist(e) { 231 | err = os.Mkdir(g, 0700) 232 | } 233 | var nntpid uint64 234 | nntpid, err = fs.nextIDForNewsgroup(newsgroup) 235 | if err == nil { 236 | err = os.Symlink(filepath.Join("..", "..", "articles", msgid), filepath.Join(g, fmt.Sprintf("%d", nntpid))) 237 | } 238 | if err != nil { 239 | log.WithFields(log.Fields{ 240 | "pkg": "fs-store", 241 | "msgid": msgid, 242 | "group": newsgroup, 243 | }).Debug("failed to link article") 244 | } 245 | } else { 246 | log.WithFields(log.Fields{ 247 | "pkg": "fs-store", 248 | "msgid": msgid, 249 | "written": n, 250 | }).Error("write to disk failed") 251 | } 252 | } else { 253 | log.WithFields(log.Fields{ 254 | "pkg": "fs-store", 255 | "msgid": msgid, 256 | "filepath": fpath, 257 | }).Error("did not open file for storage", err) 258 | } 259 | } 260 | return 261 | } 262 | 263 | func (fs FilesystemStorage) newsgroupDir(group string) string { 264 | return filepath.Join(fs.NewsgroupsDir(), group) 265 | } 266 | 267 | // check if we have the artilce with this message id 268 | func (fs FilesystemStorage) HasArticle(msgid string) (err error) { 269 | fpath := fs.ArticleDir() 270 | fpath = filepath.Join(fpath, msgid) 271 | log.WithFields(log.Fields{ 272 | "pkg": "fs-store", 273 | "msgid": msgid, 274 | "filepath": fpath, 275 | }).Debug("check for article") 276 | _, err = os.Stat(fpath) 277 | if os.IsNotExist(err) { 278 | err = ErrNoSuchArticle 279 | } 280 | return 281 | } 282 | 283 | func (fs FilesystemStorage) DeleteArticle(msgid string) (err error) { 284 | err = os.Remove(filepath.Join(fs.ArticleDir(), msgid)) 285 | return 286 | } 287 | 288 | // store attachment onto filesystem 289 | func (fs FilesystemStorage) StoreAttachment(r io.Reader, filename string) (fpath string, err error) { 290 | if fs.discardAttachments { 291 | _, err = io.Copy(ioutil.Discard, r) 292 | return 293 | } 294 | // open temp file for storage 295 | var tf *os.File 296 | tf, err = fs.obtainTempFile() 297 | if err == nil { 298 | // we have the temp file 299 | 300 | // close tempfile when done 301 | defer func() { 302 | n := tf.Name() 303 | tf.Close() 304 | os.Remove(n) 305 | }() 306 | 307 | // create hasher 308 | h := crypto.Hash() 309 | // create multiwriter 310 | mw := io.MultiWriter(tf, h) 311 | 312 | log.WithFields(log.Fields{ 313 | "pkg": "fs-store", 314 | "filename": filename, 315 | }).Debug("writing to disk") 316 | var n int64 317 | // write all of the reader to the multiwriter 318 | n, err = io.Copy(mw, r) 319 | 320 | if err == nil { 321 | // successful write 322 | 323 | // get file checksum 324 | d := h.Sum(nil) 325 | 326 | // rename file to hash + extension from filename 327 | fpath = base32.StdEncoding.EncodeToString(d) + filepath.Ext(filename) 328 | fpath = filepath.Join(fs.AttachmentDir(), fpath) 329 | 330 | _, err = os.Stat(fpath) 331 | // is that file there? 332 | if os.IsNotExist(err) { 333 | // it's not there, let's write it 334 | var f *os.File 335 | f, err = os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE, 0644) 336 | if err == nil { 337 | // file opened 338 | defer f.Close() 339 | // seek to beginning of tempfile 340 | tf.Seek(0, os.SEEK_SET) 341 | // write all of the temp file to the storage file 342 | n, err = io.Copy(f, tf) 343 | // if err == nil by here it's all good 344 | l := log.WithFields(log.Fields{ 345 | "pkg": "fs-store", 346 | "filename": filename, 347 | "hash": d, 348 | "filepath": fpath, 349 | "size": n, 350 | }) 351 | if err == nil { 352 | l.Debug("wrote attachment to disk") 353 | } else { 354 | l.Error("failed to write attachment to disk", err) 355 | } 356 | } else { 357 | log.WithFields(log.Fields{ 358 | "pkg": "fs-store", 359 | "filename": filename, 360 | "hash": d, 361 | "filepath": fpath, 362 | }).Error("failed to open file") 363 | } 364 | } else { 365 | log.WithFields(log.Fields{ 366 | "pkg": "fs-store", 367 | "filename": filename, 368 | "hash": d, 369 | "filepath": fpath, 370 | "size": n, 371 | }).Debug("attachment exists on disk") 372 | } 373 | } 374 | } else { 375 | log.WithFields(log.Fields{ 376 | "pkg": "fs-store", 377 | "filename": filename, 378 | }).Error("cannot open temp file for attachment", err) 379 | } 380 | return 381 | } 382 | 383 | // open article given message-id 384 | // does not check validity 385 | func (fs FilesystemStorage) OpenArticle(msgid string) (r *os.File, err error) { 386 | r, err = os.Open(filepath.Join(fs.ArticleDir(), msgid)) 387 | return 388 | } 389 | 390 | func (fs FilesystemStorage) ForEachInGroup(group string, chnl chan string) { 391 | g := fs.newsgroupDir(group) 392 | filepath.Walk(g, func(path string, info os.FileInfo, err error) error { 393 | if info != nil { 394 | chnl <- info.Name() 395 | } 396 | return err 397 | }) 398 | } 399 | 400 | // create a new filesystem storage directory 401 | // ensure directory and subdirectories 402 | func NewFilesytemStorage(dirname string, unpackAttachments bool) (fs FilesystemStorage, err error) { 403 | dirname, err = filepath.Abs(dirname) 404 | if err == nil { 405 | log.WithFields(log.Fields{ 406 | "pkg": "fs-store", 407 | "filepath": dirname, 408 | }).Info("Creating New Filesystem Storage") 409 | fs = FilesystemStorage{ 410 | root: dirname, 411 | discardAttachments: unpackAttachments, 412 | } 413 | err = fs.Ensure() 414 | } 415 | return 416 | } 417 | -------------------------------------------------------------------------------- /lib/store/log.go: -------------------------------------------------------------------------------- 1 | package store 2 | -------------------------------------------------------------------------------- /lib/store/null.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/util" 5 | "io" 6 | "os" 7 | ) 8 | 9 | type nullStore struct{} 10 | 11 | func (n *nullStore) discard(r io.Reader) (s string, err error) { 12 | _, err = io.Copy(util.Discard, r) 13 | s = "/dev/null" 14 | return 15 | } 16 | 17 | func (n *nullStore) HasArticle(msgid string) error { 18 | return ErrNoSuchArticle 19 | } 20 | 21 | func (n *nullStore) StoreAttachment(r io.Reader, filename string) (string, error) { 22 | return n.discard(r) 23 | } 24 | 25 | func (n *nullStore) StoreArticle(r io.Reader, msgid, newsgroup string) (string, error) { 26 | return n.discard(r) 27 | } 28 | 29 | func (n *nullStore) DeleteArticle(msgid string) (err error) { 30 | return 31 | } 32 | 33 | func (n *nullStore) Ensure() (err error) { 34 | return 35 | } 36 | 37 | func (n *nullStore) ForEachInGroup(newsgroup string, chnl chan string) { 38 | return 39 | } 40 | 41 | func (n *nullStore) OpenArticle(msgid string) (r *os.File, err error) { 42 | err = ErrNoSuchArticle 43 | return 44 | } 45 | 46 | func (n *nullStore) HasNewsgroup(newsgroup string) (has bool, err error) { 47 | has = true 48 | return 49 | } 50 | 51 | func (n *nullStore) GetAllNewsgroups() (list []string, err error) { 52 | return 53 | } 54 | 55 | func (n *nullStore) GetWatermark(newsgroup string) (hi, lo uint64, err error) { 56 | return 57 | } 58 | 59 | // create a storage backend that does nothing 60 | func NewNullStorage() Storage { 61 | return &nullStore{} 62 | } 63 | -------------------------------------------------------------------------------- /lib/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | ) 8 | 9 | var ErrNoSuchArticle = errors.New("no such article") 10 | 11 | // storage for nntp articles and attachments 12 | type Storage interface { 13 | // store an attachment that we read from an io.Reader 14 | // filename is used to hint to store what extension to store it as 15 | // returns absolute filepath where attachment was stored and nil on success 16 | // returns emtpy string and error if an error ocurred while storing 17 | StoreAttachment(r io.Reader, filename string) (string, error) 18 | 19 | // store an article that we read from an io.Reader 20 | // message id is used to hint where the article is stored as well as newsgroup 21 | // returns absolute filepath to where the article was stored and nil on success 22 | // returns empty string and error if an error ocurred while storing 23 | StoreArticle(r io.Reader, msgid, newsgroup string) (string, error) 24 | 25 | // return nil if the article with the given message id exists in this storage 26 | // return ErrNoSuchArticle if it does not exist or an error if another error occured while checking 27 | HasArticle(msgid string) error 28 | 29 | // delete article from underlying storage 30 | DeleteArticle(msgid string) error 31 | 32 | // open article for reading 33 | OpenArticle(msgid string) (*os.File, error) 34 | 35 | // ensure the underlying storage backend is created 36 | Ensure() error 37 | 38 | // iterate over all messages in a newsgroup 39 | // send results down a channel 40 | ForEachInGroup(newsgroup string, cnhl chan string) 41 | 42 | // get a list of all newsgroups 43 | GetAllNewsgroups() ([]string, error) 44 | 45 | // determine if we have a newsgroup 46 | HasNewsgroup(newsgroup string) (bool, error) 47 | 48 | // get hi/lo watermark for newsgroup 49 | GetWatermark(newsgroup string) (uint64, uint64, error) 50 | } 51 | -------------------------------------------------------------------------------- /lib/thumbnail/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // attachment thumbnailing 3 | // 4 | package thumbnail 5 | -------------------------------------------------------------------------------- /lib/thumbnail/exec.go: -------------------------------------------------------------------------------- 1 | package thumbnail 2 | 3 | import ( 4 | "os/exec" 5 | "regexp" 6 | ) 7 | 8 | // thumbnail by executing an external program 9 | type ExecThumbnailer struct { 10 | // path to executable 11 | Exec string 12 | // regular expression that checks for acceptable infiles 13 | Accept *regexp.Regexp 14 | // function to generate arguments to use with external program 15 | // inf and outf are the filenames of the input and output files respectively 16 | // if this is nil the command will be passed in 2 arguments, infile and outfile 17 | GenArgs func(inf, outf string) []string 18 | } 19 | 20 | func (exe *ExecThumbnailer) CanThumbnail(infpath string) bool { 21 | re := exe.Accept.Copy() 22 | return re.MatchString(infpath) 23 | } 24 | 25 | func (exe *ExecThumbnailer) Generate(infpath, outfpath string) (err error) { 26 | // do sanity check 27 | if exe.CanThumbnail(infpath) { 28 | var args []string 29 | if exe.GenArgs == nil { 30 | args = []string{infpath, outfpath} 31 | } else { 32 | args = exe.GenArgs(infpath, outfpath) 33 | } 34 | cmd := exec.Command(exe.Exec, args...) 35 | _, err = cmd.CombinedOutput() 36 | } else { 37 | err = ErrCannotThumbanil 38 | } 39 | return 40 | } 41 | -------------------------------------------------------------------------------- /lib/thumbnail/multi.go: -------------------------------------------------------------------------------- 1 | package thumbnail 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ErrNoThumbnailer = errors.New("no thumbnailer found") 8 | 9 | type multiThumbnailer struct { 10 | impls []Thumbnailer 11 | } 12 | 13 | // get the frist matching thumbnailer that works with the given file 14 | // if we can't find one return nil 15 | func (mth *multiThumbnailer) getThumbnailer(fpath string) Thumbnailer { 16 | for _, th := range mth.impls { 17 | if th.CanThumbnail(fpath) { 18 | return th 19 | } 20 | } 21 | return nil 22 | } 23 | 24 | func (mth *multiThumbnailer) Generate(infpath, outfpath string) (err error) { 25 | th := mth.getThumbnailer(infpath) 26 | if th == nil { 27 | err = ErrNoThumbnailer 28 | } else { 29 | err = th.Generate(infpath, outfpath) 30 | } 31 | return 32 | } 33 | 34 | func (mth *multiThumbnailer) CanThumbnail(infpath string) bool { 35 | for _, th := range mth.impls { 36 | if th.CanThumbnail(infpath) { 37 | return true 38 | } 39 | } 40 | return false 41 | } 42 | 43 | func MuxThumbnailers(th ...Thumbnailer) Thumbnailer { 44 | return &multiThumbnailer{ 45 | impls: th, 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /lib/thumbnail/thumb.go: -------------------------------------------------------------------------------- 1 | package thumbnail 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | ) 9 | 10 | var ErrCannotThumbanil = errors.New("cannot thumbnail file") 11 | 12 | // a generator of thumbnails 13 | type Thumbnailer interface { 14 | // generate thumbnail of attachment 15 | // 16 | // infpath: absolute filepath to attachment 17 | // 18 | // outfpath: absolute filepath to thumbnail 19 | // 20 | // return error if the thumbnailing fails 21 | Generate(infpath, outfpath string) error 22 | 23 | // can we generate a thumbnail for this file? 24 | CanThumbnail(infpath string) bool 25 | } 26 | 27 | // thumbnail configuration 28 | type Config struct { 29 | // width of thumbnails 30 | ThumbW int 31 | // height of thumbnails 32 | ThumbH int 33 | // only generate jpg thumbnails 34 | JpegOnly bool 35 | } 36 | 37 | var defaultCfg = &Config{ 38 | ThumbW: 300, 39 | ThumbH: 200, 40 | JpegOnly: true, 41 | } 42 | 43 | // create an imagemagick thumbnailer 44 | func ImageMagickThumbnailer(convertPath string, conf *Config) Thumbnailer { 45 | if conf == nil { 46 | conf = defaultCfg 47 | } 48 | return &ExecThumbnailer{ 49 | Exec: convertPath, 50 | Accept: regexp.MustCompilePOSIX(`\.(png|jpg|jpeg|gif|webp)$`), 51 | GenArgs: func(inf, outf string) []string { 52 | if strings.HasSuffix(inf, ".gif") { 53 | inf += "[0]" 54 | } 55 | if conf.JpegOnly { 56 | outf += ".jpeg" 57 | } 58 | return []string{"-thumbnail", fmt.Sprintf("%d", conf.ThumbW), inf, outf} 59 | }, 60 | } 61 | } 62 | 63 | // generate a thumbnailer that uses ffmpeg 64 | func FFMpegThumbnailer(ffmpegPath string, conf *Config) Thumbnailer { 65 | if conf == nil { 66 | conf = defaultCfg 67 | } 68 | return &ExecThumbnailer{ 69 | Exec: ffmpegPath, 70 | Accept: regexp.MustCompilePOSIX(`\.(mkv|mp4|avi|webm|ogv|mov|m4v|mpg)$`), 71 | GenArgs: func(inf, outf string) []string { 72 | outf += ".jpeg" 73 | return []string{"-i", inf, "-vf", fmt.Sprintf("scale=%d:%d", conf.ThumbW, conf.ThumbH), "-vframes", "1", outf} 74 | }, 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /lib/thumbnail/thumb_test.go: -------------------------------------------------------------------------------- 1 | package thumbnail 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func doTestThumb(t *testing.T, th Thumbnailer, allowed, disallowed []string) { 8 | for _, f := range allowed { 9 | if !th.CanThumbnail(f) { 10 | t.Logf("cannot thumbnail expected file: %s", f) 11 | t.Fail() 12 | } 13 | } 14 | 15 | for _, f := range disallowed { 16 | if th.CanThumbnail(f) { 17 | t.Logf("can thumbnail wrong file: %s", f) 18 | t.Fail() 19 | } 20 | } 21 | 22 | } 23 | 24 | var _image = []string{"asd.gif", "asd.jpeg", "asd.jpg", "asd.png", "asd.webp"} 25 | var _video = []string{"asd.mkv", "asd.mov", "asd.mp4", "asd.m4v", "asd.ogv", "asd.avi", "asd.mpg", "asd.webm"} 26 | var _sound = []string{"asd.flac", "asd.mp3", "asd.mp2", "asd.wav", "asd.ogg", "asd.opus", "asd.m4a"} 27 | var _misc = []string{"asd.txt", "asd.swf"} 28 | var _garbage = []string{"asd", "asd.asd", "asd.asd.asd.asd", "asd.benis"} 29 | 30 | func TestCanThumbnailImage(t *testing.T) { 31 | th := ImageMagickThumbnailer("", nil) 32 | var allowed []string 33 | var disallowed []string 34 | 35 | allowed = append(allowed, _image...) 36 | disallowed = append(disallowed, _video...) 37 | disallowed = append(disallowed, _sound...) 38 | disallowed = append(disallowed, _misc...) 39 | disallowed = append(disallowed, _garbage...) 40 | 41 | doTestThumb(t, th, allowed, disallowed) 42 | } 43 | 44 | func TestCanThumbnailVideo(t *testing.T) { 45 | th := FFMpegThumbnailer("", nil) 46 | var allowed []string 47 | var disallowed []string 48 | 49 | allowed = append(allowed, _video...) 50 | disallowed = append(disallowed, _image...) 51 | disallowed = append(disallowed, _sound...) 52 | disallowed = append(disallowed, _misc...) 53 | disallowed = append(disallowed, _garbage...) 54 | 55 | doTestThumb(t, th, allowed, disallowed) 56 | } 57 | -------------------------------------------------------------------------------- /lib/util/cache.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "regexp" 7 | "strconv" 8 | ) 9 | 10 | func GetThreadHashHTML(file string) (thread string) { 11 | exp := regexp.MustCompilePOSIX(`thread-([0-9a-f]+)\.html`) 12 | matches := exp.FindStringSubmatch(file) 13 | if len(matches) != 2 { 14 | return "" 15 | } 16 | thread = matches[1] 17 | return 18 | } 19 | 20 | func GetGroupAndPageHTML(file string) (board string, page int) { 21 | exp := regexp.MustCompilePOSIX(`(.*)-([0-9]+)\.html`) 22 | matches := exp.FindStringSubmatch(file) 23 | if len(matches) != 3 { 24 | return "", -1 25 | } 26 | var err error 27 | board = matches[1] 28 | tmp := matches[2] 29 | page, err = strconv.Atoi(tmp) 30 | if err != nil { 31 | page = -1 32 | } 33 | return 34 | } 35 | 36 | func GetGroupForCatalogHTML(file string) (group string) { 37 | exp := regexp.MustCompilePOSIX(`catalog-(.+)\.html`) 38 | matches := exp.FindStringSubmatch(file) 39 | if len(matches) != 2 { 40 | return "" 41 | } 42 | group = matches[1] 43 | return 44 | } 45 | 46 | func GetFilenameForBoardPage(webroot_dir, boardname string, pageno int, json bool) string { 47 | var ext string 48 | if json { 49 | ext = "json" 50 | } else { 51 | ext = "html" 52 | } 53 | fname := fmt.Sprintf("%s-%d.%s", boardname, pageno, ext) 54 | return filepath.Join(webroot_dir, fname) 55 | } 56 | 57 | func GetFilenameForThread(webroot_dir, root_post_id string, json bool) string { 58 | var ext string 59 | if json { 60 | ext = "json" 61 | } else { 62 | ext = "html" 63 | } 64 | fname := fmt.Sprintf("thread-%s.%s", HashMessageID(root_post_id), ext) 65 | return filepath.Join(webroot_dir, fname) 66 | } 67 | 68 | func GetFilenameForCatalog(webroot_dir, boardname string) string { 69 | fname := fmt.Sprintf("catalog-%s.html", boardname) 70 | return filepath.Join(webroot_dir, fname) 71 | } 72 | 73 | func GetFilenameForIndex(webroot_dir string) string { 74 | return filepath.Join(webroot_dir, "index.html") 75 | } 76 | 77 | func GetFilenameForBoards(webroot_dir string) string { 78 | return filepath.Join(webroot_dir, "boards.html") 79 | } 80 | 81 | func GetFilenameForHistory(webroot_dir string) string { 82 | return filepath.Join(webroot_dir, "history.html") 83 | } 84 | 85 | func GetFilenameForUkko(webroot_dir string) string { 86 | return filepath.Join(webroot_dir, "ukko.html") 87 | } 88 | -------------------------------------------------------------------------------- /lib/util/discard.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | type ioDiscard struct{} 4 | 5 | func (discard *ioDiscard) Write(d []byte) (n int, err error) { 6 | n = len(d) 7 | return 8 | } 9 | 10 | func (discard *ioDiscard) Close() (err error) { 11 | return 12 | } 13 | 14 | var Discard = new(ioDiscard) 15 | -------------------------------------------------------------------------------- /lib/util/hex.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/sha1" 5 | "fmt" 6 | ) 7 | 8 | // message id hash 9 | func HashMessageID(msgid string) string { 10 | return fmt.Sprintf("%x", sha1.Sum([]byte(msgid))) 11 | } 12 | -------------------------------------------------------------------------------- /lib/util/ip.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "github.com/majestrate/srndv2/lib/crypto/nacl" 7 | "log" 8 | "net" 9 | ) 10 | 11 | // given an address 12 | // generate a new encryption key for it 13 | // return the encryption key and the encrypted address 14 | func NewAddrEnc(addr string) (string, string) { 15 | key_bytes := nacl.RandBytes(encAddrBytes()) 16 | key := base64.StdEncoding.EncodeToString(key_bytes) 17 | return key, EncAddr(addr, key) 18 | } 19 | 20 | // xor address with a one time pad 21 | // if the address isn't long enough it's padded with spaces 22 | func EncAddr(addr, key string) string { 23 | key_bytes, err := base64.StdEncoding.DecodeString(key) 24 | 25 | if err != nil { 26 | log.Println("encAddr() key base64 decode", err) 27 | return "" 28 | } 29 | 30 | if len(addr) > len(key_bytes) { 31 | log.Println("encAddr() len(addr) > len(key_bytes)") 32 | return "" 33 | } 34 | 35 | // pad with spaces 36 | for len(addr) < len(key_bytes) { 37 | addr += " " 38 | } 39 | 40 | addr_bytes := []byte(addr) 41 | res_bytes := make([]byte, len(addr_bytes)) 42 | for idx, b := range key_bytes { 43 | res_bytes[idx] = addr_bytes[idx] ^ b 44 | } 45 | 46 | return base64.StdEncoding.EncodeToString(res_bytes) 47 | } 48 | 49 | // number of bytes to use in otp 50 | func encAddrBytes() int { 51 | return 64 52 | } 53 | 54 | func IsSubnet(cidr string) (bool, *net.IPNet) { 55 | _, ipnet, err := net.ParseCIDR(cidr) 56 | if err == nil { 57 | return true, ipnet 58 | } 59 | return false, nil 60 | } 61 | 62 | func IPNet2MinMax(inet *net.IPNet) (min, max net.IP) { 63 | netb := []byte(inet.IP) 64 | maskb := []byte(inet.Mask) 65 | maxb := make([]byte, len(netb)) 66 | 67 | for i, _ := range maxb { 68 | maxb[i] = netb[i] | (^maskb[i]) 69 | } 70 | min = net.IP(netb) 71 | max = net.IP(maxb) 72 | return 73 | } 74 | 75 | func ZeroIPString(ip net.IP) string { 76 | p := ip 77 | 78 | if len(ip) == 0 { 79 | return "" 80 | } 81 | 82 | if p4 := p.To4(); len(p4) == net.IPv4len { 83 | return fmt.Sprintf("%03d.%03d.%03d.%03d", p4[0], p4[1], p4[2], p4[3]) 84 | } 85 | if len(p) == net.IPv6len { 86 | //>IPv6 87 | //ishygddt 88 | return fmt.Sprintf("[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]", p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]) 89 | } 90 | return "?" 91 | } 92 | -------------------------------------------------------------------------------- /lib/util/nntp_login.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/sha512" 5 | "encoding/base64" 6 | "encoding/hex" 7 | "github.com/majestrate/srndv2/lib/crypto/nacl" 8 | ) 9 | 10 | // generate a login salt for nntp users 11 | func GenLoginCredSalt() (salt string) { 12 | salt = randStr(128) 13 | return 14 | } 15 | 16 | // do nntp login credential hash given password and salt 17 | func NntpLoginCredHash(passwd, salt string) (str string) { 18 | var b []byte 19 | b = append(b, []byte(passwd)...) 20 | b = append(b, []byte(salt)...) 21 | h := sha512.Sum512(b) 22 | str = base64.StdEncoding.EncodeToString(h[:]) 23 | return 24 | } 25 | 26 | // make a random string 27 | func randStr(length int) string { 28 | return hex.EncodeToString(nacl.RandBytes(length))[length:] 29 | } 30 | -------------------------------------------------------------------------------- /lib/util/post.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "strings" 4 | 5 | func IsSage(str string) bool { 6 | str = strings.ToLower(str) 7 | return str == "sage" || strings.HasPrefix(str, "sage ") 8 | } 9 | -------------------------------------------------------------------------------- /lib/util/time.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "time" 4 | 5 | // time for right now as int64 6 | func TimeNow() int64 { 7 | return time.Now().UTC().Unix() 8 | } 9 | -------------------------------------------------------------------------------- /lib/webhooks/doc.go: -------------------------------------------------------------------------------- 1 | // 2 | // nntpchan web hooks 3 | // 4 | package webhooks 5 | -------------------------------------------------------------------------------- /lib/webhooks/http.go: -------------------------------------------------------------------------------- 1 | package webhooks 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/majestrate/srndv2/lib/config" 8 | "github.com/majestrate/srndv2/lib/nntp" 9 | "github.com/majestrate/srndv2/lib/nntp/message" 10 | "github.com/majestrate/srndv2/lib/store" 11 | "io" 12 | "mime" 13 | "mime/multipart" 14 | "net/http" 15 | "net/textproto" 16 | "net/url" 17 | "regexp" 18 | "strings" 19 | ) 20 | 21 | // web hook implementation 22 | type httpWebhook struct { 23 | conf *config.WebhookConfig 24 | storage store.Storage 25 | hdr *message.HeaderIO 26 | } 27 | 28 | func (h *httpWebhook) SentArticleVia(msgid nntp.MessageID, name string) { 29 | // web hooks don't care about feed state 30 | } 31 | 32 | // we got a new article 33 | func (h *httpWebhook) GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) { 34 | h.sendArticle(msgid, group) 35 | } 36 | 37 | func (h *httpWebhook) sendArticle(msgid nntp.MessageID, group nntp.Newsgroup) { 38 | f, err := h.storage.OpenArticle(msgid.String()) 39 | if err == nil { 40 | u, _ := url.Parse(h.conf.URL) 41 | var r *http.Response 42 | var ctype string 43 | if h.conf.Dialect == "vichan" { 44 | c := textproto.NewConn(f) 45 | var hdr textproto.MIMEHeader 46 | hdr, err = c.ReadMIMEHeader() 47 | if err == nil { 48 | var body io.Reader 49 | ctype = hdr.Get("Content-Type") 50 | if ctype == "" || strings.HasPrefix(ctype, "text/plain") { 51 | ctype = "text/plain" 52 | } 53 | ctype = strings.Replace(strings.ToLower(ctype), "multipart/mixed", "multipart/form-data", 1) 54 | q := u.Query() 55 | for k, vs := range hdr { 56 | for _, v := range vs { 57 | q.Add(k, v) 58 | } 59 | } 60 | q.Set("Content-Type", ctype) 61 | u.RawQuery = q.Encode() 62 | 63 | if strings.HasPrefix(ctype, "multipart") { 64 | pr, pw := io.Pipe() 65 | log.Debug("using pipe") 66 | go func(in io.Reader, out io.WriteCloser) { 67 | _, params, _ := mime.ParseMediaType(ctype) 68 | if params == nil { 69 | // send as whatever lol 70 | io.Copy(out, in) 71 | } else { 72 | boundary, _ := params["boundary"] 73 | mpr := multipart.NewReader(in, boundary) 74 | mpw := multipart.NewWriter(out) 75 | mpw.SetBoundary(boundary) 76 | for { 77 | part, err := mpr.NextPart() 78 | if err == io.EOF { 79 | err = nil 80 | break 81 | } else if err == nil { 82 | // get part header 83 | h := part.Header 84 | // rewrite header part for php 85 | cd := h.Get("Content-Disposition") 86 | r := regexp.MustCompile(`filename="(.*)"`) 87 | // YOLO 88 | parts := r.FindStringSubmatch(cd) 89 | if len(parts) > 1 { 90 | fname := parts[1] 91 | h.Set("Content-Disposition", fmt.Sprintf(`filename="%s"; name="attachment[]"`, fname)) 92 | } 93 | // make write part 94 | wp, err := mpw.CreatePart(h) 95 | if err == nil { 96 | // write part out 97 | io.Copy(wp, part) 98 | } else { 99 | log.Errorf("error writng webhook part: %s", err.Error()) 100 | } 101 | } 102 | part.Close() 103 | } 104 | mpw.Close() 105 | } 106 | out.Close() 107 | }(c.R, pw) 108 | body = pr 109 | } else { 110 | body = f 111 | } 112 | r, err = http.Post(u.String(), ctype, body) 113 | } 114 | } else { 115 | var sz int64 116 | sz, err = f.Seek(0, 2) 117 | if err != nil { 118 | return 119 | } 120 | f.Seek(0, 0) 121 | // regular webhook 122 | ctype = "text/plain; charset=UTF-8" 123 | cl := new(http.Client) 124 | r, err = cl.Do(&http.Request{ 125 | ContentLength: sz, 126 | URL: u, 127 | Method: "POST", 128 | Body: f, 129 | }) 130 | } 131 | if err == nil && r != nil { 132 | dec := json.NewDecoder(r.Body) 133 | result := make(map[string]interface{}) 134 | err = dec.Decode(&result) 135 | if err == nil || err == io.EOF { 136 | msg, ok := result["error"] 137 | if ok { 138 | log.Warnf("hook gave error: %s", msg) 139 | } else { 140 | log.Debugf("hook response: %s", result) 141 | } 142 | } else { 143 | log.Warnf("hook response does not look like json: %s", err) 144 | } 145 | r.Body.Close() 146 | log.Infof("hook called for %s", msgid) 147 | } 148 | } else { 149 | f.Close() 150 | } 151 | if err != nil { 152 | log.Errorf("error calling web hook %s: %s", h.conf.Name, err.Error()) 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /lib/webhooks/multi.go: -------------------------------------------------------------------------------- 1 | package webhooks 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/nntp" 5 | ) 6 | 7 | // webhook multiplexer 8 | type multiWebhook struct { 9 | hooks []Webhook 10 | } 11 | 12 | // got an article 13 | func (m *multiWebhook) GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) { 14 | for _, h := range m.hooks { 15 | h.GotArticle(msgid, group) 16 | } 17 | } 18 | 19 | func (m *multiWebhook) SentArticleVia(msgid nntp.MessageID, feedname string) { 20 | for _, h := range m.hooks { 21 | h.SentArticleVia(msgid, feedname) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /lib/webhooks/webhooks.go: -------------------------------------------------------------------------------- 1 | package webhooks 2 | 3 | import ( 4 | "github.com/majestrate/srndv2/lib/config" 5 | "github.com/majestrate/srndv2/lib/nntp" 6 | "github.com/majestrate/srndv2/lib/nntp/message" 7 | "github.com/majestrate/srndv2/lib/store" 8 | ) 9 | 10 | type Webhook interface { 11 | // implements nntp.EventHooks 12 | nntp.EventHooks 13 | } 14 | 15 | // create webhook multiplexing multiple web hooks 16 | func NewWebhooks(conf []*config.WebhookConfig, st store.Storage) Webhook { 17 | h := message.NewHeaderIO() 18 | var hooks []Webhook 19 | for _, c := range conf { 20 | hooks = append(hooks, &httpWebhook{ 21 | conf: c, 22 | storage: st, 23 | hdr: h, 24 | }) 25 | } 26 | 27 | return &multiWebhook{ 28 | hooks: hooks, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /srnd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "os/signal" 8 | "runtime" 9 | "srnd" 10 | "strings" 11 | "syscall" 12 | ) 13 | 14 | func main() { 15 | 16 | daemon := new(srnd.NNTPDaemon) 17 | if len(os.Args) > 1 { 18 | action := os.Args[1] 19 | if action == "setup" { 20 | log.Println("Setting up SRNd base...") 21 | daemon.Setup() 22 | log.Println("Setup Done") 23 | } else if action == "run" { 24 | log.Printf("Starting up %s...", srnd.Version()) 25 | daemon.Setup() 26 | c := make(chan os.Signal, 1) 27 | signal.Notify(c, os.Interrupt) 28 | signal.Notify(c, syscall.SIGTERM, syscall.SIGHUP) 29 | go func() { 30 | for { 31 | sig := <-c 32 | if sig != syscall.SIGHUP { 33 | break 34 | } 35 | srnd.ReloadTemplates() 36 | daemon.Reload() 37 | } 38 | log.Println("Shutting down...") 39 | daemon.End() 40 | os.Exit(0) 41 | }() 42 | daemon.Run() 43 | } else if action == "tool" { 44 | if len(os.Args) > 2 { 45 | tool := os.Args[2] 46 | if tool == "mod" { 47 | if len(os.Args) >= 5 { 48 | action := os.Args[3] 49 | if action == "add" { 50 | pk := os.Args[4] 51 | daemon.Setup() 52 | db := daemon.GetDatabase() 53 | err := db.MarkModPubkeyGlobal(pk) 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | } else if action == "del" { 58 | pk := os.Args[4] 59 | daemon.Setup() 60 | db := daemon.GetDatabase() 61 | err := db.UnMarkModPubkeyGlobal(pk) 62 | if err != nil { 63 | log.Fatal(err) 64 | } 65 | } 66 | } else { 67 | fmt.Fprintf(os.Stdout, "usage: %s tool mod [add|del] pubkey\n", os.Args[0]) 68 | } 69 | } else if tool == "rethumb" { 70 | if len(os.Args) >= 4 { 71 | threads := runtime.NumCPU() 72 | arg := strings.ToLower(os.Args[3]) 73 | switch arg { 74 | case "missing": 75 | srnd.ThumbnailTool(threads, true) 76 | return 77 | case "all": 78 | srnd.ThumbnailTool(threads, false) 79 | return 80 | } 81 | } 82 | fmt.Fprintf(os.Stdout, "usage: %s tool rethumb [missing|all]\n", os.Args[0]) 83 | } else if tool == "keygen" { 84 | srnd.KeygenTool() 85 | } else if tool == "nntp" { 86 | if len(os.Args) >= 5 { 87 | action := os.Args[3] 88 | if action == "del-login" { 89 | daemon.Setup() 90 | daemon.DelNNTPLogin(os.Args[4]) 91 | } else if action == "add-login" { 92 | if len(os.Args) == 6 { 93 | username := os.Args[4] 94 | passwd := os.Args[5] 95 | daemon.Setup() 96 | daemon.AddNNTPLogin(username, passwd) 97 | } else { 98 | fmt.Fprintf(os.Stdout, "Usage: %s tool nntp add-login username password\n", os.Args[0]) 99 | } 100 | } else { 101 | fmt.Fprintf(os.Stdout, "Usage: %s tool nntp [add-login|del-login]\n", os.Args[0]) 102 | } 103 | } else { 104 | fmt.Fprintf(os.Stdout, "Usage: %s tool nntp [add-login|del-login]\n", os.Args[0]) 105 | } 106 | } else { 107 | fmt.Fprintf(os.Stdout, "Usage: %s tool [rethumb|keygen|nntp|mod]\n", os.Args[0]) 108 | } 109 | } else { 110 | fmt.Fprintf(os.Stdout, "Usage: %s tool [rethumb|keygen|nntp|mod]\n", os.Args[0]) 111 | } 112 | } else { 113 | log.Println("Invalid action:", action) 114 | } 115 | } else { 116 | fmt.Fprintf(os.Stdout, "Usage: %s [setup|run|tool]\n", os.Args[0]) 117 | } 118 | } 119 | --------------------------------------------------------------------------------