8 | {{ day.Log }}
10 |7 | {{if email != ""}} 8 | {{ email }} 9 | {{else}} 10 | no E-mail provided by administrator 11 | {{end}} 12 |
13 | {{if twitter != ""}} 14 |16 | @{{ twitter }} 17 |
18 | {{end}} 19 | {{if github != ""}} 20 || # | 10 |Username | 11 |Lines | 12 |KB | 13 |
|---|---|---|---|
| {{ i + 1 }} | 19 |{{user.Username}} | 20 |{{user.Lines}} | 21 |{{user.KiloBytes}} | 22 |
Generated: {{.Generated}}
28 | {{end}} -------------------------------------------------------------------------------- /server/views/footer.jet: -------------------------------------------------------------------------------- 1 | {{block footer()}} 2 | 28 | {{end}} -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 dbc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /tool/avro/message.go: -------------------------------------------------------------------------------- 1 | // Code generated by github.com/actgardner/gogen-avro. DO NOT EDIT. 2 | /* 3 | * SOURCE: 4 | * message.avsc 5 | */ 6 | 7 | package avro 8 | 9 | import ( 10 | "github.com/actgardner/gogen-avro/container" 11 | "io" 12 | ) 13 | 14 | type Message struct { 15 | Time int64 16 | Channel string 17 | Nick string 18 | Message string 19 | } 20 | 21 | func DeserializeMessage(r io.Reader) (*Message, error) { 22 | return readMessage(r) 23 | } 24 | 25 | func NewMessageWriter(writer io.Writer, codec container.Codec, recordsPerBlock int64) (*container.Writer, error) { 26 | str := &Message{} 27 | return container.NewWriter(writer, codec, recordsPerBlock, str.Schema()) 28 | } 29 | 30 | func NewMessage() *Message { 31 | v := &Message{} 32 | 33 | return v 34 | } 35 | 36 | func (r *Message) Schema() string { 37 | return "{\"fields\":[{\"name\":\"Time\",\"type\":\"long\"},{\"name\":\"Channel\",\"type\":\"string\"},{\"name\":\"Nick\",\"type\":\"string\"},{\"name\":\"Message\",\"type\":\"string\"}],\"name\":\"message\",\"type\":\"record\"}" 38 | } 39 | 40 | func (r *Message) Serialize(w io.Writer) error { 41 | return writeMessage(r, w) 42 | } 43 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | server: 4 | build: ./server 5 | container_name: orl-server 6 | user: "${USER_UID}" 7 | restart: unless-stopped 8 | env_file: 9 | - .env 10 | volumes: 11 | - ${LOGS_PATH}:/logs:ro 12 | logger: 13 | build: ./logger 14 | container_name: orl-logger 15 | user: "${USER_UID}" 16 | restart: unless-stopped 17 | env_file: 18 | - .env 19 | volumes: 20 | - ${VAR_ORL}:/logger 21 | - ${LOGS_PATH}:/logs 22 | frontend: 23 | image: nginx:latest 24 | container_name: orl-web-static 25 | restart: unless-stopped 26 | volumes: 27 | - ./package/etc/nginx/sites-enabled/overrustlelogs.net.conf:/etc/nginx/conf.d/nginx.conf:ro 28 | - ./server/assets:/var/overrustlelogs/public/assets/:ro 29 | cache: 30 | image: varnish:latest 31 | container_name: orl-web-cache 32 | restart: unless-stopped 33 | ports: 34 | - "${PORT}:80" 35 | volumes: 36 | - ./package/etc/varnish/default.vcl:/etc/varnish/default.vcl:ro 37 | bot: 38 | build: ./bot 39 | container_name: orl-bot 40 | user: "${USER_UID}" 41 | restart: unless-stopped 42 | env_file: 43 | - .env 44 | volumes: 45 | - ${VAR_ORL}:/bot 46 | - ${LOGS_PATH}:/logs:ro 47 | -------------------------------------------------------------------------------- /common/config.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/BurntSushi/toml" 7 | ) 8 | 9 | // Config settings 10 | type Config struct { 11 | DestinyGG struct { 12 | LogHost string `toml:"logHost"` 13 | SocketURL string `toml:"socketURL"` 14 | OriginURL string `toml:"originURL"` 15 | Cookie string `toml:"cookie"` 16 | } `toml:"destinyGG"` 17 | Twitch struct { 18 | LogHost string `toml:"logHost"` 19 | SocketURL string `toml:"socketURL"` 20 | OriginURL string `toml:"originURL"` 21 | ClientID string `toml:"clientID"` 22 | OAuth string `toml:"oAuth"` 23 | Nick string `toml:"nick"` 24 | Admins []string `toml:"admins"` 25 | CommandChannel string `toml:"commandChannel"` 26 | } `toml:"twitch"` 27 | Bot struct { 28 | Admins []string `toml:"admins"` 29 | } `toml:"bot"` 30 | LogHost string `toml:"logHost"` 31 | MaxOpenLogs int `toml:"maxOpenLogs"` 32 | } 33 | 34 | var config *Config 35 | 36 | // SetupConfig loads config data from json 37 | func SetupConfig(path string) *Config { 38 | config = &Config{} 39 | 40 | _, err := toml.DecodeFile(path, &config) 41 | if err != nil { 42 | log.Fatalf("error parsing config, err : %v", err) 43 | } 44 | 45 | return config 46 | } 47 | 48 | // GetConfig returns config 49 | func GetConfig() *Config { 50 | return config 51 | } 52 | -------------------------------------------------------------------------------- /scripts/update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export src="github.com/MemeLabs/overrustlelogs" 4 | 5 | ## local mode to deploy ignoring git 6 | if [[ $1 == "local" ]]; then 7 | TODO=$2 8 | MODE=local 9 | else 10 | TODO=$1 11 | MODE=default 12 | git pull 13 | fi 14 | 15 | source /etc/profile 16 | 17 | updateBot(){ 18 | docker-compose up -d --build bot 19 | echo "updated the orl-bot" 20 | } 21 | 22 | updateServer(){ 23 | docker-compose up -d --build server 24 | service varnish restart 25 | echo "updated the orl-server" 26 | } 27 | 28 | updateLogger(){ 29 | docker-compose up -d --build logger 30 | echo "updated the orl-logger" 31 | } 32 | 33 | updateServerPack(){ 34 | rm -rf /var/overrustlelogs/public/assets 35 | 36 | cp -r $HOME/overrustlelogs/server/assets /var/overrustlelogs/public/ 37 | chown -R overrustlelogs:overrustlelogs /var/overrustlelogs/public/assets 38 | 39 | service varnish restart 40 | echo "updated the server assets" 41 | } 42 | 43 | if [[ $TODO == "bot" ]]; then 44 | echo "updating the orl-bot..." 45 | updateBot 46 | elif [[ $TODO == "server" ]]; then 47 | echo "updating the orl-server..." 48 | updateServer 49 | elif [[ $TODO == "serverpack" ]]; then 50 | echo "updating the orl-server assets..." 51 | updateServerPack 52 | elif [[ $TODO == "logger" ]]; then 53 | echo "updating the orl-logger" 54 | updateLogger 55 | else 56 | echo "updating everything..." 57 | updateBot 58 | updateLogger 59 | updateServer 60 | updateServerPack 61 | echo "updating complete" 62 | fi -------------------------------------------------------------------------------- /common/parser.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | // Log timestamp format 11 | var ( 12 | MessageTimeLayout = "[2006-01-02 15:04:05 MST] " 13 | MessageTimeLayoutLength = len(MessageTimeLayout) 14 | MessageDateLayout = "2006-01-02" 15 | ) 16 | 17 | // ParseMessageLine parse log line into message struct 18 | func ParseMessageLine(b string) (*Message, error) { 19 | if len(b) < MessageTimeLayoutLength { 20 | return nil, fmt.Errorf("supplied line is too short to be parsed as message: %s", b) 21 | } 22 | 23 | ts, err := time.Parse(MessageTimeLayout, b[:MessageTimeLayoutLength]) 24 | if err != nil { 25 | return nil, fmt.Errorf("malformed date in message: %v", err) 26 | } 27 | 28 | b = b[MessageTimeLayoutLength:] 29 | nickLength := strings.IndexRune(b, ':') 30 | if nickLength >= len(b) || nickLength == -1 { 31 | return nil, fmt.Errorf("malformed nick in message: %s", b) 32 | } 33 | 34 | // should never happen 35 | if nickLength+2 > len(b) { 36 | return nil, fmt.Errorf("nickLength is longer than whole line") 37 | } 38 | 39 | return &Message{ 40 | Nick: b[:nickLength], 41 | Data: b[nickLength+2:], 42 | Time: ts, 43 | }, nil 44 | } 45 | 46 | var channelPathPattern = regexp.MustCompile("/([a-zA-Z0-9_]+) chatlog/") 47 | 48 | // ExtractChannelFromPath ... 49 | func ExtractChannelFromPath(p string) (string, error) { 50 | match := channelPathPattern.FindStringSubmatch(p) 51 | if match == nil { 52 | return "", fmt.Errorf("supplied path does not contain channel name: %s", p) 53 | } 54 | return match[1], nil 55 | } 56 | -------------------------------------------------------------------------------- /common/nicklist_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | ) 7 | 8 | func TestWrite(t *testing.T) { 9 | n := NickList{} 10 | n.Add("foo") 11 | n.Add("bar") 12 | n.Add("baz") 13 | n.Add("qux") 14 | if err := n.WriteTo("/tmp/nicks"); err != nil { 15 | log.Printf("error saving nicks list %s", err) 16 | t.Fail() 17 | } 18 | } 19 | 20 | func TestRead(t *testing.T) { 21 | n := NickList{} 22 | n.Add("foo") 23 | n.Add("bar") 24 | n.Add("baz") 25 | n.Add("qux") 26 | if err := n.WriteTo("/tmp/nicks"); err != nil { 27 | log.Printf("error writing nick list %s", err) 28 | t.Fail() 29 | return 30 | } 31 | 32 | r := NickList{} 33 | if err := ReadNickList(r, "/tmp/nicks"); err != nil { 34 | log.Printf("error reading nick list %s", err) 35 | t.Fail() 36 | return 37 | } 38 | 39 | for _, k := range []string{"foo", "bar", "baz", "qux"} { 40 | if _, ok := r[k]; !ok { 41 | log.Printf("nick not found %s", k) 42 | t.Fail() 43 | return 44 | } 45 | } 46 | } 47 | 48 | func TestRemove(t *testing.T) { 49 | n := NickList{} 50 | n.Add("foo") 51 | n.Add("bar") 52 | n.Add("baz") 53 | n.Add("qux") 54 | if err := n.WriteTo("/tmp/nicks"); err != nil { 55 | log.Printf("error writing nick list %s", err) 56 | t.Fail() 57 | return 58 | } 59 | 60 | r := NickList{} 61 | if err := ReadNickList(r, "/tmp/nicks"); err != nil { 62 | log.Printf("error reading nick list %s", err) 63 | t.Fail() 64 | return 65 | } 66 | 67 | r.Remove("foo") 68 | if err := r.WriteTo("/tmp/nicks"); err != nil { 69 | log.Printf("error writing nick list %s", err) 70 | t.Fail() 71 | return 72 | } 73 | 74 | rn := NickList{} 75 | if err := ReadNickList(rn, "/tmp/nicks"); err != nil { 76 | log.Printf("error reading nick list %s", err) 77 | t.Fail() 78 | return 79 | } 80 | 81 | for _, k := range []string{"bar", "baz", "qux"} { 82 | if _, ok := r[k]; !ok { 83 | log.Printf("nick not found %s", k) 84 | t.Fail() 85 | return 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OverRustle Logs 2 | 3 | A chat log suite for [Destiny.gg](https://www.destiny.gg/bigscreen) and [Twitch.tv](http://twitch.tv). 4 | 5 | ## Setting Up OverRustle Logs 6 | 7 | These instructions assume you are installing on Ubuntu 14.04 or higher. 8 | 9 | ### Step 1 10 | 11 | Install git. 12 | 13 | ```bash 14 | sudo apt-get install git --assume-yes 15 | ``` 16 | 17 | ### Step 2 18 | 19 | Clone the overrustlelogs repo. 20 | 21 | ```bash 22 | git clone https://github.com/MemeLabs/overrustlelogs.git 23 | 24 | cd overrustlelogs 25 | ``` 26 | 27 | ### Step 3 28 | 29 | Copy and edit the .env file. Edit the overrustlelogs.toml file. 30 | 31 | ```bash 32 | # cd into overrustlelogs if not already in there 33 | cd ./overrustlelogs 34 | cp ./.env.example ./.env 35 | 36 | # changing paths in this requires to change paths in install.sh 37 | vim .env 38 | 39 | # copy config files to working dir 40 | cp ./package/var/overrustlelogs/* 41 | 42 | # few things you need to edit here too 43 | vim ./overrustlelogs.toml 44 | 45 | # set the channels you want to log 46 | vim ./channels.json 47 | 48 | # change server_name's in the nginx config if you need 49 | vim ./package/etc/nginx/sites-enabled/overrustlelogs.net.conf 50 | ``` 51 | 52 | ### Step 4 (Docker) 53 | start the stack 54 | 55 | ```bash 56 | docker-compose up -d 57 | ``` 58 | 59 | ### Step 4 (Host) 60 | 61 | Run the install script from the repo root directory. 62 | 63 | ```bash 64 | # cd into overrustlelogs if not already in there 65 | cd overrustlelogs 66 | # use sudo if you're not root 67 | # only use all.sh if you're on ubuntu and don't have nginx, varnish, docker and 68 | # docker-compose installed, otherwise install everything manually and run install.sh afterwards 69 | ./scripts/all.sh 70 | ``` 71 | 72 | ## Updating 73 | 74 | Run the update script from the repo root directory. 75 | 76 | ```bash 77 | # cd into overrustlelogs if not already in there 78 | cd overrustlelogs 79 | # use sudo if you're not root 80 | ./scripts/update.sh 81 | ``` 82 | -------------------------------------------------------------------------------- /common/bigquerywriter.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "log" 7 | 8 | "cloud.google.com/go/bigquery" 9 | "google.golang.org/api/option" 10 | ) 11 | 12 | // BigQueryWriterConfig ... 13 | type BigQueryWriterConfig struct { 14 | ProjectID string `json:"projectID"` 15 | DatasetID string `json:"datasetID"` 16 | TableID string `json:"tableID"` 17 | ServiceAccountJSON string `json:"serviceAccountJSON"` 18 | } 19 | 20 | // BigQueryWriter ... 21 | type BigQueryWriter struct { 22 | client *bigquery.Client 23 | config BigQueryWriterConfig 24 | } 25 | 26 | // NewBigQueryWriter ... 27 | func NewBigQueryWriter(config BigQueryWriterConfig) (*BigQueryWriter, error) { 28 | client, err := bigquery.NewClient( 29 | context.Background(), 30 | config.ProjectID, 31 | option.WithServiceAccountFile(config.ServiceAccountJSON), 32 | ) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | return &BigQueryWriter{client, config}, nil 38 | } 39 | 40 | // Write ... 41 | func (w *BigQueryWriter) Write(b []byte) (int, error) { 42 | source := bigquery.NewReaderSource(bytes.NewReader(b)) 43 | source.AllowJaggedRows = true 44 | source.SourceFormat = bigquery.Avro 45 | 46 | loader := w.client.Dataset(w.config.DatasetID).Table(w.config.TableID).LoaderFrom(source) 47 | loader.CreateDisposition = bigquery.CreateIfNeeded 48 | loader.WriteDisposition = bigquery.WriteAppend 49 | 50 | job, err := loader.Run(context.Background()) 51 | if err != nil { 52 | return 0, err 53 | } 54 | status, err := job.Wait(context.Background()) 55 | if err != nil { 56 | return 0, err 57 | } 58 | if err := status.Err(); err != nil { 59 | return 0, err 60 | } 61 | 62 | stats := status.Statistics.Details.(*bigquery.LoadStatistics) 63 | log.Printf( 64 | "finished loading data into bigquery (TotalBytesProcessed: %d, InputFileBytes: %d, OutputBytes: %d, OutputRows: %d)", 65 | status.Statistics.TotalBytesProcessed, 66 | stats.InputFileBytes, 67 | stats.OutputBytes, 68 | stats.OutputRows, 69 | ) 70 | return len(b), nil 71 | } 72 | -------------------------------------------------------------------------------- /server/assets/css/orl.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --dark: #2b2a2a !important; 3 | } 4 | 5 | body { 6 | background-color: #212121 !important; 7 | color: rgba(255, 255, 255, 0.8) !important; 8 | } 9 | 10 | .bg-dark { 11 | background-color: #2b2a2a !important; 12 | } 13 | 14 | .patreon { 15 | background-color: #f96854 !important; 16 | color: #fff !important; 17 | } 18 | 19 | .btn-dark { 20 | background-color: #2b2a2a; 21 | border: 1px solid #ff5722 !important; 22 | } 23 | .btn-dark:hover { 24 | background-color: #535353; 25 | } 26 | 27 | .btn { 28 | text-decoration-style: unset; 29 | } 30 | 31 | .table-dark { 32 | color: rgba(255, 255, 255, 0.8) !important; 33 | background-color: #2b2a2a !important; 34 | } 35 | 36 | .table-row:hover { 37 | cursor: pointer; 38 | } 39 | 40 | .table-bordered { 41 | border: 0px; 42 | } 43 | 44 | .table-bordered th, 45 | .table-bordered td { 46 | border: 0; 47 | border-bottom: 1px solid #ff5722 !important; 48 | } 49 | 50 | .table-bordered thead th, 51 | .table-bordered thead td { 52 | border: 0; 53 | border-bottom: 1px solid #ff5722 !important; 54 | } 55 | 56 | .link-white { 57 | color: white; 58 | } 59 | 60 | .link-white:hover { 61 | color: #598cc2; 62 | } 63 | 64 | .link-blue { 65 | color: #598cc2; 66 | } 67 | 68 | .link-blue:hover { 69 | color: #3e71a8; 70 | } 71 | 72 | .list-group-item { 73 | margin-bottom: 0; 74 | background-color: #2b2a2a; 75 | border: 0; 76 | border-bottom: 1px solid #ff5722 !important; 77 | color: rgba(255, 255, 255, 0.8) !important; 78 | } 79 | 80 | .list-group-item:hover, 81 | .list-group-item:focus { 82 | z-index: 1; 83 | text-decoration: none; 84 | color: rgba(255, 255, 255, 0.9); 85 | background-color: #535353; 86 | } 87 | 88 | .breadcrumb { 89 | background-color: #2b2a2a; 90 | } 91 | 92 | .text { 93 | word-break: break-word; 94 | white-space: pre-line; 95 | color: rgba(255, 255, 255, 0.8) !important; 96 | } 97 | 98 | .full-width { 99 | width: 100%; 100 | } 101 | 102 | text { 103 | fill: white; 104 | } 105 | 106 | .domain { 107 | stroke: white; 108 | } 109 | 110 | .footer { 111 | background-color: #2b2a2a; 112 | } 113 | -------------------------------------------------------------------------------- /common/compress.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "strings" 7 | 8 | "github.com/DataDog/zstd" 9 | ) 10 | 11 | // WriteCompressedFile write compressed file 12 | func WriteCompressedFile(path string, data []byte) (*os.File, error) { 13 | cData, err := zstd.Compress(nil, data) 14 | if err != nil { 15 | return nil, err 16 | } 17 | 18 | f, err := os.OpenFile(gzPath(path), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 19 | if err != nil { 20 | return nil, err 21 | } 22 | defer f.Close() 23 | 24 | if _, err := f.Write(cData); err != nil { 25 | return nil, err 26 | } 27 | return f, nil 28 | } 29 | 30 | // ReadCompressedFile read compressed file 31 | func ReadCompressedFile(path string) ([]byte, error) { 32 | f, err := os.Open(gzPath(path)) 33 | if err != nil { 34 | return nil, err 35 | } 36 | defer f.Close() 37 | 38 | data, err := ioutil.ReadAll(f) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | dData, err := zstd.Decompress(nil, data) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return dData, nil 48 | } 49 | 50 | // CompressFile compress an existing file 51 | func CompressFile(path string) (*os.File, error) { 52 | s, err := os.Open(path) 53 | if err != nil { 54 | return nil, err 55 | } 56 | data, err := ioutil.ReadAll(s) 57 | s.Close() 58 | if err != nil { 59 | return nil, err 60 | } 61 | d, err := WriteCompressedFile(path, data) 62 | if err != nil { 63 | return nil, err 64 | } 65 | if err := os.Remove(path); err != nil { 66 | return nil, err 67 | } 68 | return d, nil 69 | } 70 | 71 | // UncompressFile uncompress an existing file 72 | func UncompressFile(path string) (*os.File, error) { 73 | d, err := ReadCompressedFile(path) 74 | if err != nil { 75 | return nil, err 76 | } 77 | f, err := os.OpenFile(strings.Replace(path, ".gz", "", -1), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 78 | if err != nil { 79 | return nil, err 80 | } 81 | defer f.Close() 82 | if _, err := f.Write(d); err != nil { 83 | return nil, err 84 | } 85 | if err := os.Remove(gzPath(path)); err != nil { 86 | return nil, err 87 | } 88 | return f, nil 89 | } 90 | 91 | func gzPath(path string) string { 92 | if path[len(path)-3:] != ".gz" { 93 | path += ".gz" 94 | } 95 | return path 96 | } 97 | -------------------------------------------------------------------------------- /common/avrobuffer.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/actgardner/gogen-avro/container" 9 | ) 10 | 11 | // WriterConstructor ... 12 | type WriterConstructor func(writer io.Writer, codec container.Codec, recordsPerBlock int64) (*container.Writer, error) 13 | 14 | // AvroBuffer ... 15 | type AvroBuffer struct { 16 | WriterConstructor WriterConstructor 17 | Codec container.Codec 18 | RecordsPerBlock int64 19 | BytesPerFile int 20 | Writer io.Writer 21 | avroWriter *container.Writer 22 | buffer bytes.Buffer 23 | recordCount int64 24 | } 25 | 26 | // NewAvroBuffer ... 27 | func NewAvroBuffer(writerConstructor WriterConstructor, writer io.Writer, codec container.Codec, recordsPerBlock int64, bytesPerFile int) (*AvroBuffer, error) { 28 | a := &AvroBuffer{ 29 | WriterConstructor: writerConstructor, 30 | Codec: codec, 31 | RecordsPerBlock: recordsPerBlock, 32 | BytesPerFile: bytesPerFile, 33 | Writer: writer, 34 | } 35 | 36 | if err := a.initAvroWriter(); err != nil { 37 | return nil, err 38 | } 39 | 40 | return a, nil 41 | } 42 | 43 | func (a *AvroBuffer) initAvroWriter() (err error) { 44 | a.avroWriter, err = a.WriterConstructor(&a.buffer, a.Codec, a.RecordsPerBlock) 45 | if err != nil { 46 | return fmt.Errorf("initializing avro writer: %v", err) 47 | } 48 | 49 | return 50 | } 51 | 52 | // WriteRecord ... 53 | func (a *AvroBuffer) WriteRecord(record container.AvroRecord) error { 54 | if err := a.avroWriter.WriteRecord(record); err != nil { 55 | return err 56 | } 57 | 58 | a.recordCount++ 59 | 60 | if a.buffer.Len() >= a.BytesPerFile { 61 | if err := a.Flush(); err != nil { 62 | return fmt.Errorf("writing record: %v", err) 63 | } 64 | } 65 | 66 | return nil 67 | } 68 | 69 | // Flush ... 70 | func (a *AvroBuffer) Flush() error { 71 | if a.recordCount%a.RecordsPerBlock != 0 { 72 | if err := a.avroWriter.Flush(); err != nil { 73 | return fmt.Errorf("flushing avroWriter: %v", err) 74 | } 75 | } 76 | 77 | if _, err := a.Writer.Write(a.buffer.Bytes()); err != nil { 78 | return fmt.Errorf("flushing buffer: %v", err) 79 | } 80 | a.buffer.Reset() 81 | 82 | if err := a.initAvroWriter(); err != nil { 83 | return fmt.Errorf("reinitializing writer: %v", err) 84 | } 85 | 86 | a.recordCount = 0 87 | 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /server/views/directory.jet: -------------------------------------------------------------------------------- 1 | {{extends "layout.jet"}} 2 | {{import "breadcrumbs.jet"}} 3 | {{block body()}} 4 | {{yield breadcrumbs()}} 5 |