21 |
29 |
30 | 22 | 25 | / 26 | slacklog 27 |
28 |
31 |
40 |
32 |
39 |
18 |
50 |
51 |
--------------------------------------------------------------------------------
/internal/slackadapter/conversations_history.go:
--------------------------------------------------------------------------------
1 | package slackadapter
2 |
3 | import (
4 | "context"
5 | "time"
6 |
7 | "github.com/slack-go/slack"
8 | "github.com/vim-jp/slacklog-generator/internal/slacklog"
9 | )
10 |
11 | // ConversationsHistoryParams is optional parameters for ConversationsHistory
12 | type ConversationsHistoryParams struct {
13 | Cursor Cursor `json:"cursor,omitempty"`
14 | Inclusive bool `json:"inclusive,omitempty"`
15 | Latest *time.Time `json:"latest,omitempty"`
16 | Limit int `json:"limit,omitempty"`
17 | Oldest *time.Time `json:"oldest,omitempty"`
18 | }
19 |
20 | // ConversationsHistoryResponse is response for ConversationsHistory
21 | type ConversationsHistoryResponse struct {
22 | Ok bool `json:"ok"`
23 | Messages []*slacklog.Message `json:"messages,omitempty"`
24 | HasMore bool `json:"has_more"`
25 | PinCount int `json:"pin_count"`
26 | ResponseMetadata *NextCursor `json:"response_metadata"`
27 | }
28 |
29 | // ConversationsHistory gets conversation messages in a channel.
30 | func ConversationsHistory(ctx context.Context, token, channel string, params ConversationsHistoryParams) (*ConversationsHistoryResponse, error) {
31 | client := slack.New(token)
32 | res, err := client.GetConversationHistoryContext(ctx, &slack.GetConversationHistoryParameters{
33 | ChannelID: channel,
34 | Cursor: string(params.Cursor),
35 | Limit: params.Limit,
36 | Oldest: Timestamp(params.Oldest),
37 | Latest: Timestamp(params.Latest),
38 | Inclusive: params.Inclusive,
39 | })
40 | if err != nil {
41 | return nil, err
42 | }
43 |
44 | var messages []*slacklog.Message
45 | for _, m := range res.Messages {
46 | messages = append(messages, &slacklog.Message{
47 | Message: m,
48 | })
49 | }
50 |
51 | return &ConversationsHistoryResponse{
52 | Ok: true,
53 | Messages: messages,
54 | HasMore: res.HasMore,
55 | PinCount: res.PinCount,
56 | ResponseMetadata: &NextCursor{
57 | NextCursor: Cursor(res.ResponseMetaData.NextCursor),
58 | },
59 | }, nil
60 | }
61 |
--------------------------------------------------------------------------------
/.github/workflows/update_logs.yml:
--------------------------------------------------------------------------------
1 | name: 'Update logs'
2 | on:
3 | schedule:
4 | - cron: '0 19 * * *' # Every 4:00 am on JST
5 | workflow_dispatch:
6 | inputs:
7 | date:
8 | description: 'Date to update logs (YYYY-MM-DD) (empty to yesterday)'
9 | required: false
10 | default: ''
11 |
12 | jobs:
13 | update-logs:
14 | name: 'Update log-data of https://github.com/vim-jp/slacklog'
15 | runs-on: 'ubuntu-latest'
16 |
17 | steps:
18 | - name: 'Checkout generator'
19 | uses: 'actions/checkout@v2'
20 | with:
21 | path: 'generator'
22 |
23 | - name: 'Checkout log-data'
24 | uses: 'actions/checkout@v2'
25 | with:
26 | repository: 'vim-jp/slacklog'
27 | path: 'data'
28 | ref: 'log-data'
29 | ssh-key: '${{ secrets.SLACKLOG_SSH_KEY }}'
30 |
31 | - name: 'Update logs'
32 | working-directory: './generator'
33 | id: update-logs
34 | env:
35 | SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
36 | run: |
37 | go run . fetch-users --datadir ../data/slacklog_data/
38 | go run . fetch-channels --datadir ../data/slacklog_data/
39 | date='${{ github.event.inputs.date }}'
40 | : "${date:=$(TZ=Asia/Tokyo date --date '1 day ago' --rfc-3339=date)}"
41 | go run . fetch-messages --datadir ../data/slacklog_data/ --date "${date}"
42 |
43 | go run . download-emoji --outdir ../data/emojis/ --emojiJSON ../data/slacklog_data/emoji.json
44 | go run . download-files --indir ../data/slacklog_data/ --outdir ../data/files/
45 |
46 | echo "::set-output name=date::${date}"
47 |
48 | - name: 'Push logs'
49 | working-directory: './data'
50 | run: |
51 | git add --all --intent-to-add --force
52 | if git diff --exit-code --quiet; then
53 | echo 'Nothing to update.'
54 | exit 1 # Make fail to avoid triggering 'Build pages' workflow
55 | fi
56 | git config user.email "slacklog@vim-jp.org"
57 | git config user.name "Slack Log Generator"
58 | git commit --all --message 'Log data for ${{ steps.update-logs.outputs.date }}' --quiet
59 | git push origin log-data --quiet
60 |
--------------------------------------------------------------------------------
/subcmd/fetchusers/fetchusers.go:
--------------------------------------------------------------------------------
1 | package fetchusers
2 |
3 | import (
4 | "context"
5 | "path/filepath"
6 |
7 | cli "github.com/urfave/cli/v2"
8 | "github.com/vim-jp/slacklog-generator/internal/jsonwriter"
9 | "github.com/vim-jp/slacklog-generator/internal/slackadapter"
10 | )
11 |
12 | func run(token, datadir string, excludeArchived, verbose bool) error {
13 | outfile := filepath.Join(datadir, "users.json")
14 | fw, err := jsonwriter.CreateFile(outfile, true)
15 | if err != nil {
16 | return err
17 | }
18 | err = slackadapter.IterateCursor(context.Background(),
19 | slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) {
20 | users, err := slackadapter.Users(ctx, token)
21 | if err != nil {
22 | return "", err
23 | }
24 | for _, u := range users {
25 | err := fw.Write(u)
26 | if err != nil {
27 | return "", err
28 | }
29 | }
30 | return "", nil
31 | }))
32 | if err != nil {
33 | // ロールバック相当が好ましいが今はまだその時期ではない
34 | fw.Close()
35 | return err
36 | }
37 | if err := fw.Close(); err != nil {
38 | return err
39 | }
40 |
41 | return nil
42 | }
43 |
44 | // NewCLICommand creates a cli.Command, which provides "fetch-users"
45 | // sub-command.
46 | func NewCLICommand() *cli.Command {
47 | var (
48 | token string
49 | datadir string
50 | excludeArchived bool
51 | verbose bool
52 | )
53 | return &cli.Command{
54 | Name: "fetch-users",
55 | Usage: "fetch users in the workspace",
56 | Action: func(c *cli.Context) error {
57 | return run(token, datadir, excludeArchived, verbose)
58 | },
59 | Flags: []cli.Flag{
60 | &cli.StringFlag{
61 | Name: "token",
62 | Usage: "slack token",
63 | EnvVars: []string{"SLACK_TOKEN"},
64 | Destination: &token,
65 | },
66 | &cli.StringFlag{
67 | Name: "datadir",
68 | Usage: "directory to load/save data",
69 | Value: "_logdata",
70 | Destination: &datadir,
71 | },
72 | &cli.BoolFlag{
73 | Name: "exclude-archived",
74 | Usage: "exclude archived channesls",
75 | Destination: &excludeArchived,
76 | },
77 | &cli.BoolFlag{
78 | Name: "verbose",
79 | Usage: "verbose log",
80 | Destination: &verbose,
81 | },
82 | },
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/.github/workflows/build_pages.yml:
--------------------------------------------------------------------------------
1 | name: 'Build pages'
2 | on:
3 | push:
4 | branches:
5 | - '**'
6 | workflow_run:
7 | workflows: ['Update logs']
8 | types: ['completed']
9 |
10 | jobs:
11 | build-pages:
12 | name: 'Generate htdocs and Update https://vim-jp.org/slacklog/'
13 | if: "${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}"
14 | runs-on: 'ubuntu-latest'
15 |
16 | steps:
17 | - name: 'Checkout generator'
18 | uses: 'actions/checkout@v2'
19 | with:
20 | path: 'generator'
21 |
22 | - name: 'Checkout log-data'
23 | uses: 'actions/checkout@v2'
24 | with:
25 | repository: 'vim-jp/slacklog'
26 | path: 'data'
27 | ref: 'log-data'
28 |
29 | - name: 'Checkout gh-pages'
30 | uses: 'actions/checkout@v2'
31 | with:
32 | repository: 'vim-jp/slacklog'
33 | path: 'pages'
34 | ref: 'gh-pages'
35 | ssh-key: '${{ secrets.SLACKLOG_SSH_KEY }}'
36 |
37 | - name: 'Generate htdocs'
38 | run: |
39 | cp -r data/files/ data/emojis/ generator/static/* pages/
40 | rm -fr data/files/ data/emojis/
41 | cd generator
42 | BASEURL=/slacklog go run . generate-html --filesdir ../pages/files/ --indir ../data/slacklog_data/ --outdir ../pages/
43 | go run . build-index --datadir ../data/slacklog_data --outdir ../pages/index
44 | # create finger print
45 | cd ../pages
46 | find . -type d -name '.git' -prune -o -type f -print0 | xargs -0 md5sum > ../files.txt
47 |
48 | - name: 'Save fingerprint'
49 | uses: actions/upload-artifact@v2
50 | with:
51 | name: fingerprint
52 | path: files.txt
53 |
54 | - name: 'Update https://vim-jp.org/slacklog/'
55 | if: github.ref == 'refs/heads/master'
56 | working-directory: './pages'
57 | run: |
58 | git add --all --intent-to-add --force
59 | if git diff --exit-code --quiet; then
60 | echo 'Nothing to update.'
61 | exit 0
62 | fi
63 | git config user.email "slacklog@vim-jp.org"
64 | git config user.name "Slack Log Generator"
65 | git commit --all --message 'Update pages' --quiet
66 | git push origin gh-pages --quiet
67 |
--------------------------------------------------------------------------------
/internal/slacklog/channel.go:
--------------------------------------------------------------------------------
1 | package slacklog
2 |
3 | import (
4 | "sort"
5 |
6 | "github.com/slack-go/slack"
7 | )
8 |
9 | // ChannelTable : チャンネルデータを保持する。
10 | // channelsもchannelMapも保持するチャンネルデータは同じで、channelMapはチャンネ
11 | // ルIDをキーとするmapとなっている。
12 | // ユースケースに応じてchannelsとchannelMapは使い分ける。
13 | type ChannelTable struct {
14 | Channels []Channel
15 | ChannelMap map[string]*Channel
16 | }
17 |
18 | // NewChannelTable : pathに指定したJSON形式のチャンネルデータを読み込み、
19 | // ChannelTable を生成する。
20 | // whitelistに指定したチャンネル名のみを読み込む。
21 | func NewChannelTable(path string, whitelist []string) (*ChannelTable, error) {
22 | var channels []Channel
23 | if err := ReadFileAsJSON(path, true, &channels); err != nil {
24 | return nil, err
25 | }
26 | channels = FilterChannel(channels, whitelist)
27 | sort.Slice(channels, func(i, j int) bool {
28 | return channels[i].Name < channels[j].Name
29 | })
30 | channelMap := make(map[string]*Channel, len(channels))
31 | for i, ch := range channels {
32 | channelMap[ch.ID] = &channels[i]
33 | }
34 | return &ChannelTable{
35 | Channels: channels,
36 | ChannelMap: channelMap,
37 | }, nil
38 | }
39 |
40 | // FilterChannel : whitelistに指定したチャンネル名に該当するチャンネルのみを返
41 | // す。
42 | // whitelistに'*'が含まれる場合はchannelをそのまま返す。
43 | func FilterChannel(channels []Channel, whitelist []string) []Channel {
44 | if len(whitelist) == 0 {
45 | return []Channel{}
46 | }
47 | allowed := map[string]struct{}{}
48 | for _, s := range whitelist {
49 | if s == "*" {
50 | return channels
51 | }
52 | allowed[s] = struct{}{}
53 | }
54 | newChannels := make([]Channel, 0, len(whitelist))
55 | for _, ch := range channels {
56 | _, ok := allowed[ch.Name]
57 | if ok {
58 | newChannels = append(newChannels, ch)
59 | }
60 | }
61 | return newChannels
62 | }
63 |
64 | // SortChannel sorts []Channel by name. It modify original slice.
65 | func SortChannel(channels []Channel) {
66 | sort.SliceStable(channels, func(i, j int) bool {
67 | return channels[i].Name < channels[j].Name
68 | })
69 | }
70 |
71 | // Channel represents channel object in Slack.
72 | type Channel struct {
73 | slack.Channel
74 |
75 | Pins []ChannelPin `json:"pins"`
76 | }
77 |
78 | // ChannelPin represents a pinned message for a channel.
79 | type ChannelPin struct {
80 | ID string `json:"id"`
81 | Typ string `json:"type"`
82 | Created int64 `json:"created"`
83 | User string `json:"user"`
84 | Owner string `json:"owner"`
85 | }
86 |
--------------------------------------------------------------------------------
/internal/slacklog/slack.go:
--------------------------------------------------------------------------------
1 | package slacklog
2 |
3 | import (
4 | "net/url"
5 | "path"
6 | "path/filepath"
7 | "strings"
8 | "unicode/utf8"
9 |
10 | "github.com/slack-go/slack"
11 | )
12 |
13 | // HostBySlack checks a file is hosted by slack or not.
14 | func HostBySlack(f slack.File) bool {
15 | return strings.HasPrefix(f.URLPrivate, "https://files.slack.com/")
16 | }
17 |
18 | // LocalName returns name of local downloaded file.
19 | func LocalName(f slack.File, url, suffix string) string {
20 | ext := filepath.Ext(url)
21 | nameExt := filepath.Ext(f.Name)
22 | name := f.Name[:len(f.Name)-len(nameExt)]
23 | if ext == "" {
24 | ext = nameExt
25 | if ext == "" {
26 | ext = FiletypeToExtension[f.Filetype]
27 | }
28 | }
29 | name = truncateName(name, 200)
30 | return RegulateFilename(name + suffix + ext)
31 | }
32 |
33 | func truncateName(name string, size int) string {
34 | if len(name) < size {
35 | return name
36 | }
37 | name = name[:size]
38 |
39 | if !utf8.ValidString(name) {
40 | v := make([]rune, 0, len(name))
41 | for i, r := range name {
42 | if r == utf8.RuneError {
43 | _, size := utf8.DecodeRuneInString(name[i:])
44 | if size == 1 {
45 | continue
46 | }
47 | }
48 | v = append(v, r)
49 | }
50 | name = string(v)
51 | }
52 | return name
53 | }
54 |
55 | // LocalPath returns path of local downloaded file.
56 | func LocalPath(f slack.File) string {
57 | return path.Join(f.ID, url.PathEscape(LocalName(f, f.URLPrivate, "")))
58 | }
59 |
60 | // TopLevelMimetype extracts top level type from MIME Type.
61 | func TopLevelMimetype(f slack.File) string {
62 | i := strings.Index(f.Mimetype, "/")
63 | if i < 0 {
64 | return ""
65 | }
66 | return f.Mimetype[:i]
67 | }
68 |
69 | // ThumbImagePath returns path of thumbnail image file.
70 | func ThumbImagePath(f slack.File) string {
71 | if f.Thumb1024 == "" {
72 | return LocalPath(f)
73 | }
74 | return path.Join(f.ID, url.PathEscape(LocalName(f, f.Thumb1024, "_1024")))
75 | }
76 |
77 | // ThumbImageWidth returns width of thumbnail image.
78 | func ThumbImageWidth(f slack.File) int {
79 | if f.Thumb1024 != "" {
80 | return f.Thumb1024W
81 | }
82 | return f.OriginalW
83 | }
84 |
85 | // ThumbImageHeight returns height of thumbnail image.
86 | func ThumbImageHeight(f slack.File) int {
87 | if f.Thumb1024 != "" {
88 | return f.Thumb1024H
89 | }
90 | return f.OriginalH
91 | }
92 |
93 | // ThumbVideoPath returns local path of thumbnail for the video.
94 | func ThumbVideoPath(f slack.File) string {
95 | return path.Join(f.ID, url.PathEscape(LocalName(f, f.ThumbVideo, "_video")))
96 | }
97 |
--------------------------------------------------------------------------------
/subcmd/fetchchannels/fetchchannels.go:
--------------------------------------------------------------------------------
1 | package fetchchannels
2 |
3 | import (
4 | "context"
5 | "path/filepath"
6 |
7 | cli "github.com/urfave/cli/v2"
8 | "github.com/vim-jp/slacklog-generator/internal/jsonwriter"
9 | "github.com/vim-jp/slacklog-generator/internal/slackadapter"
10 | )
11 |
12 | func run(token, datadir string, excludeArchived, verbose bool) error {
13 | outfile := filepath.Join(datadir, "channels.json")
14 | fw, err := jsonwriter.CreateFile(outfile, true)
15 | if err != nil {
16 | return err
17 | }
18 | err = slackadapter.IterateCursor(context.Background(),
19 | slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) {
20 | r, err := slackadapter.Conversations(ctx, token, slackadapter.ConversationsParams{
21 | Cursor: c,
22 | Limit: 100,
23 | ExcludeArchived: excludeArchived,
24 | Types: []string{"public_channel"},
25 | })
26 | if err != nil {
27 | return "", err
28 | }
29 | for _, c := range r.Channels {
30 | err := fw.Write(c)
31 | if err != nil {
32 | return "", err
33 | }
34 | }
35 | if m := r.ResponseMetadata; m != nil {
36 | return m.NextCursor, nil
37 | }
38 | return "", nil
39 | }))
40 | if err != nil {
41 | // ロールバック相当が好ましいが今はまだその時期ではない
42 | fw.Close()
43 | return err
44 | }
45 | if err := fw.Close(); err != nil {
46 | return err
47 | }
48 |
49 | return nil
50 | }
51 |
52 | // NewCLICommand creates a cli.Command, which provides "fetch-channels"
53 | // sub-command.
54 | func NewCLICommand() *cli.Command {
55 | var (
56 | token string
57 | datadir string
58 | excludeArchived bool
59 | verbose bool
60 | )
61 | return &cli.Command{
62 | Name: "fetch-channels",
63 | Usage: "fetch channels in the workspace",
64 | Action: func(c *cli.Context) error {
65 | return run(token, datadir, excludeArchived, verbose)
66 | },
67 | Flags: []cli.Flag{
68 | &cli.StringFlag{
69 | Name: "token",
70 | Usage: "slack token",
71 | EnvVars: []string{"SLACK_TOKEN"},
72 | Destination: &token,
73 | },
74 | &cli.StringFlag{
75 | Name: "datadir",
76 | Usage: "directory to load/save data",
77 | Value: "_logdata",
78 | Destination: &datadir,
79 | },
80 | &cli.BoolFlag{
81 | Name: "exclude-archived",
82 | Usage: "exclude archived channesls",
83 | Destination: &excludeArchived,
84 | },
85 | &cli.BoolFlag{
86 | Name: "verbose",
87 | Usage: "verbose log",
88 | Destination: &verbose,
89 | },
90 | },
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/internal/slacklog/downloader_test.go:
--------------------------------------------------------------------------------
1 | package slacklog
2 |
3 | import (
4 | "encoding/json"
5 | "io/ioutil"
6 | "net/http"
7 | "net/http/httptest"
8 | "os"
9 | "path/filepath"
10 | "testing"
11 | )
12 |
13 | func TestDownloader(t *testing.T) {
14 | tmpPath := createTmpDir(t)
15 | defer t.Cleanup(func() {
16 | cleanupTmpDir(t, tmpPath)
17 | })
18 |
19 | ts := httptest.NewServer(http.FileServer(http.Dir("testdata/downloader")))
20 | defer ts.Close()
21 |
22 | fileInfos, err := ioutil.ReadDir("testdata/downloader")
23 | if err != nil {
24 | t.Fatal(err)
25 | }
26 |
27 | d := NewDownloader("dummyToken")
28 |
29 | for _, fileInfo := range fileInfos {
30 | url := ts.URL + "/" + fileInfo.Name()
31 | path := filepath.Join(tmpPath, fileInfo.Name())
32 | d.QueueDownloadRequest(
33 | url,
34 | path,
35 | false,
36 | )
37 | }
38 | d.CloseQueue()
39 | err = d.Wait()
40 | if err != nil {
41 | t.Fatal(err)
42 | }
43 |
44 | err = dirDiff(t, "testdata/downloader", tmpPath)
45 | if err != nil {
46 | t.Fatal(err)
47 | }
48 | }
49 |
50 | func TestDownloader_usingToken(t *testing.T) {
51 | tmpPath := createTmpDir(t)
52 | defer t.Cleanup(func() {
53 | cleanupTmpDir(t, tmpPath)
54 | })
55 |
56 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
57 | t.Helper()
58 | res := map[string]interface{}{}
59 | res["token"] = r.Header.Get("Authorization")[7:]
60 | res["path"] = r.URL.Path
61 |
62 | err := json.NewEncoder(w).Encode(res)
63 | if err != nil {
64 | t.Fatal(err)
65 | }
66 | }))
67 | defer ts.Close()
68 |
69 | testToken := "dummyToken"
70 | d := NewDownloader(testToken)
71 |
72 | testFileName := "test.json"
73 | url := ts.URL + "/" + testFileName
74 | path := filepath.Join(tmpPath, testFileName)
75 | d.QueueDownloadRequest(url, path, true)
76 | d.CloseQueue()
77 |
78 | err := d.Wait()
79 | if err != nil {
80 | t.Fatal(err)
81 | }
82 |
83 | f, err := os.Open(path)
84 | if err != nil {
85 | t.Fatal(err)
86 | }
87 | var got map[string]string
88 | err = json.NewDecoder(f).Decode(&got)
89 | f.Close()
90 | if err != nil {
91 | t.Fatal(err)
92 | }
93 | gotPath, ok := got["path"]
94 | if !ok {
95 | t.Fatal("not found got[\"path\"]")
96 | }
97 | if gotPath != "/"+testFileName {
98 | t.Fatalf("want %s, but got %s", "/"+testFileName, gotPath)
99 | }
100 | gotToken, ok := got["token"]
101 | if !ok {
102 | t.Fatal("not found got[\"token\"]")
103 | }
104 | if gotToken != testToken {
105 | t.Fatalf("want %s, but got %s", testToken, gotToken)
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/scripts/site_diff.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -eu
4 |
5 | force=0
6 | clean=0
7 | update=0
8 | outdiff=""
9 |
10 | while getopts fcuo: OPT ; do
11 | case $OPT in
12 | f) force=1 ;;
13 | c) clean=1 ;;
14 | u) update=1 ;;
15 | o) outdiff="$OPTARG" ;;
16 | esac
17 | done
18 |
19 | cd "$(dirname "$0")/.." || exit "$?"
20 |
21 | outrootdir=tmp/_site_diff
22 | current_pages=${outrootdir}/current
23 | cmd=${outrootdir}/slacklog-tools
24 |
25 | build_tool() {
26 | if [ -f main.go ] ; then
27 | go build -o ${cmd} . 1>&2
28 | else
29 | cd scripts
30 | go build -o ../${cmd} ./main.go 1>&2
31 | cd ..
32 | fi
33 | }
34 |
35 | # generate-html サブコマンドと build.sh を実行して指定ディレクトリに出力する
36 | generate_site() {
37 | id=$1 ; shift
38 | outdir=${outrootdir}/${id}
39 | echo "build to: ${outdir}" 1>&2
40 | rm -rf ${outdir}
41 | build_tool
42 | tmpldir=slacklog_template/
43 | if [ -d templates ] ; then
44 | tmpldir=templates/
45 | fi
46 | logdir=slacklog_data/
47 | if [ -d _logdata ] ; then
48 | logdir=_logdata/slacklog_data/
49 | fi
50 | filesdir=files/
51 | if [ -d _logdata ] ; then
52 | filesdir=_logdata/files/
53 | fi
54 | ${cmd} generate-html --templatedir ${tmpldir} --filesdir ${filesdir} --indir ${logdir} --outdir ${outdir}/ > ${outdir}.generate-html.log 2>&1 || ( cat ${outdir}.generate-html.log && exit 1 )
55 | rm -f ${cmd}
56 | ./scripts/build.sh -o $outdir > ${outdir}.build.log 2>&1 || ( cat ${outdir}.build.log && exit 1 )
57 | }
58 |
59 | if [ $clean -ne 0 ] ; then
60 | echo "clean up $outrootdir" 1>&2
61 | rm -rf $outrootdir
62 | exit 0
63 | fi
64 |
65 | generate_site "current"
66 |
67 | if [ $update -ne 0 ] ; then
68 | echo "catching up origin/master" 1>&2
69 | git fetch -q origin master
70 | fi
71 |
72 | base_commit=$(git show-branch --merge-base origin/master HEAD)
73 | base_pages=${outrootdir}/${base_commit}
74 |
75 | if [ $force -ne 0 -o ! \( -d $base_pages \) ] ; then
76 | echo "base commit: $base_commit" 1>&2
77 |
78 | # 現在の変更とHEADの commit ID を退避する
79 | has_changes=$(git status -s -uno | wc -l)
80 | if [ $has_changes -ne 0 ] ; then
81 | git stash push -q
82 | fi
83 | current_commit=$(git rev-parse HEAD)
84 |
85 | # merge-base に巻き戻し generate-html を実行する
86 | git reset -q --hard ${base_commit}
87 | echo "move to base: $(git rev-parse HEAD)" 1>&2
88 | generate_site ${base_commit}
89 |
90 | # 退避したHEADと変更を復帰する
91 | git reset -q --hard ${current_commit}
92 | if [ $has_changes -ne 0 ] ; then
93 | git stash pop -q
94 | fi
95 |
96 | echo "return to current: $(git rev-parse HEAD)" 1>&2
97 | fi
98 |
99 | # 差分を出力
100 | if [ x"$outdiff" = x ] ; then
101 | echo "" 1>&2
102 | diff -uNrw -x sitemap.xml ${base_pages} ${current_pages} || true
103 | else
104 | diff -uNrw -x sitemap.xml ${base_pages} ${current_pages} > "$outdiff" || (
105 | echo "" 1>&2
106 | echo "have some diff, please check $outdiff" 1>&2
107 | )
108 | fi
109 |
--------------------------------------------------------------------------------
/subcmd/download_emoji.go:
--------------------------------------------------------------------------------
1 | /*
2 | リファクタリング中
3 | 処理をslacklog packageに移動していく。
4 | 一旦、必要な処理はすべてslacklog packageから一時的にエクスポートするか、このファ
5 | イル内で定義している。
6 | */
7 |
8 | package subcmd
9 |
10 | import (
11 | "encoding/json"
12 | "fmt"
13 | "os"
14 | "path/filepath"
15 | "strings"
16 |
17 | "github.com/slack-go/slack"
18 | cli "github.com/urfave/cli/v2"
19 | "github.com/vim-jp/slacklog-generator/internal/slacklog"
20 | )
21 |
22 | // DownloadEmojiCommand provides "download-emoji". It downloads and save emoji
23 | // image files.
24 | var DownloadEmojiCommand = &cli.Command{
25 | Name: "download-emoji",
26 | Usage: "download customized emoji from slack",
27 | Action: downloadEmoji,
28 | Flags: []cli.Flag{
29 | &cli.StringFlag{
30 | Name: "outdir",
31 | Usage: "emojis download target dir",
32 | Value: filepath.Join("_logdata", "emojis"),
33 | },
34 | &cli.StringFlag{
35 | Name: "emojiJSON",
36 | Usage: "emoji json path",
37 | Value: filepath.Join("_logdata", "emojis", "emoji.json"),
38 | },
39 | },
40 | }
41 |
42 | // downloadEmoji downloads and save emoji image files.
43 | func downloadEmoji(c *cli.Context) error {
44 | slackToken := os.Getenv("SLACK_TOKEN")
45 | if slackToken == "" {
46 | return fmt.Errorf("$SLACK_TOKEN required")
47 | }
48 |
49 | emojisDir := filepath.Clean(c.String("outdir"))
50 | emojiJSONPath := filepath.Clean(c.String("emojiJSON"))
51 |
52 | api := slack.New(slackToken)
53 |
54 | emojis, err := api.GetEmoji()
55 | if err != nil {
56 | return err
57 | }
58 |
59 | d := slacklog.NewDownloader(slackToken)
60 |
61 | go generateEmojiFileTargets(d, emojis, emojisDir)
62 |
63 | err = outputSummary(emojis, emojiJSONPath)
64 | if err != nil {
65 | return err
66 | }
67 |
68 | err = d.Wait()
69 | if err != nil {
70 | return err
71 | }
72 | return nil
73 | }
74 |
75 | func generateEmojiFileTargets(d *slacklog.Downloader, emojis map[string]string, outputDir string) {
76 | defer d.CloseQueue()
77 | err := os.MkdirAll(outputDir, 0777)
78 | if err != nil {
79 | fmt.Fprintf(os.Stderr, "failed to create %s: %s", outputDir, err)
80 | return
81 | }
82 |
83 | for name, url := range emojis {
84 | if strings.HasPrefix(url, "alias:") {
85 | continue
86 | }
87 | ext := filepath.Ext(url)
88 | path := filepath.Join(outputDir, name+ext)
89 | d.QueueDownloadRequest(
90 | url,
91 | path,
92 | false,
93 | )
94 | }
95 | }
96 |
97 | func outputSummary(emojis map[string]string, path string) error {
98 | exts := make(map[string]string, len(emojis))
99 | for name, url := range emojis {
100 | if strings.HasPrefix(url, "alias:") {
101 | exts[name] = url
102 | continue
103 | }
104 | ext := filepath.Ext(url)
105 | exts[name] = ext
106 | }
107 |
108 | f, err := os.Create(path)
109 | if err != nil {
110 | return err
111 | }
112 | defer f.Close()
113 | err = json.NewEncoder(f).Encode(exts)
114 | if err != nil {
115 | return err
116 | }
117 | return nil
118 | }
119 |
--------------------------------------------------------------------------------
/templates/index.tmpl:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
19 |
20 |
41 |
42 |
43 |
48 |
49 |
18 |
68 |
69 |
--------------------------------------------------------------------------------
/templates/channel_per_month/attachment.tmpl:
--------------------------------------------------------------------------------
1 | {{- range .Attachments }}
2 | {{- if eq .ServiceName "GitHub" }}
3 |
19 |
20 |
59 |
60 |
61 |
66 |
67 |
21 |
29 |
30 | 22 | 25 | / 26 | slacklog 27 |
28 |
31 |
58 |
32 |
46 |
33 |
41 |
42 | vim-jp/slacklog について
43 |
44 |
45 |
47 |
57 | 48 | Search 49 |
50 |Channels
51 | 56 |
4 |
13 |
5 | 
6 |
7 |
8 |
9 | {{ html .Title }}
10 |
11 |
12 |
14 |
16 | {{- else if eq .ServiceName "twitter" }}
17 | {{ attachmentText . }}
15 |
18 |
25 |
19 | 
20 |
21 |
22 | {{ .AuthorName }} {{ .AuthorSubname }}
23 |
24 |
26 |
28 | {{ attachmentText . }}
27 |
29 |
36 | {{- if .VideoHTML }}
37 |
30 | 
31 |
32 |
33 | {{ html .Footer }}
34 |
35 |
38 | {{ .VideoHTML }}
39 |
40 | {{- end }}
41 | {{- else if eq .ServiceName "Gyazo" }}
42 |
43 |
50 |
44 |
45 |
46 | {{ .Fallback }}
47 |
48 |
49 |
51 |
55 | {{- else if or .Title .Text }}
56 |
52 | 
53 |
54 |
57 | {{- if and .Title .TitleLink }}
58 | {{- if and .ServiceIcon .ServiceName }}
59 |
72 | {{- if .Text }}
73 |
60 |
61 |
62 | {{- end }}
63 |
64 |
65 | {{ html .Title }}
66 |
67 |
68 | {{- else if .Title }}
69 | {{ html .Title }}
70 | {{- end }}
71 |
74 |
76 | {{- if isSlackMessage .FromURL}}
77 |
78 | slacklog
79 |
80 | {{- end}}
81 | {{- end }}
82 |
83 | {{- if .ThumbURL }}
84 | {{ attachmentText . }}
75 |
85 |
89 | {{- end }}
90 | {{- end }}
91 | {{- end }}
92 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
2 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6 | github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
7 | github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
8 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
9 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
10 | github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
11 | github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
12 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
13 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
14 | github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
15 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
16 | github.com/kyokomi/emoji v2.2.2+incompatible h1:gaQFbK2+uSxOR4iGZprJAbpmtqTrHhSdgOyIMD6Oidc=
17 | github.com/kyokomi/emoji v2.2.2+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
18 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
19 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
20 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
21 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
22 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
23 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
24 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
25 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
26 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
27 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
28 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
29 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
30 | github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
31 | github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
32 | github.com/vim-jp/slacklog-slack v0.0.0-20200516060239-575febb4155b h1:TpL0ddVABxueZruJj/prrtILmnnipggBrAoEQelmWuI=
33 | github.com/vim-jp/slacklog-slack v0.0.0-20200516060239-575febb4155b/go.mod h1:sGRjv3w+ERAUMMMbldHObQPBcNSyVB7KLKYfnwUFBfw=
34 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
35 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
36 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
37 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
38 |
--------------------------------------------------------------------------------
/subcmd/download_files.go:
--------------------------------------------------------------------------------
1 | /*
2 | リファクタリング中
3 | 処理をslacklog packageに移動していく。
4 | 一旦、必要な処理はすべてslacklog packageから一時的にエクスポートするか、このファ
5 | イル内で定義している。
6 | */
7 |
8 | package subcmd
9 |
10 | import (
11 | "fmt"
12 | "os"
13 | "path/filepath"
14 |
15 | "github.com/slack-go/slack"
16 | cli "github.com/urfave/cli/v2"
17 | "github.com/vim-jp/slacklog-generator/internal/slacklog"
18 | )
19 |
20 | // DownloadFilesCommand provides "downloads-files" sub-command, it downloads
21 | // and saves files which attached to message.
22 | var DownloadFilesCommand = &cli.Command{
23 | Name: "download-files",
24 | Usage: "download files from slack.com",
25 | Action: downloadFiles,
26 | Flags: []cli.Flag{
27 | &cli.StringFlag{
28 | Name: "indir",
29 | Usage: "slacklog_data dir",
30 | Value: filepath.Join("_logdata", "slacklog_data"),
31 | },
32 | &cli.StringFlag{
33 | Name: "outdir",
34 | Usage: "files download target dir",
35 | Value: filepath.Join("_logdata", "files"),
36 | },
37 | },
38 | }
39 |
40 | // downloadFiles downloads and saves files which attached to message.
41 | func downloadFiles(c *cli.Context) error {
42 | slackToken := os.Getenv("SLACK_TOKEN")
43 | if slackToken == "" {
44 | return fmt.Errorf("$SLACK_TOKEN required")
45 | }
46 |
47 | logDir := filepath.Clean(c.String("indir"))
48 | filesDir := filepath.Clean(c.String("outdir"))
49 |
50 | s, err := slacklog.NewLogStore(logDir, &slacklog.Config{Channels: []string{"*"}})
51 | if err != nil {
52 | return err
53 | }
54 |
55 | d := slacklog.NewDownloader(slackToken)
56 |
57 | go generateMessageFileTargets(d, s, filesDir)
58 |
59 | err = d.Wait()
60 | if err != nil {
61 | return err
62 | }
63 | return nil
64 | }
65 |
66 | func urlAndSuffixes(f slack.File) map[string]string {
67 | return map[string]string{
68 | f.URLPrivate: "",
69 | f.Thumb64: "_64",
70 | f.Thumb80: "_80",
71 | f.Thumb160: "_160",
72 | f.Thumb360: "_360",
73 | f.Thumb480: "_480",
74 | f.Thumb720: "_720",
75 | f.Thumb800: "_800",
76 | f.Thumb960: "_960",
77 | f.Thumb1024: "_1024",
78 | f.Thumb360Gif: "_360_gif",
79 | f.Thumb480Gif: "_480_gif",
80 | f.DeanimateGif: "_deanimate_gif",
81 | f.ThumbVideo: "_video",
82 | }
83 | }
84 |
85 | func generateMessageFileTargets(d *slacklog.Downloader, s *slacklog.LogStore, outputDir string) {
86 | defer d.CloseQueue()
87 | channels := s.GetChannels()
88 | for _, channel := range channels {
89 | msgs, err := s.GetAllMessages(channel.ID)
90 | if err != nil {
91 | fmt.Fprintf(os.Stderr, "failed to get messages on %s channel: %s", channel.Name, err)
92 | return
93 | }
94 |
95 | for _, msg := range msgs {
96 | for _, f := range msg.Files {
97 | if !slacklog.HostBySlack(f) {
98 | continue
99 | }
100 | // avoid github's file size limit
101 | if f.Size >= 104857600 {
102 | continue
103 | }
104 |
105 | targetDir := filepath.Join(outputDir, f.ID)
106 | err := os.MkdirAll(targetDir, 0777)
107 | if err != nil {
108 | fmt.Fprintf(os.Stderr, "failed to create %s directory: %s", targetDir, err)
109 | return
110 | }
111 |
112 | for url, suffix := range urlAndSuffixes(f) {
113 | if url == "" {
114 | continue
115 | }
116 | d.QueueDownloadRequest(
117 | url,
118 | filepath.Join(targetDir, slacklog.LocalName(f, url, suffix)),
119 | true,
120 | )
121 | }
122 | }
123 | }
124 | }
125 | }
126 |
--------------------------------------------------------------------------------
/.github/workflows/reviewdog.yml:
--------------------------------------------------------------------------------
1 | name: reviewdog
2 | on: [pull_request]
3 | jobs:
4 | # NOTE: golangci-lint doesn't report multiple errors on the same line from
5 | # different linters and just report one of the errors?
6 |
7 | golangci-lint:
8 | name: runner / golangci-lint (pre-build docker image)
9 | runs-on: ubuntu-latest
10 | steps:
11 | - name: Check out code into the Go module directory
12 | uses: actions/checkout@v2
13 | - name: golangci-lint
14 | # uses: ./ # Build with Dockerfile
15 | uses: docker://reviewdog/action-golangci-lint:v1 # Pre-built image
16 | with:
17 | github_token: ${{ secrets.github_token }}
18 |
19 | golangci-lint-all-in-one:
20 | name: runner / golangci-lint-all-in-one
21 | runs-on: ubuntu-latest
22 | steps:
23 | - name: Check out code into the Go module directory
24 | uses: actions/checkout@v2
25 | - name: golangci-lint (All-In-One config)
26 | uses: docker://reviewdog/action-golangci-lint:v1
27 | with:
28 | github_token: ${{ secrets.github_token }}
29 |
30 | govet:
31 | name: runner / govet
32 | runs-on: ubuntu-latest
33 | steps:
34 | - name: Check out code into the Go module directory
35 | uses: actions/checkout@v2
36 | - name: govet
37 | uses: docker://reviewdog/action-golangci-lint:v1
38 | with:
39 | github_token: ${{ secrets.github_token }}
40 | tool_name: govet
41 |
42 | staticcheck:
43 | name: runner / staticcheck
44 | runs-on: ubuntu-latest
45 | steps:
46 | - name: Check out code into the Go module directory
47 | uses: actions/checkout@v2
48 | - name: staticcheck
49 | uses: docker://reviewdog/action-golangci-lint:v1
50 | with:
51 | github_token: ${{ secrets.github_token }}
52 | tool_name: staticcheck
53 |
54 | golint:
55 | name: runner / golint
56 | runs-on: ubuntu-latest
57 | steps:
58 | - name: Check out code into the Go module directory
59 | uses: actions/checkout@v2
60 | - name: golint
61 | uses: docker://reviewdog/action-golangci-lint:v1
62 | with:
63 | github_token: ${{ secrets.github_token }}
64 | tool_name: golint
65 | level: warning
66 |
67 | errcheck:
68 | name: runner / errcheck
69 | runs-on: ubuntu-latest
70 | steps:
71 | - name: Check out code into the Go module directory
72 | uses: actions/checkout@v2
73 | - name: errcheck
74 | uses: docker://reviewdog/action-golangci-lint:v1
75 | with:
76 | github_token: ${{ secrets.github_token }}
77 | tool_name: errcheck
78 | level: warning
79 |
80 | misspell:
81 | name: runner / misspell
82 | runs-on: ubuntu-latest
83 | steps:
84 | - name: Check out code into the Go module directory
85 | uses: actions/checkout@v2
86 | - name: misspell
87 | uses: docker://reviewdog/action-golangci-lint:v1
88 | with:
89 | github_token: ${{ secrets.github_token }}
90 | tool_name: misspell
91 | level: info
92 |
93 | shellcheck:
94 | name: runner / shellcheck
95 | runs-on: ubuntu-latest
96 | steps:
97 | - name: Check out code into the Go module directory
98 | uses: actions/checkout@v2
99 | - name: shellcheck
100 | uses: reviewdog/action-shellcheck@v1
101 | with:
102 | github_token: ${{ secrets.github_token }}
103 | level: info
104 | path: "scripts"
105 | pattern: "*.sh"
106 |
--------------------------------------------------------------------------------
/internal/slacklog/converter.go:
--------------------------------------------------------------------------------
1 | package slacklog
2 |
3 | import (
4 | "html"
5 | "net/url"
6 | "os"
7 | "regexp"
8 | "strings"
9 |
10 | "github.com/kyokomi/emoji"
11 | )
12 |
13 | // TextConverter : markdown形式のテキストをHTMLに変換するための構造体。
14 | type TextConverter struct {
15 | // key: emoji name
16 | // value: emoji URL
17 | emojis map[string]string
18 | // key: user ID
19 | // value: display name
20 | users map[string]string
21 | re regexps
22 | // baseURL is root path for public site, configured by `BASEURL` environment variable.
23 | baseURL string
24 | }
25 |
26 | // NewTextConverter : TextConverter を生成する
27 | func NewTextConverter(users, emojis map[string]string) *TextConverter {
28 | re := regexps{}
29 | // TODO tokenize/parse message.Text
30 | re.linkWithTitle = regexp.MustCompile(`<(https?://[^>]+?)\|(.+?)>`)
31 | re.link = regexp.MustCompile(`<(https?://[^>]+?)>`)
32 | // go regexp does not support back reference
33 | re.code = regexp.MustCompile("`{3}|｀{3}")
34 | re.codeShort = regexp.MustCompile("[`｀]([^`]+?)[`｀]")
35 | re.del = regexp.MustCompile(`~([^~]+?)~`)
36 | re.mention = regexp.MustCompile(`<@(\w+?)>`)
37 | re.channel = regexp.MustCompile(`<#([^|]+?)\|([^&]+?)>`)
38 | re.emoji = regexp.MustCompile(`:[^\s!"#$%&()=^/?\\\[\]<>,.;@{}~:]+:`)
39 | re.newLine = regexp.MustCompile(`\n`)
40 |
41 | return &TextConverter{
42 | emojis: emojis,
43 | users: users,
44 | re: re,
45 | baseURL: os.Getenv("BASEURL"),
46 | }
47 | }
48 |
49 | type regexps struct {
50 | linkWithTitle, link,
51 | code, codeShort,
52 | del,
53 | mention, channel, emoji,
54 | newLine *regexp.Regexp
55 | }
56 |
57 | func (c *TextConverter) escapeSpecialChars(text string) string {
58 | text = html.EscapeString(html.UnescapeString(text))
59 | text = strings.Replace(text, "{{", "{{", -1)
60 | return strings.Replace(text, "{%", "{%", -1)
61 | }
62 |
63 | func (c *TextConverter) escape(text string) string {
64 | text = html.EscapeString(html.UnescapeString(text))
65 | text = c.re.newLine.ReplaceAllString(text, " ")
66 | return text
67 | }
68 |
69 | func (c *TextConverter) bindEmoji(emojiExp string) string {
70 | name := emojiExp[1 : len(emojiExp)-1]
71 | extension, ok := c.emojis[name]
72 | if !ok {
73 | char, ok := emoji.CodeMap()[emojiExp]
74 | if ok {
75 | return char
76 | }
77 | return emojiExp
78 | }
79 | for 7 <= len(extension) && extension[:6] == "alias:" {
80 | name = extension[6:]
81 | extension, ok = c.emojis[name]
82 | if !ok {
83 | return emojiExp
84 | }
85 | }
86 | src := c.baseURL + "/emojis/" + url.PathEscape(name) + extension
87 | return "
86 | 
87 |
88 | ") 109 | chunks := c.re.code.Split(text, -1) 110 | for i, s := range chunks { 111 | if i%2 == 0 { 112 | s = c.re.linkWithTitle.ReplaceAllString(s, "${2}") 113 | s = c.re.link.ReplaceAllString(s, "${1}") 114 | s = c.re.codeShort.ReplaceAllString(s, "
${1}")
115 | s = c.re.del.ReplaceAllString(s, "" + s + "" 121 | } 122 | chunks[i] = s 123 | } 124 | return strings.Join(chunks, "") 125 | } 126 | -------------------------------------------------------------------------------- /README_ja.md: -------------------------------------------------------------------------------- 1 | # slacklog 2 | 3 | ## これは何 4 | 5 | vim-jp Slack のログを HTML 化するプロジェクトです。 6 | 7 | Slack 上では無料枠のため 8 | 9 | * 古いメッセージが見れない 10 | * ワークスペースに参加していない人には見えないため、ブログ等から引用などして参照したい 11 | * 知見が消えるのはもったいない 12 | 13 | といった問題があり、それらを解決するため作られました。 14 | 15 | ## 開発に参加するには 16 | 17 | 興味を持った方は vim-jp Slack や [@tyru](https://twitter.com/_tyru_), [@thinca](https://twitter.com/thinca) 等に声をかけて頂ければ GitHub の slacklog Team に招待します (Slack token などもその際共有します)。
18 | vim-jp Slack への参加方法はこちらをどうぞ。
19 | [vim-jp » vim-jpのチャットルームについて](https://vim-jp.org/docs/chat.html) 20 | 21 | ## 開発に必要なもの 22 | 23 | - Go 24 | - (あれば)GNU Make 25 | 26 | ## 環境変数 27 | 28 | `.env.template` からコピーして `.env` ファイルを作成してください。 29 | (各環境変数の説明はファイルを参照) 30 | 31 | ## 開発方法 32 | 33 | ### HTML の生成 34 | 35 | ログを展開 36 | 37 | ```console 38 | $ make logdata 39 | ``` 40 | 41 | HTMLの生成 42 | 43 | 以下のコマンドを実行すると`_site`以下に生成されます 44 | 45 | ```console 46 | scripts/generate_html.sh 47 | scripts/build.sh 48 | ``` 49 | 50 | GNU Makeがあれば`make`もしくは`gmake`を実行するだけで生成されます 51 | 52 | ### 添付ファイルと絵文字のダウンロード 53 | 54 | ```console 55 | scripts/download_emoji.sh 56 | scripts/download_files.sh 57 | ``` 58 | 59 | ### 開発サーバーの起動 60 | 61 | 特定のツールに依存していないので、各自お好きなサーバーを`_site`以下で起動してください 62 | 63 | 開発サーバーの起動(例): 64 | 65 | ```console 66 | python -m http.server --directory=_site 67 | ``` 68 | 69 | ### geneate-html サブコマンドの出力の差分の確認方法 70 | 71 | 以下のコマンドで自分が変更した結果として生じた generate-html の出力内容の差分 72 | を確認できます。 73 | 74 | ```console 75 | $ ./scripts/site_diff.sh 76 | ``` 77 | 78 | `site_diff.sh` では現在のHEADでの generate-html の結果と merge-base での 79 | geneate-html の結果の diff を取得しています。 80 | 出力先は `./tmp/site_diff/current/` および 81 | `./tmp/site_diff/{merge-base-commit-id}/` ディレクトリとなっています。 82 | 83 | merge-base の算出基準はローカルの origin/master です。そのため origin/master が 84 | リモート(GitHub)の物よりも古いと出力内容が異なり、差分も異なる場合があります。 85 | `-u` オプションを使うと merge-base の算出前にローカルの origin/master を更新し 86 | ます。変更がなくても更新にそれなりに時間がかかるため、デフォルトではオフになっ 87 | ており明示的に指定するようにしています。 88 | 89 | merge-base の出力結果はキャッシュし再利用しています。このキャッシュを無視して強 90 | 制的に再出力するには `-f` オプションを使ってください。 91 | 92 | ```console 93 | $ ./scripts/site_diff.sh -f 94 | ``` 95 | 96 | 全てのキャッシュを破棄したい場合には `-c` オプションを使ってください。`-c` オプ 97 | ションでは `./tmp/site_diff/` ディレクトリを消すだけで差分の出力は行いません。 98 | 99 | ```console 100 | $ ./scripts/site_diff.sh -c 101 | ``` 102 | 103 | 差分だけを特定のファイルに出力するには `-o {filename}` オプションを使ってくださ 104 | い。リダイレクト (` > filename`) では差分以外の動作ログも含まれる場合がありま 105 | す。 106 | 107 | 注意事項: `./scripts/site_diff.sh` は未コミットな変更を stash を用いて保存・復 108 | 帰しているため staged な変更が unstaged に巻き戻ることに留意してください。 109 | 110 | ## log-data の更新手順 111 | 112 | log-data ブランチにはSlackからエクスポートしたデータを格納し、それを本番の生成 113 | に利用しています。log-data ブランチの更新手順は以下の通りです。 114 | 115 | 1. Slack からログをエクスポート(今はできる人が限られてる) 116 | 2. ログをワーキングスペースに展開する 117 | 3. `convert-exported-logs` サブコマンドを実行する 118 | 119 | ```console 120 | $ go run . convert-exported-logs {indir} {outdir} 121 | ``` 122 | 123 | 4. 更新内容を log-data ブランチに `commit --amend` して `push -f` 124 | 125 | ## Pull Request の影響の確認の方法 126 | 127 | 以下の手順で Pull Request への `site_diff.sh` の実行結果を 128 | Artifacts として Web から取得できます。レビューの際に利用してください。 129 | 130 | 1. Pull Request の Checks タブを開く 131 | 2. CI ワークフロー(右側)を選択 132 | 3. Compare Pages and Site ジョブ(右側)を選択 133 | 4. Artifacts ドロワー(左側)を選択 134 | 5. `diffs-{数値}` アーティファクトをダウンロード 135 | 136 | 以下のスクリーンショットは、上記の選択個所をマーキングしたものです。 137 | (SSには3つのアーティファクトが表示されますが、現在は2つになっています) 138 | 139 |  140 | 141 | Artifacts はそれぞれ zip としてダウンロードできます。 142 | `diffs-*.zip` には `sites_diff.sh` の差分が含まれています。 143 | `log-*.zip` は動作ログが含まれていますが、こちらはCIの動作デバッグ目的のものです。 144 | 末尾の数値は [`${{ github.run_id }}`](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context) 由来です。 145 | 146 | ### Artifacts に差分を出力している理由 147 | 148 | Artifacts に差分を出力している主な理由は2つあります。1つ目は、小さな変更でも差 149 | 分をオンライン上のどこかに出力しないと、レビューの負荷が高すぎてそれを解消した 150 | かったという動機です。 151 | 152 | 2つ目は、テストデータとして実際のログを使っているため、差分とはいえログの一部の 153 | コピーが消せない状態で永続化されるのを避けたい、という動機です。vim-jp slackで 154 | は参加者の「忘れられる権利」を尊重しています。 155 | 156 | 以上の理由から消せる状態でデータ=差分をオンライン上にホストできる GitHub 157 | Actions の Artifacts を利用しています。 158 | 159 | ## LICNESE 160 | 161 |
この 作品 は クリエイティブ・コモンズ 表示 4.0 国際 ライセンスの下に提供されています。 162 | -------------------------------------------------------------------------------- /internal/slacklog/store.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | // LogStore : ログデータを各種テーブルを介して取得するための構造体。 10 | // MessageTableはチャンネル毎に用意しているためmtsはチャンネルIDをキーとするmap 11 | // となっている。 12 | type LogStore struct { 13 | path string 14 | ut *UserTable 15 | ct *ChannelTable 16 | et *EmojiTable 17 | // key: channel ID 18 | mts map[string]*MessageTable 19 | } 20 | 21 | // NewLogStore : 各テーブルを生成して、LogStoreを生成する。 22 | func NewLogStore(dirPath string, cfg *Config) (*LogStore, error) { 23 | ut, err := NewUserTable(filepath.Join(dirPath, "users.json")) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | ct, err := NewChannelTable(filepath.Join(dirPath, "channels.json"), cfg.Channels) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | et, err := NewEmojiTable(filepath.Join(dirPath, cfg.EmojiJSONPath)) 34 | if err != nil { 35 | if !os.IsNotExist(err) { 36 | return nil, err 37 | } 38 | // EmojiTable is not required, so if the file just doesn't exist, continue 39 | // processing. 40 | } 41 | 42 | mts := make(map[string]*MessageTable, len(ct.Channels)) 43 | for _, ch := range ct.Channels { 44 | mts[ch.ID] = NewMessageTable() 45 | } 46 | 47 | return &LogStore{ 48 | path: dirPath, 49 | ut: ut, 50 | ct: ct, 51 | et: et, 52 | mts: mts, 53 | }, nil 54 | } 55 | 56 | // GetChannels gets all stored channgels. 57 | func (s *LogStore) GetChannels() []Channel { 58 | return s.ct.Channels 59 | } 60 | 61 | // HasNextMonth returns a channel has next key or not. 62 | func (s *LogStore) HasNextMonth(channelID string, key MessageMonthKey) bool { 63 | if mt, ok := s.mts[channelID]; ok && mt != nil { 64 | _, ok := mt.MsgsMap[key.Next()] 65 | return ok 66 | } 67 | return false 68 | } 69 | 70 | // HasPrevMonth returns a channel has previous logs or not. 71 | func (s *LogStore) HasPrevMonth(channelID string, key MessageMonthKey) bool { 72 | if mt, ok := s.mts[channelID]; ok && mt != nil { 73 | _, ok := mt.MsgsMap[key.Prev()] 74 | return ok 75 | } 76 | return false 77 | } 78 | 79 | // GetMessagesPerMonth gets a messages map for the channel. 80 | // Messages map have all message split in per month. 81 | func (s *LogStore) GetMessagesPerMonth(channelID string) (MessagesMap, error) { 82 | mt, ok := s.mts[channelID] 83 | if !ok { 84 | return nil, fmt.Errorf("not found channel: id=%s", channelID) 85 | } 86 | if err := mt.ReadLogDir(filepath.Join(s.path, channelID), false); err != nil { 87 | return nil, err 88 | } 89 | 90 | return mt.MsgsMap, nil 91 | } 92 | 93 | // GetAllMessages returns all messages as array. 94 | func (s *LogStore) GetAllMessages(channelID string) (Messages, error) { 95 | mt, ok := s.mts[channelID] 96 | if !ok { 97 | return nil, fmt.Errorf("not found channel: id=%s", channelID) 98 | } 99 | err := mt.ReadLogDir(filepath.Join(s.path, channelID), true) 100 | if err != nil { 101 | return nil, err 102 | } 103 | var allMsgs Messages 104 | for _, msgs := range mt.MsgsMap { 105 | allMsgs = append(allMsgs, msgs...) 106 | } 107 | return allMsgs, nil 108 | } 109 | 110 | // GetUserByID gets a user by (user) ID. 111 | // Sometimes by bot ID. 112 | func (s *LogStore) GetUserByID(userID string) (*User, bool) { 113 | u, ok := s.ut.UserMap[userID] 114 | return u, ok 115 | } 116 | 117 | // GetDisplayNameByUserID gets display name for the user. 118 | func (s *LogStore) GetDisplayNameByUserID(userID string) string { 119 | if user, ok := s.ut.UserMap[userID]; ok { 120 | if user.Profile.RealName != "" { 121 | return user.Profile.RealName 122 | } 123 | if user.Profile.DisplayName != "" { 124 | return user.Profile.DisplayName 125 | } 126 | } 127 | return "" 128 | } 129 | 130 | // GetDisplayNameMap gets a map from user ID to user's display name. 131 | func (s *LogStore) GetDisplayNameMap() map[string]string { 132 | ret := make(map[string]string, len(s.ut.UserMap)) 133 | for id, u := range s.ut.UserMap { 134 | ret[id] = s.GetDisplayNameByUserID(u.ID) 135 | } 136 | return ret 137 | } 138 | 139 | // GetEmojiMap gets a map from emoji name to its file extension (image type). 140 | func (s *LogStore) GetEmojiMap() map[string]string { 141 | return s.et.NameToExt 142 | } 143 | 144 | // GetThread gets a thread (chain of messages) by channel ID and Ts (thread root's Ts). 145 | func (s *LogStore) GetThread(channelID, ts string) (*Thread, bool) { 146 | mt, ok := s.mts[channelID] 147 | if !ok { 148 | return nil, false 149 | } 150 | if t, ok := mt.ThreadMap[ts]; ok { 151 | return t, true 152 | } 153 | return nil, false 154 | } 155 | -------------------------------------------------------------------------------- /internal/slacklog/downloader.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net" 7 | "net/http" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | // Downloader : ダウンロード処理をするワーカを管理するための構造体。 15 | // TODO: 今のところ、ダウンロード処理中にエラーが発生してもキューに積まれたタス 16 | // クが全て完了するまでは分からない(Wait()の返り値として見るまでは)。 17 | type Downloader struct { 18 | token string 19 | 20 | httpClient *http.Client 21 | targetCh chan downloadTarget 22 | workerWg sync.WaitGroup 23 | 24 | // firstError stores the first error raised in workers. 25 | firstError error 26 | 27 | // errsMu: firstError に触る際は必ずコレでロックを取る 28 | errsMu sync.Mutex 29 | } 30 | 31 | var downloadWorkerNum = 8 32 | 33 | // NewDownloader creates a downloader for Slack with the token. 34 | func NewDownloader(token string) *Downloader { 35 | // http.DefaultTransportの値からMaxConnsPerHostのみ修正 36 | t := &http.Transport{ 37 | Proxy: http.ProxyFromEnvironment, 38 | DialContext: (&net.Dialer{ 39 | Timeout: 30 * time.Second, 40 | KeepAlive: 30 * time.Second, 41 | DualStack: true, 42 | }).DialContext, 43 | ForceAttemptHTTP2: true, 44 | MaxIdleConns: 100, 45 | // ワーカ起動のロジックにバグがあったとしてもこのhttp.Transportを利用してい 46 | // る限りは多量のリクエストが飛ばないように念の為downloadWorkerNumでコネク 47 | // ション数を制限しておく 48 | MaxConnsPerHost: downloadWorkerNum, 49 | IdleConnTimeout: 90 * time.Second, 50 | TLSHandshakeTimeout: 10 * time.Second, 51 | ExpectContinueTimeout: 1 * time.Second, 52 | } 53 | 54 | cli := &http.Client{Transport: t} 55 | // 無効なSlack API tokenを食わせても、リダイレクトされ、200が返ってきてエラー 56 | // かどうか判別できない。 57 | // 一方で、なぜか .svg ファイルのダウンロード時にはリダイレクトが発生する。 58 | // リダイレクト先の URL を見て雰囲気で認証ページに行ってそうだったらエラーにする 59 | cli.CheckRedirect = func(req *http.Request, via []*http.Request) error { 60 | _, redirEixsts := req.URL.Query()["redir"] 61 | if strings.HasSuffix(req.URL.Host, ".slack.com") && 62 | req.URL.Path == "/" && 63 | redirEixsts { 64 | return http.ErrUseLastResponse 65 | } 66 | return nil 67 | } 68 | 69 | d := &Downloader{ 70 | token: token, 71 | httpClient: cli, 72 | targetCh: make(chan downloadTarget), 73 | } 74 | 75 | for i := 0; i < downloadWorkerNum; i++ { 76 | d.workerWg.Add(1) 77 | go func() { 78 | defer d.workerWg.Done() 79 | d.runWorker() 80 | }() 81 | } 82 | 83 | return d 84 | } 85 | 86 | // QueueDownloadRequest : ダウンロード処理をqueueに積む 87 | func (d *Downloader) QueueDownloadRequest(url, outputPath string, withToken bool) { 88 | d.targetCh <- downloadTarget{ 89 | url: url, 90 | outputPath: outputPath, 91 | withToken: withToken, 92 | } 93 | } 94 | 95 | // Wait : ワーカが全て実行終了するまで待つ。 96 | // ダウンロード処理中にエラーが発生していた場合は最初に発生した1つを返す。 97 | // 他のエラーはログに出力している。 98 | func (d *Downloader) Wait() error { 99 | d.workerWg.Wait() 100 | d.errsMu.Lock() 101 | defer d.errsMu.Unlock() 102 | return d.firstError 103 | } 104 | 105 | // CloseQueue : ダウンロードキューへの追加が完了したことをDownloaderに通知する 106 | // ために実行する。 107 | // TODO: 2回実行するとpanicしてしまうのを修正する。Downloaderに状態でも持たせる 108 | // とよいだろうか。 109 | func (d *Downloader) CloseQueue() { 110 | close(d.targetCh) 111 | } 112 | 113 | func (d *Downloader) runWorker() { 114 | for t := range d.targetCh { 115 | err := d.download(t) 116 | if err != nil { 117 | d.errsMu.Lock() 118 | if d.firstError == nil { 119 | d.firstError = err 120 | } 121 | d.errsMu.Unlock() 122 | fmt.Printf("download failed for url=%s: %s\n", t.url, err) 123 | } 124 | } 125 | } 126 | 127 | // downloadTarget : Downloaderにダウンロードする対象を指定するために使う。 128 | type downloadTarget struct { 129 | url string 130 | outputPath string 131 | // ダウンロード時にSlack API tokenを利用するかどうかを指定する 132 | withToken bool 133 | } 134 | 135 | func (d *Downloader) download(t downloadTarget) error { 136 | _, err := os.Stat(t.outputPath) 137 | if err == nil { 138 | // Just skip already downloaded file 139 | fmt.Printf("already exist: %s\n", t.outputPath) 140 | return nil 141 | } 142 | // `err != nil` has two cases at here. first is "not exist" as expected. 143 | // and second is I/O error as unexpected. 144 | if !os.IsNotExist(err) { 145 | return err 146 | } 147 | 148 | fmt.Printf("Downloading: %s\n", t.outputPath) 149 | 150 | req, err := http.NewRequest("GET", t.url, nil) 151 | if err != nil { 152 | return err 153 | } 154 | 155 | if t.withToken { 156 | req.Header.Add("Authorization", "Bearer "+d.token) 157 | } 158 | 159 | resp, err := d.httpClient.Do(req) 160 | if err != nil { 161 | return err 162 | } 163 | defer resp.Body.Close() 164 | 165 | if resp.StatusCode/100 != 2 { 166 | // Some files cannot download by unknown reason. 167 | // Just ignore. 168 | fmt.Fprintf(os.Stderr, "ERROR (ignored): [%s]: %s\n", resp.Status, t.url) 169 | return nil 170 | } 171 | 172 | w, err := os.Create(t.outputPath) 173 | if err != nil { 174 | return err 175 | } 176 | defer w.Close() 177 | 178 | _, err = io.Copy(w, resp.Body) 179 | if err != nil { 180 | return err 181 | } 182 | return nil 183 | } 184 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # slacklog 2 | 3 | ## What 4 | 5 | A project to htmlize vim-jp Slack logs. 6 | 7 | This solves the following problems due to using free tier 8 | 9 | * No old messages to see 10 | * Can't see unless you join the workspace, even though you'd like to refer from something such as your blogs 11 | * We lose our knowledge base 12 | 13 | ## How to contribute 14 | 15 | [@tyru](https://twitter.com/_tyru_) and [@thinca](https://twitter.com/thinca) will invite you to slacklog Team if you contact us via vim-jp Slack or Twitter. We'll share Slack token as well. 16 | 17 | How to join vim-jp Slack (Japanese): 18 |
この 作品 は クリエイティブ・コモンズ 表示 4.0 国際 ライセンスの下に提供されています。 170 | -------------------------------------------------------------------------------- /subcmd/convert_exported_logs.go: -------------------------------------------------------------------------------- 1 | /* 2 | リファクタリング中 3 | 処理をslacklog packageに移動していく。 4 | 一旦、必要な処理はすべてslacklog packageから一時的にエクスポートするか、このファ 5 | イル内で定義している。 6 | */ 7 | 8 | package subcmd 9 | 10 | import ( 11 | "encoding/json" 12 | "fmt" 13 | "io" 14 | "os" 15 | "path/filepath" 16 | "regexp" 17 | 18 | cli "github.com/urfave/cli/v2" 19 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 20 | ) 21 | 22 | // ConvertExportedLogsCommand provides "convert-exported-logs". It converts log 23 | // which exported from Slack, to generator can treat. 24 | var ConvertExportedLogsCommand = &cli.Command{ 25 | Name: "convert-exported-logs", 26 | Usage: "convert slack exported logs to API download logs", 27 | Action: convertExportedLogs, 28 | Flags: []cli.Flag{ 29 | &cli.StringFlag{ 30 | Name: "indir", 31 | Usage: "exported log dir", 32 | Value: "_old_logs", 33 | }, 34 | &cli.StringFlag{ 35 | Name: "outdir", 36 | Usage: "slacklog_data dir", 37 | Value: filepath.Join("_logdata", "slacklog_data"), 38 | }, 39 | }, 40 | } 41 | 42 | // convertExportedLogs converts log which exported from Slack, to generator can 43 | // treat. 44 | func convertExportedLogs(c *cli.Context) error { 45 | inDir := filepath.Clean(c.String("indir")) 46 | outDir := filepath.Clean(c.String("outdir")) 47 | inChannelsFile := filepath.Join(inDir, "channels.json") 48 | 49 | channels, _, err := readChannels(inChannelsFile, []string{"*"}) 50 | if err != nil { 51 | return fmt.Errorf("could not read channels.json: %w", err) 52 | } 53 | 54 | if err := os.MkdirAll(outDir, 0777); err != nil { 55 | return fmt.Errorf("could not create %s directory: %w", outDir, err) 56 | } 57 | 58 | err = copyFile(filepath.Join(inDir, "channels.json"), filepath.Join(outDir, "channels.json")) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | err = copyFile(filepath.Join(inDir, "users.json"), filepath.Join(outDir, "users.json")) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | for _, channel := range channels { 69 | messages, err := ReadAllMessages(filepath.Join(inDir, channel.Name)) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | for _, message := range messages { 75 | message.RemoveTokenFromURLs() 76 | } 77 | 78 | channelDir := filepath.Join(outDir, channel.ID) 79 | if err := os.MkdirAll(channelDir, 0777); err != nil { 80 | return fmt.Errorf("could not create %s directory: %w", channelDir, err) 81 | } 82 | 83 | messagesPerDay := groupMessagesByDay(messages) 84 | for key, msgs := range messagesPerDay { 85 | err = writeMessages(filepath.Join(channelDir, key+".json"), msgs) 86 | if err != nil { 87 | return err 88 | } 89 | } 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func copyFile(from string, to string) error { 96 | r, err := os.Open(from) 97 | if err != nil { 98 | return err 99 | } 100 | defer r.Close() 101 | 102 | w, err := os.Create(to) 103 | if err != nil { 104 | return err 105 | } 106 | defer w.Close() 107 | 108 | _, err = io.Copy(w, r) 109 | 110 | return err 111 | } 112 | 113 | func readChannels(channelJSONPath string, cfgChannels []string) ([]slacklog.Channel, map[string]*slacklog.Channel, error) { 114 | var channels []slacklog.Channel 115 | err := slacklog.ReadFileAsJSON(channelJSONPath, true, &channels) 116 | if err != nil { 117 | return nil, nil, err 118 | } 119 | 120 | channels = slacklog.FilterChannel(channels, cfgChannels) 121 | slacklog.SortChannel(channels) 122 | channelMap := make(map[string]*slacklog.Channel, len(channels)) 123 | for i, ch := range channels { 124 | channelMap[ch.ID] = &channels[i] 125 | } 126 | return channels, channelMap, err 127 | } 128 | 129 | // ReadAllMessages reads all message files in the directory. 130 | func ReadAllMessages(inDir string) ([]*slacklog.Message, error) { 131 | dir, err := os.Open(inDir) 132 | if err != nil { 133 | return nil, err 134 | } 135 | defer dir.Close() 136 | 137 | names, err := dir.Readdirnames(0) 138 | if err != nil { 139 | return nil, err 140 | } 141 | 142 | r, err := regexp.Compile(`\d{4}-\d{2}-\d{2}\.json`) 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | var messages []*slacklog.Message 148 | for _, name := range names { 149 | if !r.MatchString(name) { 150 | continue 151 | } 152 | var msgs []*slacklog.Message 153 | err := slacklog.ReadFileAsJSON(filepath.Join(inDir, name), false, &msgs) 154 | if err != nil { 155 | return nil, err 156 | } 157 | messages = append(messages, msgs...) 158 | } 159 | 160 | return messages, nil 161 | } 162 | 163 | func groupMessagesByDay(messages []*slacklog.Message) map[string][]*slacklog.Message { 164 | messagesPerDay := map[string][]*slacklog.Message{} 165 | for _, msg := range messages { 166 | time := slacklog.TsToDateTime(msg.Timestamp).Format("2006-01-02") 167 | messagesPerDay[time] = append(messagesPerDay[time], msg) 168 | } 169 | return messagesPerDay 170 | } 171 | 172 | func writeMessages(filename string, messages []*slacklog.Message) error { 173 | file, err := os.Create(filename) 174 | if err != nil { 175 | return err 176 | } 177 | defer file.Close() 178 | encoder := json.NewEncoder(file) 179 | encoder.SetEscapeHTML(false) 180 | err = encoder.Encode(messages) 181 | if err != nil { 182 | return err 183 | } 184 | return nil 185 | } 186 | -------------------------------------------------------------------------------- /internal/slacklog/indexer.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strconv" 10 | "strings" 11 | "unicode/utf16" 12 | ) 13 | 14 | const gramN = 2 15 | 16 | type Indexer struct { 17 | s *LogStore 18 | gramIndex messageIndex 19 | channelNumbers map[int]Channel 20 | } 21 | 22 | func NewIndexer(s *LogStore) *Indexer { 23 | return &Indexer{ 24 | s: s, 25 | gramIndex: messageIndex{}, 26 | channelNumbers: map[int]Channel{}, 27 | } 28 | } 29 | 30 | func (idx *Indexer) Build() error { 31 | channelNumber := 0 32 | for _, c := range idx.s.GetChannels() { 33 | channelNumber++ 34 | idx.channelNumbers[channelNumber] = c 35 | msgs, err := idx.s.GetAllMessages(c.ID) 36 | if err != nil { 37 | return err 38 | } 39 | for _, m := range msgs { 40 | runes := []rune(m.Text) 41 | textLen := len(runes) 42 | for i := range runes { 43 | for n := 1; n <= gramN; n++ { 44 | if textLen <= i+n-1 { 45 | break 46 | } 47 | key := string(runes[i : i+n]) 48 | idx.gramIndex.Add(key, channelNumber, m, i) 49 | } 50 | } 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | func (idx *Indexer) Output(outDir string) error { 57 | channelFilepath := filepath.Join(outDir, "channel") 58 | err := idx.writeChannelFile(channelFilepath, idx.channelNumbers) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | for key, mPositions := range idx.gramIndex { 64 | s := outDir 65 | for _, u := range utf16.Encode([]rune(key)) { 66 | s = filepath.Join(s, fmt.Sprintf("%02x", u>>8), fmt.Sprintf("%02x", u&0xff)) 67 | } 68 | err := idx.writeIndexFile(s+".index", mPositions) 69 | if err != nil { 70 | return err 71 | } 72 | } 73 | return nil 74 | } 75 | 76 | func (idx *Indexer) writeChannelFile(path string, channelNumbers map[int]Channel) error { 77 | err := os.MkdirAll(filepath.Dir(path), 0o777) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | f, err := os.Create(path) 83 | if err != nil { 84 | return err 85 | } 86 | defer f.Close() 87 | 88 | fw := bufio.NewWriter(f) 89 | for channelNumber, channel := range channelNumbers { 90 | _, err := fw.Write([]byte(fmt.Sprintf("%d\t%s\t%s\n", channelNumber, channel.ID, channel.Name))) 91 | if err != nil { 92 | return err 93 | } 94 | } 95 | 96 | err = fw.Flush() 97 | if err != nil { 98 | return err 99 | } 100 | 101 | return nil 102 | } 103 | 104 | func (idx *Indexer) writeIndexFile(path string, mPositions messagePositions) error { 105 | err := os.MkdirAll(filepath.Dir(path), 0o777) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | f, err := os.Create(path) 111 | if err != nil { 112 | return err 113 | } 114 | defer f.Close() 115 | 116 | fw := bufio.NewWriter(f) 117 | for channelNumber, mposMap := range mPositions { 118 | _, err := fw.Write(vintBytes(channelNumber)) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | _, err = fw.Write(vintBytes(len(mposMap))) 124 | if err != nil { 125 | return err 126 | } 127 | 128 | for ts, positions := range mposMap { 129 | tsParts := strings.SplitN(ts, ".", 2) 130 | if len(tsParts) != 2 { 131 | channel := idx.channelNumbers[channelNumber] 132 | return fmt.Errorf("Invalid timestamp %s (%s)", ts, channel.ID) 133 | } 134 | 135 | tsSec, err := strconv.Atoi(tsParts[0]) 136 | if err != nil { 137 | return err 138 | } 139 | err = binary.Write(fw, binary.BigEndian, uint32(tsSec)) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | tsMicrosec, err := strconv.Atoi(tsParts[1]) 145 | if err != nil { 146 | return err 147 | } 148 | _, err = fw.Write(vintBytes(tsMicrosec)) 149 | if err != nil { 150 | return err 151 | } 152 | 153 | if len(positions) == 0 { 154 | channel := idx.channelNumbers[channelNumber] 155 | return fmt.Errorf("Empty positions: %s: %s: %s", path, channel.ID, ts) 156 | } 157 | for _, pos := range positions { 158 | _, err = fw.Write(vintBytes(pos + 1)) 159 | if err != nil { 160 | return err 161 | } 162 | } 163 | err = fw.WriteByte(0) 164 | if err != nil { 165 | return err 166 | } 167 | } 168 | } 169 | 170 | err = fw.Flush() 171 | if err != nil { 172 | return err 173 | } 174 | 175 | return nil 176 | } 177 | 178 | type messageIndex map[string]messagePositions 179 | 180 | func (mi messageIndex) Add(key string, channelNumber int, mes *Message, pos int) { 181 | mp, ok := mi[key] 182 | if !ok { 183 | mp = messagePositions{} 184 | mi[key] = mp 185 | } 186 | mp.Add(channelNumber, mes, pos) 187 | } 188 | 189 | type messagePositions map[int]map[string][]int 190 | 191 | func (mp messagePositions) Add(channelNumber int, mes *Message, pos int) { 192 | mposMap, ok := mp[channelNumber] 193 | if !ok { 194 | mposMap = map[string][]int{} 195 | mp[channelNumber] = mposMap 196 | } 197 | 198 | mposMap[mes.Timestamp] = append(mposMap[mes.Timestamp], pos) 199 | } 200 | 201 | func vintBytes(n int) []byte { 202 | if n == 0 { 203 | return []byte{0} 204 | } 205 | bytes := []byte{} 206 | cont := false 207 | for n != 0 { 208 | b := byte(n & 0b01111111) 209 | if cont { 210 | b |= 0b10000000 211 | } 212 | bytes = append([]byte{b}, bytes...) 213 | n >>= 7 214 | cont = true 215 | } 216 | return bytes 217 | } 218 | -------------------------------------------------------------------------------- /subcmd/fetchmessages/fetchmessages.go: -------------------------------------------------------------------------------- 1 | package fetchmessages 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | 12 | "github.com/slack-go/slack" 13 | cli "github.com/urfave/cli/v2" 14 | "github.com/vim-jp/slacklog-generator/internal/jsonwriter" 15 | "github.com/vim-jp/slacklog-generator/internal/slackadapter" 16 | "github.com/vim-jp/slacklog-generator/internal/slacklog" 17 | ) 18 | 19 | const dateFormat = "2006-01-02" 20 | 21 | func toDateString(ti time.Time) string { 22 | return ti.Format(dateFormat) 23 | } 24 | 25 | func parseDateString(s string) (time.Time, error) { 26 | l, err := time.LoadLocation("Asia/Tokyo") 27 | if err != nil { 28 | return time.Time{}, err 29 | } 30 | ti, err := time.ParseInLocation(dateFormat, s, l) 31 | if err != nil { 32 | return time.Time{}, err 33 | } 34 | return ti, nil 35 | } 36 | 37 | // Run runs "fetch-messages" sub-command. It fetch messages of a channel by a 38 | // day. 39 | func Run(args []string) error { 40 | var ( 41 | token string 42 | datadir string 43 | date string 44 | verbose bool 45 | ) 46 | fs := flag.NewFlagSet("fetch-messages", flag.ExitOnError) 47 | fs.StringVar(&token, "token", os.Getenv("SLACK_TOKEN"), `slack token. can be set by SLACK_TOKEN env var`) 48 | fs.StringVar(&datadir, "datadir", "_logdata", `directory to load/save data`) 49 | fs.StringVar(&date, "date", toDateString(time.Now()), `target date to get`) 50 | fs.BoolVar(&verbose, "verbose", false, "verbose log") 51 | err := fs.Parse(args) 52 | if err != nil { 53 | return err 54 | } 55 | if token == "" { 56 | return errors.New("SLACK_TOKEN environment variable requied") 57 | } 58 | return run(token, datadir, date, verbose) 59 | } 60 | 61 | func run(token, datadir, date string, verbose bool) error { 62 | oldest, err := parseDateString(date) 63 | if err != nil { 64 | return err 65 | } 66 | latest := oldest.AddDate(0, 0, 1) 67 | 68 | ct, err := slacklog.NewChannelTable(filepath.Join(datadir, "channels.json"), []string{"*"}) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | for _, sch := range ct.Channels { 74 | outdir := filepath.Join(datadir, sch.ID) 75 | if err := os.MkdirAll(outdir, 0755); err != nil { 76 | return fmt.Errorf("making outdir: %w", err) 77 | } 78 | outfile := filepath.Join(outdir, toDateString(oldest)+".json") 79 | fw, err := jsonwriter.CreateFile(outfile, true) 80 | if err != nil { 81 | return err 82 | } 83 | err = slackadapter.IterateCursor(context.Background(), 84 | slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) { 85 | r, err := slackadapter.ConversationsHistory(ctx, token, sch.ID, slackadapter.ConversationsHistoryParams{ 86 | Cursor: c, 87 | Limit: 100, 88 | Oldest: &oldest, 89 | Latest: &latest, 90 | }) 91 | if err != nil { 92 | return "", err 93 | } 94 | for _, message := range r.Messages { 95 | if message.IsRootOfThread() { 96 | client := slack.New(token) 97 | err = slackadapter.IterateCursor(ctx, slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) { 98 | msgs, hasMore, nextCursor, err := client.GetConversationRepliesContext(ctx, &slack.GetConversationRepliesParameters{ 99 | ChannelID: sch.ID, 100 | Cursor: string(c), 101 | Timestamp: message.Timestamp, 102 | }) 103 | if err != nil { 104 | return "", err 105 | } 106 | for _, m := range msgs { 107 | sMes := slacklog.Message{ 108 | Message: m, 109 | } 110 | // スレッドのルートとブロードキャストメッセージは通常のログに含まれるのでここでは弾く 111 | if !sMes.IsRootOfThread() && sMes.SubType != "thread_broadcast" { 112 | r.Messages = append(r.Messages, &sMes) 113 | } 114 | } 115 | if hasMore { 116 | return slackadapter.Cursor(nextCursor), nil 117 | } 118 | return "", nil 119 | })) 120 | if err != nil { 121 | return "", err 122 | } 123 | } 124 | } 125 | for _, m := range r.Messages { 126 | err := fw.Write(m) 127 | if err != nil { 128 | return "", err 129 | } 130 | } 131 | if m := r.ResponseMetadata; r.HasMore && m != nil { 132 | return m.NextCursor, nil 133 | } 134 | // HasMore && ResponseMetadata == nil は明らかにエラーだがいま 135 | // は握りつぶしてる 136 | return "", nil 137 | })) 138 | if err != nil { 139 | // ロールバック相当が好ましいが今はまだその時期ではない 140 | fw.Close() 141 | return err 142 | } 143 | err = fw.Close() 144 | if err != nil { 145 | return err 146 | } 147 | } 148 | 149 | return nil 150 | } 151 | 152 | // NewCLICommand creates a cli.Command, which provides "fetch-messages" 153 | // sub-command. 154 | func NewCLICommand() *cli.Command { 155 | var ( 156 | token string 157 | datadir string 158 | date string 159 | verbose bool 160 | ) 161 | return &cli.Command{ 162 | Name: "fetch-messages", 163 | Usage: "fetch messages of channel by day", 164 | Action: func(c *cli.Context) error { 165 | return run(token, datadir, date, verbose) 166 | }, 167 | Flags: []cli.Flag{ 168 | &cli.StringFlag{ 169 | Name: "token", 170 | Usage: "slack token", 171 | EnvVars: []string{"SLACK_TOKEN"}, 172 | Destination: &token, 173 | }, 174 | &cli.StringFlag{ 175 | Name: "datadir", 176 | Usage: "directory to load/save data", 177 | Value: "_logdata", 178 | Destination: &datadir, 179 | }, 180 | &cli.StringFlag{ 181 | Name: "date", 182 | Usage: "target date to get", 183 | Value: toDateString(time.Now()), 184 | Destination: &date, 185 | }, 186 | &cli.BoolFlag{ 187 | Name: "verbose", 188 | Usage: "verbose log", 189 | Destination: &verbose, 190 | }, 191 | }, 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /internal/slacklog/filetype.go: -------------------------------------------------------------------------------- 1 | package slacklog 2 | 3 | // FiletypeToExtension is a map which maps filetype to filename extension. 4 | // 5 | // Based on
${links.length} 件ヒットしました (${processTime / 1000} 秒)
${links.join("")}`; 168 | } catch (e) { 169 | resultElement.innerHTML = `検索中にエラーが発生しました: ${e.name}: ${e.message}`; 170 | } 171 | }; 172 | 173 | const searchFromParams = () => { 174 | const query = new URL(location).searchParams.get("q"); 175 | if (query) { 176 | text.value = query; 177 | execute(); 178 | } else { 179 | text.value = ""; 180 | resultElement.innerHTML = ""; 181 | } 182 | }; 183 | 184 | const moveToResultPage = () => { 185 | const params = new URLSearchParams(); 186 | params.set("q", text.value); 187 | history.pushState({q: text.value}, document.title, `${location.pathname}?${params.toString()}`); 188 | execute(); 189 | }; 190 | 191 | document.getElementById("search-button").addEventListener("click", moveToResultPage); 192 | document.getElementById("search-text").addEventListener("keypress", (event) => { 193 | if (event.key === "Enter") { 194 | moveToResultPage(); 195 | } 196 | }); 197 | window.addEventListener("popstate", (e) => { 198 | console.log(e); 199 | searchFromParams(); 200 | }); 201 | 202 | searchFromParams(); 203 | }); 204 | -------------------------------------------------------------------------------- /templates/channel_per_month/index.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |
21 |
174 |
175 |
--------------------------------------------------------------------------------
/internal/slacklog/message.go:
--------------------------------------------------------------------------------
1 | package slacklog
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "path/filepath"
7 | "regexp"
8 | "sort"
9 | "strconv"
10 | "strings"
11 |
12 | "github.com/slack-go/slack"
13 | )
14 |
15 | // Messages is an array of `*Message`.
16 | type Messages []*Message
17 |
18 | // Sort sorts messages by `Message.Ts` ascendant order.
19 | func (msgs Messages) Sort() {
20 | sort.SliceStable(msgs, func(i, j int) bool {
21 | // must be the same digits, so no need to convert the timestamp to a number
22 | return msgs[i].Timestamp < msgs[j].Timestamp
23 | })
24 | }
25 |
26 | // MessagesMap is a map, maps MessageMonthKey to Messages.
27 | type MessagesMap map[MessageMonthKey]Messages
28 |
29 | // Keys returns all keys in the map.
30 | func (mm MessagesMap) Keys() []MessageMonthKey {
31 | keys := make([]MessageMonthKey, 0, len(mm))
32 | for key := range mm {
33 | keys = append(keys, key)
34 | }
35 | return keys
36 | }
37 |
38 | // MessageTable : メッセージデータを保持する
39 | // スレッドは投稿時刻からどのスレッドへの返信かが判断できるためThreadMapのキー
40 | // はtsである。
41 | // MsgsMapは月毎にメッセージを保持する。そのためキーは投稿月である。
42 | // loadedFilesはすでに読み込んだファイルパスを保持する。
43 | // loadedFilesは同じファイルを二度読むことを防ぐために用いている。
44 | type MessageTable struct {
45 | // key: timestamp
46 | AllMessageMap map[string]*Message
47 | // key: thread timestamp
48 | ThreadMap map[string]*Thread
49 | MsgsMap MessagesMap
50 | // key: file path
51 | loadedFiles map[string]struct{}
52 | }
53 |
54 | // NewMessageTable : MessageTableを生成する。
55 | // 他のテーブルと違い、メッセージファイルは量が多いため、NewMessageTable()実行
56 | // 時には読み込まず、(*MessageTable).ReadLogDir()/(*MessageTable).ReadLogFile()
57 | // 実行時に読み込ませる。
58 | func NewMessageTable() *MessageTable {
59 | return &MessageTable{
60 | AllMessageMap: map[string]*Message{},
61 | ThreadMap: map[string]*Thread{},
62 | MsgsMap: MessagesMap{},
63 | loadedFiles: map[string]struct{}{},
64 | }
65 | }
66 |
67 | // ReadLogDir : pathに指定したディレクトリに存在するJSON形式のメッセージデータ
68 | // を読み込む。
69 | // すでにそのディレクトリが読み込み済みの場合は処理をスキップする。
70 | // デフォルトでは特定のサブタイプを持つメッセージのみをmsgMapに登録するが、
71 | // readAllMessages が true である場合はすべてのメッセージを登録する。
72 | func (m *MessageTable) ReadLogDir(path string, readAllMessages bool) error {
73 | dir, err := os.Open(path)
74 | if err != nil {
75 | return err
76 | }
77 | defer dir.Close()
78 | names, err := dir.Readdirnames(0)
79 | if err != nil {
80 | return err
81 | }
82 | // ReadLogFile()は日付順に処理する必要があり、そのためにファイル名でソート
83 | // している
84 | sort.Strings(names)
85 | for _, name := range names {
86 | if err := m.ReadLogFile(filepath.Join(path, name), readAllMessages); err != nil {
87 | return err
88 | }
89 | }
90 | return nil
91 | }
92 |
93 | // "{year}-{month}-{day}.json"
94 | var reMsgFilename = regexp.MustCompile(`^(\d{4})-(\d{2})-\d{2}\.json$`)
95 |
96 | // ReadLogFile : pathに指定したJSON形式のメッセージデータを読み込む。
97 | // すでにそのファイルが読み込み済みの場合は処理をスキップする。
98 | // readAllMessagesがfalseである場合は特定のサブタイプを持つメッセージのみをmsgMapに登録する。
99 | func (m *MessageTable) ReadLogFile(path string, readAllMessages bool) error {
100 | path, err := filepath.Abs(path)
101 | if err != nil {
102 | return err
103 | }
104 | if _, ok := m.loadedFiles[path]; ok {
105 | return nil
106 | }
107 |
108 | match := reMsgFilename.FindStringSubmatch(filepath.Base(path))
109 | if len(match) == 0 {
110 | fmt.Fprintf(os.Stderr, "[warning] skipping %s ...\n", path)
111 | return nil
112 | }
113 |
114 | var msgs Messages
115 | err = ReadFileAsJSON(path, true, &msgs)
116 | if err != nil {
117 | return fmt.Errorf("failed to unmarshal %s: %w", path, err)
118 | }
119 |
120 | // assort messages, visible and threaded.
121 | var visibleMsgs Messages
122 | for _, msg := range msgs {
123 | if !readAllMessages && !msg.isVisible() {
124 | continue
125 | }
126 |
127 | // スレッドに所属してるメッセージは ThreadMap へスレッド毎に分別しておく
128 | threadTs := msg.ThreadTimestamp
129 | if threadTs != "" {
130 | thread, ok := m.ThreadMap[threadTs]
131 | if !ok {
132 | thread = &Thread{}
133 | if rootMsg, ok := m.AllMessageMap[threadTs]; ok {
134 | thread.rootMsg = rootMsg
135 | }
136 | m.ThreadMap[threadTs] = thread
137 | }
138 | thread.Put(msg)
139 | }
140 |
141 | if !readAllMessages && msg.isThreadChild() {
142 | continue
143 | }
144 |
145 | m.AllMessageMap[msg.Timestamp] = msg
146 |
147 | visibleMsgs = append(visibleMsgs, msg)
148 | }
149 |
150 | key, err := NewMessageMonthKey(match[1], match[2])
151 | if err != nil {
152 | return err
153 | }
154 | if len(visibleMsgs) != 0 {
155 | m.MsgsMap[key] = append(m.MsgsMap[key], visibleMsgs...)
156 | }
157 |
158 | for _, msgs := range m.MsgsMap {
159 | msgs.Sort()
160 | var lastUser string
161 | for _, msg := range msgs {
162 | if lastUser == msg.User {
163 | msg.Trail = true
164 | } else {
165 | lastUser = msg.User
166 | }
167 | }
168 | }
169 |
170 | for _, thread := range m.ThreadMap {
171 | thread.replies.Sort()
172 | }
173 |
174 | // loaded marker
175 | m.loadedFiles[path] = struct{}{}
176 | return nil
177 | }
178 |
179 | // MessageMonthKey is a key for messages.
180 | type MessageMonthKey struct {
181 | year int
182 | month int
183 | }
184 |
185 | // NewMessageMonthKey creates MessageMonthKey key from two strings: which
186 | // represents year and month.
187 | func NewMessageMonthKey(year, month string) (MessageMonthKey, error) {
188 | y, err := strconv.Atoi(year)
189 | if err != nil {
190 | return MessageMonthKey{}, err
191 | }
192 | m, err := strconv.Atoi(month)
193 | if err != nil {
194 | return MessageMonthKey{}, err
195 | }
196 | return MessageMonthKey{year: y, month: m}, nil
197 | }
198 |
199 | // Next gets a key for next month.
200 | func (k MessageMonthKey) Next() MessageMonthKey {
201 | if k.month >= 12 {
202 | return MessageMonthKey{year: k.year + 1, month: 1}
203 | }
204 | return MessageMonthKey{year: k.year, month: k.month + 1}
205 | }
206 |
207 | // Prev gets a key for previous month.
208 | func (k MessageMonthKey) Prev() MessageMonthKey {
209 | if k.month <= 1 {
210 | return MessageMonthKey{year: k.year - 1, month: 12}
211 | }
212 | return MessageMonthKey{year: k.year, month: k.month - 1}
213 | }
214 |
215 | // Year returns string represents year.
216 | func (k MessageMonthKey) Year() string {
217 | return fmt.Sprintf("%4d", k.year)
218 | }
219 |
220 | // Month returns string represents month.
221 | func (k MessageMonthKey) Month() string {
222 | return fmt.Sprintf("%02d", k.month)
223 | }
224 |
225 | // NextYear returns a string for next year.
226 | func (k MessageMonthKey) NextYear() string {
227 | if k.month >= 12 {
228 | return fmt.Sprintf("%4d", k.year+1)
229 | }
230 | return fmt.Sprintf("%4d", k.year)
231 | }
232 |
233 | // NextMonth returns a string for next month.
234 | func (k MessageMonthKey) NextMonth() string {
235 | if k.month >= 12 {
236 | return "01"
237 | }
238 | return fmt.Sprintf("%02d", k.month+1)
239 | }
240 |
241 | // PrevYear returns a string for previous year.
242 | func (k MessageMonthKey) PrevYear() string {
243 | if k.month <= 1 {
244 | return fmt.Sprintf("%4d", k.year-1)
245 | }
246 | return fmt.Sprintf("%4d", k.year)
247 | }
248 |
249 | // PrevMonth returns a string for previous month.
250 | func (k MessageMonthKey) PrevMonth() string {
251 | if k.month <= 1 {
252 | return "12"
253 | }
254 | return fmt.Sprintf("%02d", k.month-1)
255 | }
256 |
257 | // Message : メッセージ
258 | // エクスポートしたYYYY-MM-DD.jsonの中身を保持する。
259 | // https://slack.com/intl/ja-jp/help/articles/220556107-Slack-%E3%81%8B%E3%82%89%E3%82%A8%E3%82%AF%E3%82%B9%E3%83%9D%E3%83%BC%E3%83%88%E3%81%97%E3%81%9F%E3%83%87%E3%83%BC%E3%82%BF%E3%81%AE%E8%AA%AD%E3%81%BF%E6%96%B9
260 | type Message struct {
261 | slack.Message
262 |
263 | // Trail shows the user of message is same as the previous one.
264 | // FIXME: 本来はココに書いてはいけない
265 | Trail bool `json:"-"`
266 | }
267 |
268 | // isVisible : 表示すべきメッセージ種別かを判定する。
269 | // 例えばchannel_joinなどは投稿された出力する必要がないため、falseを返す。
270 | func (m *Message) isVisible() bool {
271 | return m.SubType == "" ||
272 | m.SubType == "bot_message" ||
273 | m.SubType == "slackbot_response" ||
274 | m.SubType == "thread_broadcast"
275 | }
276 |
277 | // isBotMessage : メッセージがBotからの物かを判定する。
278 | func (m *Message) isBotMessage() bool {
279 | return m.SubType == "bot_message" ||
280 | m.SubType == "slackbot_response"
281 | }
282 |
283 | // IsRootOfThread : メッセージがスレッドの最初のメッセージであるかを判定する。
284 | func (m Message) IsRootOfThread() bool {
285 | return m.Timestamp == m.ThreadTimestamp
286 | }
287 |
288 | // isThreadChild returns true when a message should be shown in a thread only.
289 | func (m *Message) isThreadChild() bool {
290 | return m.ThreadTimestamp != "" && m.Timestamp != m.ThreadTimestamp && m.SubType != "thread_broadcast"
291 | }
292 |
293 | var reToken = regexp.MustCompile(`\?t=xoxe-[-a-f0-9]+$`)
294 |
295 | func removeToken(s string) string {
296 | return reToken.ReplaceAllLiteralString(s, "")
297 | }
298 |
299 | // RemoveTokenFromURLs removes the token from URLs in a message.
300 | func (m *Message) RemoveTokenFromURLs() {
301 | for i, f := range m.Files {
302 | f.URLPrivate = removeToken(f.URLPrivate)
303 | f.URLPrivateDownload = removeToken(f.URLPrivateDownload)
304 | f.Thumb64 = removeToken(f.Thumb64)
305 | f.Thumb80 = removeToken(f.Thumb80)
306 | f.Thumb160 = removeToken(f.Thumb160)
307 | f.Thumb360 = removeToken(f.Thumb360)
308 | f.Thumb480 = removeToken(f.Thumb480)
309 | f.Thumb720 = removeToken(f.Thumb720)
310 | f.Thumb800 = removeToken(f.Thumb800)
311 | f.Thumb960 = removeToken(f.Thumb960)
312 | f.Thumb1024 = removeToken(f.Thumb1024)
313 | f.Thumb360Gif = removeToken(f.Thumb360Gif)
314 | f.Thumb480Gif = removeToken(f.Thumb480Gif)
315 | f.DeanimateGif = removeToken(f.DeanimateGif)
316 | f.ThumbVideo = removeToken(f.ThumbVideo)
317 | m.Files[i] = f
318 | }
319 | }
320 |
321 | var filenameReplacer = strings.NewReplacer(
322 | `\`, "_",
323 | "/", "_",
324 | ":", "_",
325 | "*", "_",
326 | "?", "_",
327 | `"`, "_",
328 | "<", "_",
329 | ">", "_",
330 | "|", "_",
331 | )
332 |
333 | // RegulateFilename replaces unusable characters as filepath by '_'.
334 | func RegulateFilename(s string) string {
335 | return filenameReplacer.Replace(s)
336 | }
337 |
338 | // MessageIcons represents icon for each message.
339 | type MessageIcons struct {
340 | Image48 string `json:"image_48"`
341 | }
342 |
--------------------------------------------------------------------------------
/internal/slacklog/generator.go:
--------------------------------------------------------------------------------
1 | package slacklog
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "fmt"
7 | "html"
8 | "io/ioutil"
9 | "log"
10 | "net/url"
11 | "os"
12 | "path/filepath"
13 | "regexp"
14 | "sort"
15 | "strings"
16 | "sync"
17 | "text/template"
18 |
19 | "github.com/kyokomi/emoji"
20 | "github.com/slack-go/slack"
21 | )
22 |
23 | // HTMLGenerator : ログデータからHTMLを生成するための構造体。
24 | type HTMLGenerator struct {
25 | // text/template形式のテンプレートが置いてあるディレクトリ
26 | templateDir string
27 | // files がおいてあるディレクトリ
28 | filesDir string
29 | // ログデータを取得するためのLogStore
30 | s *LogStore
31 | // markdown形式のテキストを変換するためのTextConverter
32 | c *TextConverter
33 | cfg Config
34 |
35 | // baseURL is root path for public site, configured by `BASEURL` environment variable.
36 | baseURL string
37 |
38 | // filesBaseURL is root path for attachment files, configured by `FILES_BASEURL` environment variable.
39 | filesBaseURL string
40 |
41 | // ueMap is a set of unknown emojis.
42 | ueMap map[string]struct{}
43 | ueMu sync.Mutex
44 | }
45 |
46 | // maxEmbeddedFileSize : 添付ファイルの埋め込みを行うファイルサイズ
47 | // これ以下の場合、表示されるようになる
48 | const maxEmbeddedFileSize = 102400
49 |
50 | // NewHTMLGenerator : HTMLGeneratorを生成する。
51 | func NewHTMLGenerator(templateDir string, filesDir string, s *LogStore) *HTMLGenerator {
52 | users := s.GetDisplayNameMap()
53 | emojis := s.GetEmojiMap()
54 | c := NewTextConverter(users, emojis)
55 |
56 | baseURL := os.Getenv("BASEURL")
57 | filesBaseURL := os.Getenv("FILES_BASEURL")
58 | if filesBaseURL == "" {
59 | filesBaseURL = baseURL + "/files"
60 | }
61 |
62 | return &HTMLGenerator{
63 | templateDir: templateDir,
64 | filesDir: filesDir,
65 | s: s,
66 | c: c,
67 | baseURL: baseURL,
68 | filesBaseURL: filesBaseURL,
69 | }
70 | }
71 |
72 | // Generate はoutDirにログデータの変換結果を生成する。
73 | // 目標とする構造は以下となる:
74 | // - outDir/
75 | // - index.html // generateIndex()
76 | // - ${channel_id}/ // generateChannelDir()
77 | // - index.html // generateChannelIndex()
78 | // - ${YYYY}/
79 | // - ${MM}/
80 | // - index.html // generateMessageDir()
81 | func (g *HTMLGenerator) Generate(outDir string) error {
82 | channels := g.s.GetChannels()
83 |
84 | createdChannels := []Channel{}
85 | var (
86 | wg sync.WaitGroup
87 | mu sync.Mutex
88 | errs []error
89 | )
90 | for i := range channels {
91 | wg.Add(1)
92 | go func(i int) {
93 | defer wg.Done()
94 | isCreated, err := g.generateChannelDir(
95 | filepath.Join(outDir, channels[i].ID),
96 | channels[i],
97 | )
98 | if err != nil {
99 | log.Printf("generateChannelDir(%s) failed: %s", channels[i].ID, err)
100 | mu.Lock()
101 | errs = append(errs, err)
102 | mu.Unlock()
103 | return
104 | }
105 | if isCreated {
106 | mu.Lock()
107 | createdChannels = append(createdChannels, channels[i])
108 | mu.Unlock()
109 | }
110 | }(i)
111 | }
112 | wg.Wait()
113 | if len(errs) > 0 {
114 | return errs[0]
115 | }
116 |
117 | if err := g.generateIndex(filepath.Join(outDir, "index.html"), createdChannels); err != nil {
118 | return err
119 | }
120 |
121 | return nil
122 | }
123 |
124 | func (g *HTMLGenerator) generateIndex(path string, channels []Channel) error {
125 | params := make(map[string]interface{})
126 | SortChannel(channels)
127 | params["baseURL"] = g.baseURL
128 | params["channels"] = channels
129 | tmplPath := filepath.Join(g.templateDir, "index.tmpl")
130 | name := filepath.Base(tmplPath)
131 | t, err := template.New(name).ParseFiles(tmplPath)
132 | if err != nil {
133 | return err
134 | }
135 | if err := executeAndWrite(t, params, path); err != nil {
136 | return err
137 | }
138 | return nil
139 | }
140 |
141 | func (g *HTMLGenerator) generateChannelDir(path string, channel Channel) (bool, error) {
142 | msgsMap, err := g.s.GetMessagesPerMonth(channel.ID)
143 | if err != nil {
144 | return false, err
145 | }
146 | if len(msgsMap) == 0 {
147 | return false, nil
148 | }
149 |
150 | if err := os.MkdirAll(path, 0777); err != nil {
151 | return false, fmt.Errorf("could not create %s directory: %w", path, err)
152 | }
153 |
154 | if err := g.generateChannelIndex(
155 | channel,
156 | msgsMap.Keys(),
157 | filepath.Join(path, "index.html"),
158 | ); err != nil {
159 | return true, err
160 | }
161 |
162 | for key, mm := range msgsMap {
163 | if err := g.generateMessageDir(
164 | channel,
165 | key,
166 | mm,
167 | filepath.Join(path, key.Year(), key.Month()),
168 | ); err != nil {
169 | return true, err
170 | }
171 | }
172 | return true, nil
173 | }
174 |
175 | func (g *HTMLGenerator) generateChannelIndex(channel Channel, keys []MessageMonthKey, path string) error {
176 | sort.Slice(keys, func(i, j int) bool {
177 | if keys[i].year < keys[j].year {
178 | return false
179 | } else if keys[i].year > keys[j].year {
180 | return true
181 | }
182 | return keys[i].month > keys[j].month
183 | })
184 |
185 | params := make(map[string]interface{})
186 | params["baseURL"] = g.baseURL
187 | params["channel"] = channel
188 | params["keys"] = keys
189 |
190 | tempPath := filepath.Join(g.templateDir, "channel_index.tmpl")
191 | name := filepath.Base(tempPath)
192 | t, err := template.New(name).ParseFiles(tempPath)
193 | if err != nil {
194 | return err
195 | }
196 | if err := executeAndWrite(t, params, path); err != nil {
197 | return err
198 | }
199 | return nil
200 | }
201 |
202 | func (g *HTMLGenerator) generateMessageDir(channel Channel, key MessageMonthKey, msgs Messages, path string) error {
203 | if err := os.MkdirAll(path, 0777); err != nil {
204 | return fmt.Errorf("could not create %s directory: %w", path, err)
205 | }
206 |
207 | params := make(map[string]interface{})
208 | params["baseURL"] = g.baseURL
209 | params["filesBaseURL"] = g.filesBaseURL
210 | params["channel"] = channel
211 | params["monthKey"] = key
212 | params["msgs"] = msgs
213 |
214 | // TODO check below subtypes work correctly
215 | // TODO support more subtypes
216 |
217 | t, err := template.New("").
218 | Funcs(map[string]interface{}{
219 | "visible": g.isVisibleMessage,
220 | "datetime": func(ts string) string {
221 | return TsToDateTime(ts).Format("2日 15:04:05")
222 | },
223 | "threadMessageTime": func(msgTs, threadTs string) string {
224 | return LevelOfDetailTime(TsToDateTime(msgTs), TsToDateTime(threadTs))
225 | },
226 | "slackPermalink": func(ts string) string {
227 | return strings.Replace(ts, ".", "", 1)
228 | },
229 | "username": func(msg *Message) string {
230 | if msg.Username != "" {
231 | return g.c.escapeSpecialChars(msg.Username)
232 | }
233 | return g.c.escapeSpecialChars(g.s.GetDisplayNameByUserID(msg.User))
234 | },
235 | "userIconUrl": func(msg *Message) string {
236 | if msg.Icons != nil && msg.Icons.Image48 != "" {
237 | return msg.Icons.Image48
238 | }
239 | userID := msg.User
240 | if userID == "" && msg.BotID != "" {
241 | userID = msg.BotID
242 | }
243 | user, ok := g.s.GetUserByID(userID)
244 | if !ok {
245 | return "" // TODO show default icon
246 | }
247 | return user.Profile.Image48
248 | },
249 | "text": g.generateMessageText,
250 | "reactions": g.getReactions,
251 | "attachmentText": g.generateAttachmentText,
252 | "fileHTML": g.generateFileHTML,
253 | "threadMtime": func(ts string) string {
254 | if t, ok := g.s.GetThread(channel.ID, ts); ok {
255 | return LevelOfDetailTime(t.LastReplyTime(), TsToDateTime(ts))
256 | }
257 | return ""
258 | },
259 | "threads": func(ts string) Messages {
260 | if t, ok := g.s.GetThread(channel.ID, ts); ok {
261 | return t.Replies()
262 | }
263 | return nil
264 | },
265 | "threadNum": func(ts string) int {
266 | if t, ok := g.s.GetThread(channel.ID, ts); ok {
267 | return t.ReplyCount()
268 | }
269 | return 0
270 | },
271 | "threadRootText": func(ts string) string {
272 | thread, ok := g.s.GetThread(channel.ID, ts)
273 | if !ok {
274 | return ""
275 | }
276 | runes := []rune(thread.RootText())
277 | text := string(runes)
278 | if len(runes) > 20 {
279 | text = string(runes[:20]) + " ..."
280 | }
281 | return g.c.escape(text)
282 | },
283 | "hasPrevMonth": func(key MessageMonthKey) bool {
284 | return g.s.HasPrevMonth(channel.ID, key)
285 | },
286 | "hasNextMonth": func(key MessageMonthKey) bool {
287 | return g.s.HasNextMonth(channel.ID, key)
288 | },
289 | "hostBySlack": HostBySlack,
290 | "localPath": LocalPath,
291 | "topLevelMimetype": TopLevelMimetype,
292 | "thumbImagePath": ThumbImagePath,
293 | "thumbImageWidth": ThumbImageWidth,
294 | "thumbImageHeight": ThumbImageHeight,
295 | "thumbVideoPath": ThumbVideoPath,
296 | "stringsJoin": strings.Join,
297 | "genAttachedURL": func(ts json.Number, fromURL string) string {
298 | timestamp := string(ts)
299 | if len(strings.Split(timestamp, ".")) < 2 {
300 | return ""
301 | }
302 | postTime := TsToDateTime(timestamp)
303 | c := strings.Split(fromURL, "/")
304 | if len(c) < 2 {
305 | return ""
306 | }
307 | channel := c[len(c)-2]
308 | return fmt.Sprintf("/%s/%d/%02d/#ts-%s", channel, postTime.Year(), postTime.Month(), ts)
309 | },
310 | "isSlackMessage": func(fromURL string) bool {
311 | return len(fromURL) >= 25 && fromURL[0:25] == "https://vim-jp.slack.com/"
312 | },
313 | "getBaseURL": func() string {
314 | return g.baseURL
315 | },
316 | }).
317 | ParseGlob(filepath.Join(g.templateDir, "channel_per_month", "*.tmpl"))
318 | if err != nil {
319 | return err
320 | }
321 | tmpl := t.Lookup("index.tmpl")
322 | if tmpl == nil {
323 | return errors.New("no index.tmpl in channel_per_month/ dir")
324 | }
325 | err = executeAndWrite(tmpl, params, filepath.Join(path, "index.html"))
326 | if err != nil {
327 | return err
328 | }
329 | return nil
330 | }
331 |
332 | func (g *HTMLGenerator) isVisibleMessage(msg Message) bool {
333 | return msg.SubType == "" || msg.SubType == "bot_message" || msg.SubType == "slackbot_response" || msg.SubType == "thread_broadcast"
334 | }
335 |
336 | func (g *HTMLGenerator) generateMessageText(msg Message) string {
337 | text := g.c.ToHTML(msg.Text)
338 | if msg.Edited != nil && g.cfg.EditedSuffix != "" {
339 | text += "" + html.EscapeString(g.cfg.EditedSuffix) + ""
340 | }
341 | return text
342 | }
343 |
344 | func (g *HTMLGenerator) generateAttachmentText(attachment slack.Attachment) string {
345 | return g.c.ToHTML(attachment.Text)
346 | }
347 |
348 | // generateFileHTML : 'text/plain' な添付ファイルをHTMLに埋め込む
349 | // 存在しない場合、エラーを表示する
350 | func (g *HTMLGenerator) generateFileHTML(file slack.File) string {
351 | if file.Size > maxEmbeddedFileSize {
352 | return `file size is too big to embed. please download from above link to see.`
353 | }
354 | path := filepath.Join(g.filesDir, file.ID, LocalName(file, file.URLPrivate, ""))
355 | src, err := ioutil.ReadFile(path)
356 | if err != nil {
357 | if os.IsNotExist(err) {
358 | return fmt.Sprintf(`no files found: %s`, err)
359 | }
360 | return fmt.Sprintf(`failed to read a file: %s`, err)
361 | }
362 | ftype := file.Filetype
363 | if file.Filetype == "text" {
364 | ftype = "none"
365 | }
366 | return "
22 |
23 |
165 |
166 |
167 |
172 |
173 |
24 |
32 |
33 | 25 | 28 | / 29 | slacklog 30 |
31 |
34 |
164 |
35 |
41 |
42 |
43 | {{- range .msgs }}
44 | {{- if visible . }}
45 |
46 |
161 | {{- end }}
162 | {{- end }}
163 |
47 |
160 |
48 | 
49 |
50 |
57 |
58 |
159 |
59 | {{ text . }}
60 |
61 |
62 |
63 | {{- $rs := reactions .}}
64 | {{- if $rs }}
65 |
82 |
83 |
84 | {{- if and (ne .ThreadTimestamp "") (ne .ThreadTimestamp .Timestamp) }}
85 |
86 | このスレッドに返信しました : {{- threadRootText .ThreadTimestamp }}
87 |
88 | {{- end }}
89 |
90 | {{- if .Attachments }}
91 |
66 |
80 | {{- end }}
81 | -
67 | {{- range $rs }}
68 |
-
69 |
70 | {{- if not .Default }}
71 |
{{ .Count }} 72 | {{- else }} 73 | {{ .Name }}{{ .Count }} 74 | {{- end }} 75 | 76 |
77 | {{- end }}
78 |
92 | {{ template "attachment.tmpl" . }}
93 |
94 | {{- end }}
95 |
96 | {{- if .Files }}
97 |
98 | {{- range .Files }}
99 |
121 | {{- end }}
122 |
123 | {{- if threads .Timestamp }}
124 |
100 | {{- if hostBySlack . }}
101 |
102 | {{- if eq (topLevelMimetype .) "image" }}
103 | 
104 | {{- else if eq (topLevelMimetype .) "video" }}
105 |
107 | {{- else }}
108 | [[ダウンロード: {{ .Title }}({{ .PrettyType }})]]
109 | {{- end }}
110 |
111 | {{- if eq .Mimetype "text/plain" }}
112 |
119 | {{- end }}
120 | {{ fileHTML . }}
113 | {{- end }}
114 | {{- else }}
115 | {{- /* TODO: 外部ページへのリンクだということがわかりやすい表示にする */ -}}
116 | {{ .Title }}
117 | {{- end }}
118 |
125 |
157 | {{- end }}
158 | 126 | 127 | {{- threadNum .ThreadTimestamp }} 件の返信 最終返信:{{- threadMtime .ThreadTimestamp }} 128 | 129 |
130 |
131 | {{- range threads .Timestamp }}
132 |
156 |
133 | {{- if eq .SubType "thread_broadcast" }}
134 |
154 | {{- end }}
155 |
135 |
138 | {{- end }}
139 | #←
136 | チャンネルにも投稿済
137 |
140 |
141 |
142 |
143 | {{ username . }}
144 | {{ datetime .Timestamp }}
145 |
146 |
147 | {{ text . }}
148 |
149 |
150 | {{- if .Attachments }}
151 | {{ template "attachment.tmpl" . }}
152 | {{- end }}
153 | " + html.EscapeString(string(src)) + ""
367 | }
368 |
369 | // ReactionInfo is information for a reaction.
370 | type ReactionInfo struct {
371 | EmojiPath string
372 | Name string
373 | Count int
374 | Users []string
375 | Default bool
376 | }
377 |
378 | func (g *HTMLGenerator) getReactions(msg Message) []ReactionInfo {
379 | var info []ReactionInfo
380 |
381 | for _, reaction := range msg.Reactions {
382 | users := make([]string, 0, len(reaction.Users))
383 | for _, user := range reaction.Users {
384 | n := g.s.GetDisplayNameByUserID(user)
385 | if n == "" {
386 | continue
387 | }
388 | users = append(users, n)
389 | }
390 |
391 | // custom emoji case
392 | emojiExt, ok := g.s.et.NameToExt[reaction.Name]
393 | if ok {
394 | info = append(info, ReactionInfo{
395 | EmojiPath: url.PathEscape(reaction.Name + emojiExt),
396 | Name: reaction.Name,
397 | Count: reaction.Count,
398 | Users: users,
399 | Default: false,
400 | })
401 | continue
402 | }
403 |
404 | // fallback to unicode.
405 | emojiStr := ":" + reaction.Name + ":"
406 | unicodeEmojis := g.emojiToString(emojiStr)
407 | if unicodeEmojis == "" {
408 | // This may be a deleted emoji. Show `:emoji:` as is.
409 | unicodeEmojis = emojiStr
410 | }
411 | info = append(info, ReactionInfo{
412 | Name: unicodeEmojis,
413 | Count: reaction.Count,
414 | Users: users,
415 | Default: true,
416 | })
417 | }
418 |
419 | return info
420 | }
421 |
422 | var rxEmoji = regexp.MustCompile(`:[^:]+:`)
423 |
424 | func (g *HTMLGenerator) emojiToString(emojiSeq string) string {
425 | b := &strings.Builder{}
426 | for _, s := range rxEmoji.FindAllString(emojiSeq, -1) {
427 | ch, ok := emoji.CodeMap()[s]
428 | if !ok {
429 | g.ueMu.Lock()
430 | if g.ueMap == nil {
431 | g.ueMap = map[string]struct{}{}
432 | }
433 | if _, ok := g.ueMap[s]; !ok {
434 | g.ueMap[s] = struct{}{}
435 | }
436 | g.ueMu.Unlock()
437 | continue
438 | }
439 | b.WriteString(ch)
440 | }
441 | return b.String()
442 | }
443 |
444 | // executeAndWrite executes a template and writes contents to a file.
445 | func executeAndWrite(tmpl *template.Template, data interface{}, filename string) error {
446 | f, err := os.Create(filename)
447 | if err != nil {
448 | return err
449 | }
450 | defer f.Close()
451 | err = tmpl.Execute(f, data)
452 | if err != nil {
453 | return err
454 | }
455 | return nil
456 | }
457 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | Attribution 4.0 International
2 |
3 | =======================================================================
4 |
5 | Creative Commons Corporation ("Creative Commons") is not a law firm and
6 | does not provide legal services or legal advice. Distribution of
7 | Creative Commons public licenses does not create a lawyer-client or
8 | other relationship. Creative Commons makes its licenses and related
9 | information available on an "as-is" basis. Creative Commons gives no
10 | warranties regarding its licenses, any material licensed under their
11 | terms and conditions, or any related information. Creative Commons
12 | disclaims all liability for damages resulting from their use to the
13 | fullest extent possible.
14 |
15 | Using Creative Commons Public Licenses
16 |
17 | Creative Commons public licenses provide a standard set of terms and
18 | conditions that creators and other rights holders may use to share
19 | original works of authorship and other material subject to copyright
20 | and certain other rights specified in the public license below. The
21 | following considerations are for informational purposes only, are not
22 | exhaustive, and do not form part of our licenses.
23 |
24 | Considerations for licensors: Our public licenses are
25 | intended for use by those authorized to give the public
26 | permission to use material in ways otherwise restricted by
27 | copyright and certain other rights. Our licenses are
28 | irrevocable. Licensors should read and understand the terms
29 | and conditions of the license they choose before applying it.
30 | Licensors should also secure all rights necessary before
31 | applying our licenses so that the public can reuse the
32 | material as expected. Licensors should clearly mark any
33 | material not subject to the license. This includes other CC-
34 | licensed material, or material used under an exception or
35 | limitation to copyright. More considerations for licensors:
36 | wiki.creativecommons.org/Considerations_for_licensors
37 |
38 | Considerations for the public: By using one of our public
39 | licenses, a licensor grants the public permission to use the
40 | licensed material under specified terms and conditions. If
41 | the licensor's permission is not necessary for any reason--for
42 | example, because of any applicable exception or limitation to
43 | copyright--then that use is not regulated by the license. Our
44 | licenses grant only permissions under copyright and certain
45 | other rights that a licensor has authority to grant. Use of
46 | the licensed material may still be restricted for other
47 | reasons, including because others have copyright or other
48 | rights in the material. A licensor may make special requests,
49 | such as asking that all changes be marked or described.
50 | Although not required by our licenses, you are encouraged to
51 | respect those requests where reasonable. More considerations
52 | for the public:
53 | wiki.creativecommons.org/Considerations_for_licensees
54 |
55 | =======================================================================
56 |
57 | Creative Commons Attribution 4.0 International Public License
58 |
59 | By exercising the Licensed Rights (defined below), You accept and agree
60 | to be bound by the terms and conditions of this Creative Commons
61 | Attribution 4.0 International Public License ("Public License"). To the
62 | extent this Public License may be interpreted as a contract, You are
63 | granted the Licensed Rights in consideration of Your acceptance of
64 | these terms and conditions, and the Licensor grants You such rights in
65 | consideration of benefits the Licensor receives from making the
66 | Licensed Material available under these terms and conditions.
67 |
68 |
69 | Section 1 -- Definitions.
70 |
71 | a. Adapted Material means material subject to Copyright and Similar
72 | Rights that is derived from or based upon the Licensed Material
73 | and in which the Licensed Material is translated, altered,
74 | arranged, transformed, or otherwise modified in a manner requiring
75 | permission under the Copyright and Similar Rights held by the
76 | Licensor. For purposes of this Public License, where the Licensed
77 | Material is a musical work, performance, or sound recording,
78 | Adapted Material is always produced where the Licensed Material is
79 | synched in timed relation with a moving image.
80 |
81 | b. Adapter's License means the license You apply to Your Copyright
82 | and Similar Rights in Your contributions to Adapted Material in
83 | accordance with the terms and conditions of this Public License.
84 |
85 | c. Copyright and Similar Rights means copyright and/or similar rights
86 | closely related to copyright including, without limitation,
87 | performance, broadcast, sound recording, and Sui Generis Database
88 | Rights, without regard to how the rights are labeled or
89 | categorized. For purposes of this Public License, the rights
90 | specified in Section 2(b)(1)-(2) are not Copyright and Similar
91 | Rights.
92 |
93 | d. Effective Technological Measures means those measures that, in the
94 | absence of proper authority, may not be circumvented under laws
95 | fulfilling obligations under Article 11 of the WIPO Copyright
96 | Treaty adopted on December 20, 1996, and/or similar international
97 | agreements.
98 |
99 | e. Exceptions and Limitations means fair use, fair dealing, and/or
100 | any other exception or limitation to Copyright and Similar Rights
101 | that applies to Your use of the Licensed Material.
102 |
103 | f. Licensed Material means the artistic or literary work, database,
104 | or other material to which the Licensor applied this Public
105 | License.
106 |
107 | g. Licensed Rights means the rights granted to You subject to the
108 | terms and conditions of this Public License, which are limited to
109 | all Copyright and Similar Rights that apply to Your use of the
110 | Licensed Material and that the Licensor has authority to license.
111 |
112 | h. Licensor means the individual(s) or entity(ies) granting rights
113 | under this Public License.
114 |
115 | i. Share means to provide material to the public by any means or
116 | process that requires permission under the Licensed Rights, such
117 | as reproduction, public display, public performance, distribution,
118 | dissemination, communication, or importation, and to make material
119 | available to the public including in ways that members of the
120 | public may access the material from a place and at a time
121 | individually chosen by them.
122 |
123 | j. Sui Generis Database Rights means rights other than copyright
124 | resulting from Directive 96/9/EC of the European Parliament and of
125 | the Council of 11 March 1996 on the legal protection of databases,
126 | as amended and/or succeeded, as well as other essentially
127 | equivalent rights anywhere in the world.
128 |
129 | k. You means the individual or entity exercising the Licensed Rights
130 | under this Public License. Your has a corresponding meaning.
131 |
132 |
133 | Section 2 -- Scope.
134 |
135 | a. License grant.
136 |
137 | 1. Subject to the terms and conditions of this Public License,
138 | the Licensor hereby grants You a worldwide, royalty-free,
139 | non-sublicensable, non-exclusive, irrevocable license to
140 | exercise the Licensed Rights in the Licensed Material to:
141 |
142 | a. reproduce and Share the Licensed Material, in whole or
143 | in part; and
144 |
145 | b. produce, reproduce, and Share Adapted Material.
146 |
147 | 2. Exceptions and Limitations. For the avoidance of doubt, where
148 | Exceptions and Limitations apply to Your use, this Public
149 | License does not apply, and You do not need to comply with
150 | its terms and conditions.
151 |
152 | 3. Term. The term of this Public License is specified in Section
153 | 6(a).
154 |
155 | 4. Media and formats; technical modifications allowed. The
156 | Licensor authorizes You to exercise the Licensed Rights in
157 | all media and formats whether now known or hereafter created,
158 | and to make technical modifications necessary to do so. The
159 | Licensor waives and/or agrees not to assert any right or
160 | authority to forbid You from making technical modifications
161 | necessary to exercise the Licensed Rights, including
162 | technical modifications necessary to circumvent Effective
163 | Technological Measures. For purposes of this Public License,
164 | simply making modifications authorized by this Section 2(a)
165 | (4) never produces Adapted Material.
166 |
167 | 5. Downstream recipients.
168 |
169 | a. Offer from the Licensor -- Licensed Material. Every
170 | recipient of the Licensed Material automatically
171 | receives an offer from the Licensor to exercise the
172 | Licensed Rights under the terms and conditions of this
173 | Public License.
174 |
175 | b. No downstream restrictions. You may not offer or impose
176 | any additional or different terms or conditions on, or
177 | apply any Effective Technological Measures to, the
178 | Licensed Material if doing so restricts exercise of the
179 | Licensed Rights by any recipient of the Licensed
180 | Material.
181 |
182 | 6. No endorsement. Nothing in this Public License constitutes or
183 | may be construed as permission to assert or imply that You
184 | are, or that Your use of the Licensed Material is, connected
185 | with, or sponsored, endorsed, or granted official status by,
186 | the Licensor or others designated to receive attribution as
187 | provided in Section 3(a)(1)(A)(i).
188 |
189 | b. Other rights.
190 |
191 | 1. Moral rights, such as the right of integrity, are not
192 | licensed under this Public License, nor are publicity,
193 | privacy, and/or other similar personality rights; however, to
194 | the extent possible, the Licensor waives and/or agrees not to
195 | assert any such rights held by the Licensor to the limited
196 | extent necessary to allow You to exercise the Licensed
197 | Rights, but not otherwise.
198 |
199 | 2. Patent and trademark rights are not licensed under this
200 | Public License.
201 |
202 | 3. To the extent possible, the Licensor waives any right to
203 | collect royalties from You for the exercise of the Licensed
204 | Rights, whether directly or through a collecting society
205 | under any voluntary or waivable statutory or compulsory
206 | licensing scheme. In all other cases the Licensor expressly
207 | reserves any right to collect such royalties.
208 |
209 |
210 | Section 3 -- License Conditions.
211 |
212 | Your exercise of the Licensed Rights is expressly made subject to the
213 | following conditions.
214 |
215 | a. Attribution.
216 |
217 | 1. If You Share the Licensed Material (including in modified
218 | form), You must:
219 |
220 | a. retain the following if it is supplied by the Licensor
221 | with the Licensed Material:
222 |
223 | i. identification of the creator(s) of the Licensed
224 | Material and any others designated to receive
225 | attribution, in any reasonable manner requested by
226 | the Licensor (including by pseudonym if
227 | designated);
228 |
229 | ii. a copyright notice;
230 |
231 | iii. a notice that refers to this Public License;
232 |
233 | iv. a notice that refers to the disclaimer of
234 | warranties;
235 |
236 | v. a URI or hyperlink to the Licensed Material to the
237 | extent reasonably practicable;
238 |
239 | b. indicate if You modified the Licensed Material and
240 | retain an indication of any previous modifications; and
241 |
242 | c. indicate the Licensed Material is licensed under this
243 | Public License, and include the text of, or the URI or
244 | hyperlink to, this Public License.
245 |
246 | 2. You may satisfy the conditions in Section 3(a)(1) in any
247 | reasonable manner based on the medium, means, and context in
248 | which You Share the Licensed Material. For example, it may be
249 | reasonable to satisfy the conditions by providing a URI or
250 | hyperlink to a resource that includes the required
251 | information.
252 |
253 | 3. If requested by the Licensor, You must remove any of the
254 | information required by Section 3(a)(1)(A) to the extent
255 | reasonably practicable.
256 |
257 | 4. If You Share Adapted Material You produce, the Adapter's
258 | License You apply must not prevent recipients of the Adapted
259 | Material from complying with this Public License.
260 |
261 |
262 | Section 4 -- Sui Generis Database Rights.
263 |
264 | Where the Licensed Rights include Sui Generis Database Rights that
265 | apply to Your use of the Licensed Material:
266 |
267 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right
268 | to extract, reuse, reproduce, and Share all or a substantial
269 | portion of the contents of the database;
270 |
271 | b. if You include all or a substantial portion of the database
272 | contents in a database in which You have Sui Generis Database
273 | Rights, then the database in which You have Sui Generis Database
274 | Rights (but not its individual contents) is Adapted Material; and
275 |
276 | c. You must comply with the conditions in Section 3(a) if You Share
277 | all or a substantial portion of the contents of the database.
278 |
279 | For the avoidance of doubt, this Section 4 supplements and does not
280 | replace Your obligations under this Public License where the Licensed
281 | Rights include other Copyright and Similar Rights.
282 |
283 |
284 | Section 5 -- Disclaimer of Warranties and Limitation of Liability.
285 |
286 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
287 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
288 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
289 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
290 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
291 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
292 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
293 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
294 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
295 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
296 |
297 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
298 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
299 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
300 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
301 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
302 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
303 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
304 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
305 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
306 |
307 | c. The disclaimer of warranties and limitation of liability provided
308 | above shall be interpreted in a manner that, to the extent
309 | possible, most closely approximates an absolute disclaimer and
310 | waiver of all liability.
311 |
312 |
313 | Section 6 -- Term and Termination.
314 |
315 | a. This Public License applies for the term of the Copyright and
316 | Similar Rights licensed here. However, if You fail to comply with
317 | this Public License, then Your rights under this Public License
318 | terminate automatically.
319 |
320 | b. Where Your right to use the Licensed Material has terminated under
321 | Section 6(a), it reinstates:
322 |
323 | 1. automatically as of the date the violation is cured, provided
324 | it is cured within 30 days of Your discovery of the
325 | violation; or
326 |
327 | 2. upon express reinstatement by the Licensor.
328 |
329 | For the avoidance of doubt, this Section 6(b) does not affect any
330 | right the Licensor may have to seek remedies for Your violations
331 | of this Public License.
332 |
333 | c. For the avoidance of doubt, the Licensor may also offer the
334 | Licensed Material under separate terms or conditions or stop
335 | distributing the Licensed Material at any time; however, doing so
336 | will not terminate this Public License.
337 |
338 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
339 | License.
340 |
341 |
342 | Section 7 -- Other Terms and Conditions.
343 |
344 | a. The Licensor shall not be bound by any additional or different
345 | terms or conditions communicated by You unless expressly agreed.
346 |
347 | b. Any arrangements, understandings, or agreements regarding the
348 | Licensed Material not stated herein are separate from and
349 | independent of the terms and conditions of this Public License.
350 |
351 |
352 | Section 8 -- Interpretation.
353 |
354 | a. For the avoidance of doubt, this Public License does not, and
355 | shall not be interpreted to, reduce, limit, restrict, or impose
356 | conditions on any use of the Licensed Material that could lawfully
357 | be made without permission under this Public License.
358 |
359 | b. To the extent possible, if any provision of this Public License is
360 | deemed unenforceable, it shall be automatically reformed to the
361 | minimum extent necessary to make it enforceable. If the provision
362 | cannot be reformed, it shall be severed from this Public License
363 | without affecting the enforceability of the remaining terms and
364 | conditions.
365 |
366 | c. No term or condition of this Public License will be waived and no
367 | failure to comply consented to unless expressly agreed to by the
368 | Licensor.
369 |
370 | d. Nothing in this Public License constitutes or may be interpreted
371 | as a limitation upon, or waiver of, any privileges and immunities
372 | that apply to the Licensor or You, including from the legal
373 | processes of any jurisdiction or authority.
374 |
375 |
376 | =======================================================================
377 |
378 | Creative Commons is not a party to its public
379 | licenses. Notwithstanding, Creative Commons may elect to apply one of
380 | its public licenses to material it publishes and in those instances
381 | will be considered the “Licensor.” The text of the Creative Commons
382 | public licenses is dedicated to the public domain under the CC0 Public
383 | Domain Dedication. Except for the limited purpose of indicating that
384 | material is shared under a Creative Commons public license or as
385 | otherwise permitted by the Creative Commons policies published at
386 | creativecommons.org/policies, Creative Commons does not authorize the
387 | use of the trademark "Creative Commons" or any other trademark or logo
388 | of Creative Commons without its prior written consent including,
389 | without limitation, in connection with any unauthorized modifications
390 | to any of its public licenses or any other arrangements,
391 | understandings, or agreements concerning use of licensed material. For
392 | the avoidance of doubt, this paragraph does not form part of the
393 | public licenses.
394 |
395 | Creative Commons may be contacted at creativecommons.org.
396 |
397 |
--------------------------------------------------------------------------------